Skip to content

Commit b408532

Browse files
etpetersonetpeterson
committed
More advanced testing parameters
1 parent 70f1b88 commit b408532

File tree

2 files changed

+120
-74
lines changed

2 files changed

+120
-74
lines changed

‎tests/IVIMmodels/unit_tests/algorithms.json

Lines changed: 91 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -9,94 +9,125 @@
99
"IAR_LU_subtracted"
1010
],
1111
"IAR_LU_biexp": {
12-
"xfail_names": [
13-
"Vein",
14-
"Blood RV",
15-
"Blood LV",
16-
"Blood RA"
17-
],
1812
"tolerances": {
19-
"f": 5,
20-
"D": 5,
21-
"Dp": 25
13+
"rtol": {
14+
"f": 5,
15+
"D": 5,
16+
"Dp": 25
17+
},
18+
"atol": {
19+
"f": 1e-2,
20+
"D": 1e-2,
21+
"Dp": 1e-1
22+
}
2223
}
2324
},
2425
"IAR_LU_modified_mix": {
25-
"xfail_names": [
26-
"Blood LV"
27-
],
2826
"tolerances": {
29-
"f": 5,
30-
"D": 5,
31-
"Dp": 25
27+
"rtol": {
28+
"f": 5,
29+
"D": 5,
30+
"Dp": 25
31+
},
32+
"atol": {
33+
"f": 1e-2,
34+
"D": 1e-2,
35+
"Dp": 1e-1
36+
}
3237
}
3338
},
3439
"IAR_LU_modified_topopro": {
35-
"xfail_names": [
36-
"Vein",
37-
"Blood RV",
38-
"Blood LV",
39-
"Blood RA"
40-
],
4140
"tolerances": {
42-
"f": 5,
43-
"D": 5,
44-
"Dp": 25
41+
"rtol": {
42+
"f": 5,
43+
"D": 5,
44+
"Dp": 25
45+
},
46+
"atol": {
47+
"f": 1e-2,
48+
"D": 1e-2,
49+
"Dp": 1e-1
50+
}
4551
}
4652
},
4753
"IAR_LU_segmented_2step": {
48-
"xfail_names": [
49-
"Blood RV",
50-
"Blood LV",
51-
"Blood RA"
52-
],
5354
"tolerances": {
54-
"f": 5,
55-
"D": 5,
56-
"Dp": 25
55+
"rtol": {
56+
"f": 5,
57+
"D": 5,
58+
"Dp": 25
59+
},
60+
"atol": {
61+
"f": 1e-2,
62+
"D": 1e-2,
63+
"Dp": 1e-1
64+
}
5765
}
5866
},
5967
"IAR_LU_segmented_3step": {
60-
"xfail_names": [
61-
"Blood RV",
62-
"Blood LV",
63-
"Blood RA"
64-
],
6568
"tolerances": {
66-
"f": 5,
67-
"D": 5,
68-
"Dp": 25
69+
"rtol": {
70+
"f": 5,
71+
"D": 5,
72+
"Dp": 25
73+
},
74+
"atol": {
75+
"f": 1e-2,
76+
"D": 1e-2,
77+
"Dp": 1e-1
78+
}
6979
}
7080
},
7181
"IAR_LU_subtracted": {
72-
"xfail_names": [
73-
"Blood RV",
74-
"Blood LV",
75-
"Blood RA"
76-
],
7782
"tolerances": {
78-
"f": 5,
79-
"D": 5,
80-
"Dp": 25
83+
"rtol": {
84+
"f": 5,
85+
"D": 5,
86+
"Dp": 25
87+
},
88+
"atol": {
89+
"f": 1e-2,
90+
"D": 1e-2,
91+
"Dp": 1e-1
92+
}
8193
}
8294
},
8395
"ETP_SRI_LinearFitting": {
84-
"xfail_names": [
85-
"Artery",
86-
"Blood RV",
87-
"Blood LV",
88-
"myocardium RV",
89-
"myocardium ra",
90-
"Liver",
91-
"Blood RA"
92-
],
96+
"xfail_names": {
97+
"Vein": true
98+
},
9399
"options": {
94100
"thresholds": [500]
95101
},
96102
"tolerances": {
97-
"f": 5,
98-
"D": 5,
99-
"Dp": 25
103+
"rtol": {
104+
"f": 5,
105+
"D": 5,
106+
"Dp": 25
107+
},
108+
"atol": {
109+
"f": 0,
110+
"D": 0,
111+
"Dp": 0
112+
},
113+
"dynamic_rtol": {
114+
"offset": 0,
115+
"noise": 0,
116+
"ratio": 0,
117+
"noiseCrossRatio": 10,
118+
"f": 1,
119+
"D": 1,
120+
"Dp": 2
121+
},
122+
"dynamic_atol": {
123+
"offset": 5e-2,
124+
"noise": 0,
125+
"ratio": 0,
126+
"noiseCrossRatio": 10,
127+
"f": 1,
128+
"D": 1,
129+
"Dp": 2
130+
}
100131
}
101132
}
102133
}

‎tests/IVIMmodels/unit_tests/test_ivim_fit.py

Lines changed: 29 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -75,29 +75,44 @@ def data_ivim_fit_saved():
7575
for name, data in all_data.items():
7676
for algorithm in algorithms:
7777
algorithm_dict = algorithm_information.get(algorithm, {})
78-
xfail = name in algorithm_dict.get("xfail_names", [])
78+
xfail = {"xfail": name in algorithm_dict.get("xfail_names", {}),
79+
"strict": algorithm_dict.get("xfail_names", {}).get(name, True)}
7980
kwargs = algorithm_dict.get("options", {})
80-
tolerances = algorithm_dict.get("tolerances", None)
81+
tolerances = algorithm_dict.get("tolerances", {})
8182
yield name, bvals, data, algorithm, xfail, kwargs, tolerances
8283

8384

8485
@pytest.mark.parametrize("name, bvals, data, algorithm, xfail, kwargs, tolerances", data_ivim_fit_saved())
8586
def test_ivim_fit_saved(name, bvals, data, algorithm, xfail, kwargs, tolerances, request):
86-
if xfail:
87-
mark = pytest.mark.xfail(reason="xfail", strict=True)
87+
if xfail["xfail"]:
88+
mark = pytest.mark.xfail(reason="xfail", strict=xfail["strict"])
8889
request.node.add_marker(mark)
8990
fit = OsipiBase(algorithm=algorithm, **kwargs)
9091
signal = np.asarray(data['data'])
9192
signal = np.abs(signal)
9293
signal /= signal[0]
94+
ratio = 1 / signal[0]
9395
# has_negatives = np.any(signal<0)
94-
if tolerances is None:
95-
# signal = np.abs(signal)
96-
# ratio = 1 / signal[0]
97-
# signal /= signal[0]
98-
# tolerance = 1e-2 + 25 * data['noise'] * ratio # totally empirical
99-
# tolerance = 1
100-
tolerances = {"f": 5, "D": 5, "Dp": 25}
96+
if "dynamic_rtol" in tolerances:
97+
dyn_rtol = tolerances["dynamic_rtol"]
98+
scale = dyn_rtol["offset"] + dyn_rtol["ratio"]*ratio + dyn_rtol["noise"]*data["noise"] + dyn_rtol["noiseCrossRatio"]*ratio*data["noise"]
99+
tolerances["rtol"] = {"f": scale*dyn_rtol["f"], "D": scale*dyn_rtol["D"], "Dp": scale*dyn_rtol["Dp"]}
100+
else:
101+
tolerances["rtol"] = tolerances.get("rtol", {"f": 5, "D": 5, "Dp": 25})
102+
if "dynamic_atol" in tolerances:
103+
dyn_atol = tolerances["dynamic_atol"]
104+
scale = dyn_atol["offset"] + dyn_atol["ratio"]*ratio + dyn_atol["noise"]*data["noise"] + dyn_atol["noiseCrossRatio"]*ratio*data["noise"]
105+
tolerances["atol"] = {"f": scale*dyn_atol["f"], "D": scale*dyn_atol["D"], "Dp": scale*dyn_atol["Dp"]}
106+
else:
107+
tolerances["atol"] = tolerances.get("atol", {"f": 1e-2, "D": 1e-2, "Dp": 1e-1})
108+
# if tolerances:
109+
# # signal = np.abs(signal)
110+
# # ratio = 1 / signal[0]
111+
# # signal /= signal[0]
112+
# # tolerance = 1e-2 + 25 * data['noise'] * ratio # totally empirical
113+
# # tolerance = 1
114+
# tolerances = {"rtol": {"f": 5, "D": 5, "Dp": 25},
115+
# "atol": {"f": 1e-2, "D": 1e-2, "Dp": 1e-1}}
101116
#if has_negatives: # this fitting doesn't do well with negatives
102117
# tolerance += 1
103118
#if data['f'] == 1.0:
@@ -108,6 +123,6 @@ def test_ivim_fit_saved(name, bvals, data, algorithm, xfail, kwargs, tolerances,
108123
#npt.assert_allclose([data['f'], data['D']], [f_fit, D_fit], atol=tolerance)
109124
#npt.assert_allclose(data['Dp'], Dp_fit, atol=1e-1) # go easy on the perfusion as it's a linear fake
110125
[f_fit, Dp_fit, D_fit] = fit.ivim_fit(signal, bvals)
111-
npt.assert_allclose(data['f'], f_fit, rtol=tolerances["f"])
112-
npt.assert_allclose(data['D'], D_fit, rtol=tolerances["D"])
113-
npt.assert_allclose(data['Dp'], Dp_fit, rtol=tolerances["Dp"]) # go easy on the perfusion as it's a linear fake
126+
npt.assert_allclose(data['f'], f_fit, rtol=tolerances["rtol"]["f"], atol=tolerances["atol"]["f"])
127+
npt.assert_allclose(data['D'], D_fit, rtol=tolerances["rtol"]["D"], atol=tolerances["atol"]["D"])
128+
npt.assert_allclose(data['Dp'], Dp_fit, rtol=tolerances["rtol"]["Dp"], atol=tolerances["atol"]["Dp"])

0 commit comments

Comments
 (0)