Merge branch 'main' into main

Author: Unique-Usman

.github/workflows/website.yml

@@ -46,9 +46,20 @@ jobs:
         with:
           name: 'Data'
 
+      - name: Run the test that generates the plots report.
+        run: |
+          pytest tests/IVIMmodels/unit_tests/test_ivim_fit.py --json-report
+          mv .report.json utilities/
+          python utilities/report-summary.py .report.json report-summary.json
+
+      - name: 'Filter and compress results file.'
+        run: python utilities/reduce_output_size.py test_output.csv test_output.csv.gz
+
       - name: move data to the dashboard folder
         run: |
-          mv test_output.csv website/dashboard
+          mv test_output.csv.gz website/dashboard
+          mv report-summary.json website/dashboard
+
 
       - name: Build documentation
         run: |

.gitignore

@@ -10,6 +10,8 @@ __pycache__/
 *.raw
 bvals.txt
 download
+.Introduction_to_TF24_IVIM-MRI_CodeCollection_github_and_IVIM_Analysis_using_Python.ipynb
+.ipynb_checkpoints
 md5sums.txt
 *.gz
 *.zip
@@ -22,3 +24,5 @@ md5sums.txt
 nosetests.xml
 coverage.xml
 *.pyc
+phantoms/MR_XCAT_qMRI/*.json
+phantoms/MR_XCAT_qMRI/*.txt

README.md

@@ -6,6 +6,8 @@ The ISMRM Open Science Initiative for Perfusion Imaging (OSIPI) is an initiative
 
 This **IVIM code collection** code library is maintained by OSIPI [Taskforce 2.4](https://www.osipi.org/task-force-2-4/) (*currently not available*) and aims to collect, test and share open-source code related to intravoxel incoherent motion (IVIM) analysis of diffusion encoded MRI data to be used in research and software development. Code contributions can include any code related IVIM analysis (denoising, motion correction, model fitting, etc.), but at an initial phase, development of tests and other features of the repository will predominantly focus on fitting algorithms. A goal of the IVIM OSIPI task force is to develop a fully tested and harmonized code library, building upon the contributions obtained through this initiative. Documentation and analysis are available on the [OSIPI TF2.4](https://osipi.github.io/TF2.4_IVIM-MRI_CodeCollection/).
 
+We have some useful tools and further documentation on https://osipi.github.io/TF2.4_IVIM-MRI_CodeCollection/.
+
 ## How to contribute
 
 If you would like to get involve in OSIPI and work within the task force, please email the contacts listed on our website.
@@ -17,6 +19,9 @@ If you would like to contribute with code, please follow the instructions below:
 *   [Guidelines for IVIM code contribution](doc/guidelines_for_contributions.md)
 *   [Guidelines to creating a test file](doc/creating_test.md) 
 
+If you would like to use code from the repository and/or are new to Github or IVIM, please see the jupyter notebook below:
+*   [Introduction to TF2.4_IVIM-MRI_CodeCollection github and IVIM Analysis using Python](doc/Introduction_to_TF24_IVIM-MRI_CodeCollection_github_and_IVIM_Analysis_using_Python.ipynb)
+
 ## Repository Organization
 
 The repository is organized in four main folders along with configuration files for automated testing. 
@@ -32,4 +37,4 @@ The **utils** folder contains various helpful tools.
 ## View Testing Reports
 [![Unit tests](https://github.com/OSIPI/TF2.4_IVIM-MRI_CodeCollection/actions/workflows/unit_test.yml/badge.svg?branch=main)](https://github.com/OSIPI/TF2.4_IVIM-MRI_CodeCollection/actions/workflows/unit_test.yml)
 [![Algorithm Analysis](https://github.com/OSIPI/TF2.4_IVIM-MRI_CodeCollection/actions/workflows/analysis.yml/badge.svg?branch=main)](https://github.com/OSIPI/TF2.4_IVIM-MRI_CodeCollection/actions/workflows/analysis.yml)
-[![Build & Deploy Documentation](https://github.com/OSIPI/TF2.4_IVIM-MRI_CodeCollection/actions/workflows/documentation.yml/badge.svg?branch=main)](https://github.com/OSIPI/TF2.4_IVIM-MRI_CodeCollection/actions/workflows/documentation.yml)
+[![Build & Deploy Website](https://github.com/OSIPI/TF2.4_IVIM-MRI_CodeCollection/actions/workflows/website.yml/badge.svg?branch=main)](https://github.com/OSIPI/TF2.4_IVIM-MRI_CodeCollection/actions/workflows/website.yml)

doc/Introduction_to_TF24_IVIM-MRI_CodeCollection_github_and_IVIM_Analysis_using_Python.ipynb

@@ -11,7 +11,7 @@
 # code adapted from MAtlab code by Eric Schrauben: https://github.com/schrau24/XCAT-ERIC
 # This code generates a 4D IVIM phantom as nifti file
 
-def phantom(bvalue, noise, TR=3000, TE=40, motion=False, rician=False, interleaved=False):
+def phantom(bvalue, noise, TR=3000, TE=40, motion=False, rician=False, interleaved=False,T1T2=True):
     np.random.seed(42)
     if motion:
         states = range(1,21)
@@ -23,10 +23,10 @@ def phantom(bvalue, noise, TR=3000, TE=40, motion=False, rician=False, interleav
 
         # Access the variables in the loaded .mat file
         XCAT = mat_data['IMG']
-        XCAT = XCAT[0:-1:2,0:-1:2,10:160:4]
+        XCAT = XCAT[-1:0:-2,-1:0:-2,10:160:4]
 
         D, f, Ds = contrast_curve_calc()
-        S, Dim, fim, Dpim, legend = XCAT_to_MR_DCE(XCAT, TR, TE, bvalue, D, f, Ds)
+        S, Dim, fim, Dpim, legend = XCAT_to_MR_DCE(XCAT, TR, TE, bvalue, D, f, Ds,T1T2=T1T2)
         if state == 1:
             Dim_out = Dim
             fim_out = fim
@@ -95,15 +95,19 @@ def contrast_curve_calc():
     D[8] = 3e-3  # 8 Blood ra
     D[10] = 1.37e-3  # 8 Muscle: 10.3389/fresc.2022.910068
     D[13] = 1.5e-3  # 13 liver: Delattre et al. doi: 10.1097/RLI.0b013e31826ef901
+    D[14] = 3.0e-3  # 13 Galbladder
     D[17] = 1.67e-3  # 17 esophagus : Huang et al. doi: 10.1259/bjr.20170421
     D[18] = 1.67e-3  # 18 esophagus cont : Huang et al. doi: 10.1259/bjr.20170421
     D[20] = 1.5e-3  # 20 stomach wall: Li et al. doi: 10.3389/fonc.2022.821586
+    D[21] = 3.0e-3  # 21 stomach content
     D[22] = 1.3e-3  # 22 Pancreas (from literature)
     D[23] = 2.12e-3  # 23 right kydney cortex : van Baalen et al. Doi: jmri.25519
     D[24] = 2.09e-3  # 23 right kydney medulla : van Baalen et al. Doi: jmri.25519
     D[25] = 2.12e-3  # 23 left kydney cortex : van Baalen et al. Doi: jmri.25519
     D[26] = 2.09e-3  # 23 left kydney medulla : van Baalen et al. Doi: jmri.25519
     D[30] = 1.3e-3  # 30 spleen : Taimouri et al. Doi: 10.1118/1.4915495
+    D[34] = 4.1e-4  # 34 spinal cord :doi: 10.3389/fonc.2022.961473
+    D[35] = 0.43e-3  # 35 Bone marrow : https://pubmed.ncbi.nlm.nih.gov/30194746/
     D[36] = 3e-3  # 36 artery
     D[37] = 3e-3  # 37 vein
     D[40] = 1.31e-3  # 40 asc lower intestine : Hai-Jing et al. doi: 10.1097/RCT.0000000000000926
@@ -124,15 +128,19 @@ def contrast_curve_calc():
     f[8] = 1.00  # 8 Blood ra
     f[10] = 0.10  # 8 Muscle: 10.3389/fresc.2022.910068
     f[13] = 0.11  # 13 liver : Delattre et al. doi: 10.1097/RLI.0b013e31826ef901
+    f[14] = 0  # 13 Gal
     f[17] = 0.32  # 17 esophagus : Huang et al. doi: 10.1259/bjr.20170421
     f[18] = 0.32  # 18 esophagus cont : Huang et al. doi: 10.1259/bjr.20170421
     f[20] = 0.3  # 20 stomach wall: Li et al. doi: 10.3389/fonc.2022.821586
+    f[21] = 0.0  # 20 stomach content
     f[22] = 0.15  # 22 Pancreas (from literature)
     f[23] = 0.097  # 23 right kydney cortex : van Baalen et al. Doi: jmri.25519
     f[24] = 0.158  # 23 right kydney medulla : van Baalen et al. Doi: jmri.25519
     f[25] = 0.097  # 23 left kydney cortex : van Baalen et al. Doi: jmri.25519
     f[26] = 0.158  # 23 left kydney medulla : van Baalen et al. Doi: jmri.25519
     f[30] = 0.2  # 30 spleen : Taimouri et al. Doi: 10.1118/1.4915495
+    f[34] = 0.178  # 34 spinal cord :doi: 10.3389/fonc.2022.961473
+    f[35] = 0.145  # 35 Bone marrow : https://pubmed.ncbi.nlm.nih.gov/30194746/
     f[36] = 1.0  # 36 artery
     f[37] = 1.0  # 37 vein
     f[40] = 0.69  # 40 asc lower intestine : Hai-Jing et al. doi: 10.1097/RCT.0000000000000926
@@ -153,15 +161,19 @@ def contrast_curve_calc():
     Ds[8] = 0.1  # 8 Blood ra
     Ds[10] = 0.0263  # 8 Muscle: 10.3389/fresc.2022.910068
     Ds[13] = 0.1  # 13 liver: Delattre et al. doi: 10.1097/RLI.0b013e31826ef901
+    Ds[14] = 0.1  # 14 Gal
     Ds[17] = 0.03  # 17 esophagus : Huang et al. doi: 10.1259/bjr.20170421
     Ds[18] = 0.03  # 18 esophagus cont : Huang et al. doi: 10.1259/bjr.20170421
     Ds[20] = 0.012  # 20 stomach wall: Li et al. doi: 10.3389/fonc.2022.821586
+    Ds[21] = 0.0  # 20 stomach content
     Ds[22] = 0.01  # 22 Pancreas (from literature)
     Ds[23] = 0.02  # 23 right kydney cortex : van Baalen et al. Doi: jmri.25519
     Ds[24] = 0.019  # 23 right kydney medulla : van Baalen et al. Doi: jmri.25519
     Ds[25] = 0.02  # 23 left kydney cortex : van Baalen et al. Doi: jmri.25519
     Ds[26] = 0.019  # 23 left kydney medulla : van Baalen et al. Doi: jmri.25519
     Ds[30] = 0.03  # 30 spleen : Taimouri et al. Doi: 10.1118/1.4915495
+    Ds[34] = 0.0289  # 34 spinal cord :doi: 10.3389/fonc.2022.961473
+    Ds[35] = 0.05  # 35 Bone marrow :
     Ds[36] = 0.1  # 36 artery
     Ds[37] = 0.1  # 37 vein
     Ds[40] = 0.029  # 40 asc lower intestine : Hai-Jing et al. doi: 10.1097/RCT.0000000000000926
@@ -175,7 +187,7 @@ def contrast_curve_calc():
     return D, f, Ds
 
 
-def XCAT_to_MR_DCE(XCAT, TR, TE, bvalue, D, f, Ds, b0=3, ivim_cont = True):
+def XCAT_to_MR_DCE(XCAT, TR, TE, bvalue, D, f, Ds, b0=3, ivim_cont = True, T1T2=True):
     ###########################################################################################
     # This script converts XCAT tissue values to MR contrast based on the SSFP signal equation.
     # Christopher W. Roy 2018-12-04 # fetal.xcmr@gmail.com
@@ -285,7 +297,7 @@ def XCAT_to_MR_DCE(XCAT, TR, TE, bvalue, D, f, Ds, b0=3, ivim_cont = True):
     Tissue[19] = [1045.5, 37.3, 1201, 44]
     Tissue[20] = [981.5, 36, 1232.9, 37.20]
     #Tissue[20] = [981.5, 36, 1232.9, 37.20]
-    Tissue[21] = [0, 0, 0, 0]
+    Tissue[21] = [2500, 1250, 4000, 2000]
     Tissue[22] = [584, 46, 725, 43]
     Tissue[23] = [828, 71, 1168, 66]
     Tissue[24] = [1412, 85, 1545, 81]
@@ -351,7 +363,6 @@ def XCAT_to_MR_DCE(XCAT, TR, TE, bvalue, D, f, Ds, b0=3, ivim_cont = True):
             else:
                 T1 = Tissue[iTissue, 2]
                 T2 = Tissue[iTissue, 3]
-
             if ivim_cont and not np.isnan([D[iTissue], f[iTissue], Ds[iTissue]]).any():
                 # note we are assuming blood fraction has the same T1 as tissue fraction here for simplicity. Can be changed in future.
                 Dtemp=D[iTissue]
@@ -362,8 +373,11 @@ def XCAT_to_MR_DCE(XCAT, TR, TE, bvalue, D, f, Ds, b0=3, ivim_cont = True):
                 ftemp=np.random.rand(1)*0.5
                 Dstemp=5e-3+np.random.rand(1)*1e-1
             S0 = ivim(bvalue,Dtemp,ftemp,Dstemp)
-            if T1 > 0 or T2 > 0:
-                MR = MR + np.tile(np.expand_dims(XCAT == iTissue,3),len(S0)) * S0 * (1 - 2 * np.exp(-(TR - TE / 2) / T1) + np.exp(-TR / T1)) * np.exp(-TE / T2)
+            if T1T2:
+                if T1 > 0 or T2 > 0:
+                    MR = MR + np.tile(np.expand_dims(XCAT == iTissue,3),len(S0)) * S0 * (1 - 2 * np.exp(-(TR - TE / 2) / T1) + np.exp(-TR / T1)) * np.exp(-TE / T2)
+            else:
+                MR = MR + np.tile(np.expand_dims(XCAT == iTissue,3),len(S0)) * S0
             Dim = Dim + (XCAT == iTissue) * Dtemp
             fim = fim + (XCAT == iTissue) * ftemp
             Dpim = Dpim + (XCAT == iTissue) * Dstemp
@@ -403,6 +417,7 @@ def parse_bvalues_file(file_path):
     parser.add_argument("-n", "--noise", type=float, default=0.0005, help="Noise")
     parser.add_argument("-m", "--motion", action="store_true", help="Motion flag")
     parser.add_argument("-i", "--interleaved", action="store_true", help="Interleaved flag")
+    parser.add_argument("-u", "--T1T2", action="store_true", help="weight signal with T1T2") # note, set this to zero when generating test data for unit testing!
     args = parser.parse_args()
 
     if args.bvalues_file and args.bvalue:
@@ -420,14 +435,15 @@ def parse_bvalues_file(file_path):
     noise = args.noise
     motion = args.motion
     interleaved = args.interleaved
+    T1T2 = args.T1T2
     download_data()
     for key, bvalue in bvalues.items():
         bvalue = np.array(bvalue)
-        sig, XCAT, Dim, fim, Dpim, legend = phantom(bvalue, noise, motion=motion, interleaved=interleaved)
+        sig, XCAT, Dim, fim, Dpim, legend = phantom(bvalue, noise, motion=motion, interleaved=interleaved,T1T2=T1T2)
         # sig = np.flip(sig,axis=0)
         # sig = np.flip(sig,axis=1)
         res=np.eye(4)
-        res[2]=2
+        res[2,2]=2
 
         voxel_selector_fraction = 0.5
         D, f, Ds = contrast_curve_calc()

phantoms/MR_XCAT_qMRI/sim_ivim_sig.py

@@ -14,3 +14,4 @@ tqdm
 pandas
 sphinx
 sphinx_rtd_theme
+pytest-json-report

requirements.txt

@@ -3,6 +3,7 @@
 from scipy.stats import norm
 import pathlib
 import sys
+from tqdm import tqdm
 
 class OsipiBase:
     """The base class for OSIPI IVIM fitting"""
@@ -47,7 +48,7 @@ def initialize(**kwargs):
         pass
 
     #def osipi_fit(self, data=None, bvalues=None, thresholds=None, bounds=None, initial_guess=None, **kwargs):
-    def osipi_fit(self, data, bvalues, **kwargs):
+    def osipi_fit(self, data, bvalues=None, **kwargs):
         """Fits the data with the bvalues
         Returns [S0, f, Dstar, D]
         """
@@ -68,7 +69,6 @@ def osipi_fit(self, data, bvalues, **kwargs):
         #kwargs["bvalues"] = use_bvalues
 
         #args = [data, use_bvalues, use_thresholds]
-        args = [data, use_bvalues]
         #if self.required_bounds or self.required_bounds_optional:
             #args.append(use_bounds)
         #if self.required_initial_guess or self.required_initial_guess_optional:
@@ -83,7 +83,13 @@ def osipi_fit(self, data, bvalues, **kwargs):
 
         #args = [data, use_bvalues, use_initial_guess, use_bounds, use_thresholds]
         #args = [arg for arg in args if arg is not None]
-        results = self.ivim_fit(*args, **kwargs)
+
+        # Assuming the last dimension of the data is the signal values of each b-value
+        results = np.empty(list(data.shape[:-1])+[3]) # Create an array with the voxel dimensions + the ones required for the fit
+        for ijk in tqdm(np.ndindex(data.shape[:-1]), total=np.prod(data.shape[:-1])):
+            args = [data[ijk], use_bvalues]
+            fit = list(self.ivim_fit(*args, **kwargs))
+            results[ijk] = fit
 
         #self.parameter_estimates = self.ivim_fit(data, bvalues)
         return results

src/wrappers/OsipiBase.py

@@ -100,7 +100,7 @@
     },
     "ETP_SRI_LinearFitting": {
         "xfail_names": {
-            "Vein": true
+            "Blood RV": true
         },
         "options": {
             "thresholds": [500]

tests/IVIMmodels/unit_tests/algorithms.json

@@ -3,6 +3,12 @@
 #Run like this:
 #Rscript --vanilla tests/IVIMmodels/unit_tests/compare.r test_output.csv test_reference.csv reference_output.csv test_results.csv
 
+# If this script fails:
+# 1. Save the "Comparison" file from the run on Github, OR run this file directly
+# 2. Find the file producted "test_reference.csv" on Github, or whatever the "reference_file" variable was called
+# 3. This replaces "tests/IVIMmodels/unit_tests/reference_output.csv" in the repository
+# 4. For the algorithm "IAR_LU_modified_mix", replace the "f_f_alpha, Dp_f_alpha, D_f_alpha, f_t_alpha, Dp_t_alpha, D_t_alpha" columns with "0.01,0.01,0.01,0.0,0.0,0.0"
+
 args = commandArgs(trailingOnly=TRUE)
 # Define file paths
 test_file <- "test_output.csv"