Split analysis into matrix jobs

Author: etpeterson

.github/workflows/analysis.yml

@@ -7,22 +7,71 @@ on:
       - 'analysis/**'
 
 jobs:
-  build:
+  read_algorithms:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Read algorithms
+        id: read_algorithms
+        run: |
+          algorithms=$(cat tests/IVIMmodels/unit_tests/algorithms.json)
+          echo "::set-output name=algorithms::$algorithms"
 
+  build:
     runs-on: ubuntu-latest
+    needs: read_algorithms
     continue-on-error: false
     strategy:
       fail-fast: false
+      matrix:
+        algorithm: ${{ fromJson(needs.read_algorithms.outputs.algorithms).algorithms }}
+        SNR: [10 30 50 100 200]
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.11"
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.11"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
+      - name: Generate fitting data
+        run: |
+          pip install pytest
+          python -m pytest -m slow --selectAlgorithm ${{ matrix.algorithm }} --saveFileName test_output.csv --SNR ${{ matrix.SNR }} --fitCount 300 --saveDurationFileName test_duration.csv
+      - name: Upload raw data
+        uses: actions/upload-artifact@v3
+        with:
+            name: RawData/raw_data_${{ matrix.algorithm }}_${{ matrix.SNR }}
+            path: |
+              test_output.csv
+              test_duration.csv
+  merge:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+      - name: Merge fitting results
+        run: |
+          cat artifacts/raw_data_*/test_output.csv > test_output.csv
+      - name: Merge timing results
+        run: |
+          cat artifacts/raw_data_*/test_duration.csv > test_duration.csv
+      - name: Upload merged artifacts
+        uses: actions/upload-artifacts@v3
+        with:
+          name: Raw Data
+          path: |
+            test_output.csv
+            test_duration.csv
+  analyze:
+    runs-on: ubuntu-latest
+    needs: merge
+    steps:
       - name: Set up R
         uses: r-lib/actions/setup-r@v2
         with: 
@@ -36,23 +85,16 @@ jobs:
             any::tidyverse
             any::data.table
             any::ggplot2
-      - name: Generate fitting data
-        run: |
-          pip install pytest
-          python -m pytest -m slow --saveFileName test_output.csv --SNR 10 30 50 100 200 --fitCount 300 --saveDurationFileName test_duration.csv
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: Raw Data
       - name: Generate figures
         run: Rscript --vanilla tests/IVIMmodels/unit_tests/analyze.r test_output.csv test_duration.csv
-      - name: Upload raw data
-        uses: actions/upload-artifact@v3
-        with:
-            name: Raw data
-            path: |
-              test_output.csv
-              test_duration.csv
       - name: Upload figures
         uses: actions/upload-artifact@v3
         with:
-            name: Fit figures
+            name: Fit Figures
             path: |
               D.pdf
               f.pdf

conftest.py

@@ -67,6 +67,20 @@ def pytest_addoption(parser):
         type=str,
         help="Saved duration results file name",
     )
+    parser.addoption(
+        "--selectAlgorithm",
+        default=[""],
+        nargs="+",
+        type=str,
+        help="Drop all algorithms except for these from the list"
+    )
+    parser.addoption(
+        "--dropAlgorithm",
+        default=[""],
+        nargs="+",
+        type=str,
+        help="Drop this algorithm from the list"
+    )
 
 
 @pytest.fixture(scope="session")
@@ -115,7 +129,6 @@ def save_duration_file(request):
 def rtol(request):
     return request.config.getoption("--rtol")
 
-
 @pytest.fixture(scope="session")
 def atol(request):
     return request.config.getoption("--atol")
@@ -137,23 +150,23 @@ def pytest_generate_tests(metafunc):
     if "SNR" in metafunc.fixturenames:
         metafunc.parametrize("SNR", metafunc.config.getoption("SNR"))
     if "ivim_algorithm" in metafunc.fixturenames:
-        algorithms = algorithm_list(metafunc.config.getoption("algorithmFile"))
+        algorithms = algorithm_list(metafunc.config.getoption("algorithmFile"), metafunc.config.getoption("selectAlgorithm"), metafunc.config.getoption("dropAlgorithm"))
         metafunc.parametrize("ivim_algorithm", algorithms)
-    if "algorithm_file" in metafunc.fixturenames:
-        metafunc.parametrize("algorithm_file", metafunc.config.getoption("algorithmFile"))
     if "ivim_data" in metafunc.fixturenames:
         data = data_list(metafunc.config.getoption("dataFile"))
         metafunc.parametrize("ivim_data", data)
-    if "data_file" in metafunc.fixturenames:
-        metafunc.parametrize("data_file", metafunc.config.getoption("dataFile"))
 
 
-def algorithm_list(filename):
+def algorithm_list(filename, selected, dropped):
     current_folder = pathlib.Path.cwd()
     algorithm_path = current_folder / filename
     with algorithm_path.open() as f:
         algorithm_information = json.load(f)
-    return algorithm_information["algorithms"]
+    algorithms = set(algorithm_information["algorithms"])
+    algorithms = algorithms - set(dropped)
+    if len(selected) > 0 and selected[0]:
+        algorithms = algorithms & set(selected)
+    return list(algorithms)
 
 def data_list(filename):
     current_folder = pathlib.Path.cwd()