From 5a6e6523e753d1720b97fdedd7ec7cb3e839c525 Mon Sep 17 00:00:00 2001 From: Kris Fedorenko <1648280+krisfed@users.noreply.github.com> Date: Thu, 22 May 2025 13:30:56 -0400 Subject: [PATCH 1/6] Move creation of KV store dict to python layer --- PythonModule/ZarrPy.py | 28 +++++++++++++++++++++++++++- Zarr.m | 6 ++---- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/PythonModule/ZarrPy.py b/PythonModule/ZarrPy.py index 704395d..f5bbf88 100644 --- a/PythonModule/ZarrPy.py +++ b/PythonModule/ZarrPy.py @@ -7,6 +7,32 @@ import numpy as np import tensorstore as ts +def createKVStore(isRemote, objPath, bucketName="") -> dict: + """ + Creates a KV store (a python dictionary) for reading or writing + a Zarr file + + Parameters: + - isRemote (bool): whether the resource to be accessed with this +KV store is remote (s3) or local + - objPath (str): path to local Zarr file or to s3 object + - bucketName (str): If file is remote, this should be the S3 bucket +name + + Returns: + - KVStore (dict): Key-Value store as required by tensorstore to work +with Zarr + """ + KVStore = dict(path=objPath); + + if isRemote: + KVStore['driver'] = 's3' + KVStore['bucket'] = bucketName + else: + KVStore['driver'] = 'file' + + return KVStore + def createZarr(kvstore_schema, data_shape, chunk_shape, tstoreDataType, zarrDataType, compressor, fillvalue): """ Creates a new Zarr array and writes data to it. @@ -72,4 +98,4 @@ def readZarr (kvstore_schema): # Read a subset of the data data = zarr_file[...].read().result() - return data \ No newline at end of file + return data diff --git a/Zarr.m b/Zarr.m index 3402b35..b16351f 100644 --- a/Zarr.m +++ b/Zarr.m @@ -198,14 +198,12 @@ function makeZarrGroups(existingParentPath, newGroupsPath) % Extract the S3 bucket name and path [bucketName, objectPath] = obj.extractS3BucketNameAndPath(obj.Path); % Create a Python dictionary for the KV store driver - RemoteStoreSchema = dictionary(["driver", "bucket", "path"], ["s3", bucketName, objectPath]); - obj.KVStoreSchema = py.dict(RemoteStoreSchema); + obj.KVStoreSchema = py.ZarrPy.createKVStore(obj.isRemote, objectPath, bucketName); else % Local file % use full path obj.Path = Zarr.getFullPath(path); - FileStoreSchema = dictionary(["driver", "path"], ["file", obj.Path]); - obj.KVStoreSchema = py.dict(FileStoreSchema); + obj.KVStoreSchema = py.ZarrPy.createKVStore(obj.isRemote, obj.Path); end end From 1720462f8647d29a05c2c30244c1bf5b75c11d33 Mon Sep 17 00:00:00 2001 From: Kris Fedorenko <1648280+krisfed@users.noreply.github.com> Date: Thu, 22 May 2025 13:46:57 -0400 Subject: [PATCH 2/6] Update test workflow --- .github/workflows/test_setup.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_setup.yml b/.github/workflows/test_setup.yml index 9e743a5..f1dc040 100644 --- a/.github/workflows/test_setup.yml +++ b/.github/workflows/test_setup.yml @@ -5,9 +5,8 @@ on: branches: - main - branch-bug - - update-test-branch - test-branch7 - - test-branch8 + - test-branch-pyDict jobs: test: @@ -15,15 +14,16 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-14] + matlab-version: ['R2023b','R2024a'] steps: - name: Checkout code uses: actions/checkout@v2 - - name: Set up MATLAB R2024a + - name: Set up MATLAB uses: matlab-actions/setup-matlab@v2 with: - release: R2024a + release: ${{matrix.matlab-version}} - name: Set up Python uses: actions/setup-python@v5 From 5d8d1cd2a05e29f20f37e5b4bc18861a5046e827 Mon Sep 17 00:00:00 2001 From: Kris Fedorenko <1648280+krisfed@users.noreply.github.com> Date: Thu, 22 May 2025 14:04:56 -0400 Subject: [PATCH 3/6] Revert adding MATLAB releases to test --- .github/workflows/test_setup.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_setup.yml b/.github/workflows/test_setup.yml index f1dc040..79fc79f 100644 --- a/.github/workflows/test_setup.yml +++ b/.github/workflows/test_setup.yml @@ -14,7 +14,6 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-14] - matlab-version: ['R2023b','R2024a'] steps: - name: Checkout code @@ -23,7 +22,7 @@ jobs: - name: Set up MATLAB uses: matlab-actions/setup-matlab@v2 with: - release: ${{matrix.matlab-version}} + release: R2024a - name: Set up Python uses: actions/setup-python@v5 From 96b98f8f6743664dd610f410d35c9641bf0b490b Mon Sep 17 00:00:00 2001 From: Kris Fedorenko <1648280+krisfed@users.noreply.github.com> Date: Thu, 22 May 2025 15:38:51 -0400 Subject: [PATCH 4/6] Add back multiple MATLAB releases to test workflow --- .github/workflows/test_setup.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_setup.yml b/.github/workflows/test_setup.yml index 79fc79f..f1dc040 100644 --- a/.github/workflows/test_setup.yml +++ b/.github/workflows/test_setup.yml @@ -14,6 +14,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-14] + matlab-version: ['R2023b','R2024a'] steps: - name: Checkout code @@ -22,7 +23,7 @@ jobs: - name: Set up MATLAB uses: matlab-actions/setup-matlab@v2 with: - release: R2024a + release: ${{matrix.matlab-version}} - name: Set up Python uses: actions/setup-python@v5 From 0cd66da00c64e78ec8e37608dcaa7bc5e5d620a5 Mon Sep 17 00:00:00 2001 From: Kris Fedorenko <1648280+krisfed@users.noreply.github.com> Date: Fri, 23 May 2025 11:54:14 -0400 Subject: [PATCH 5/6] Disabling MATLAB tests for now due to outtage. Will run tests locally before merging. --- .github/workflows/test_setup.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_setup.yml b/.github/workflows/test_setup.yml index f1dc040..2b828bb 100644 --- a/.github/workflows/test_setup.yml +++ b/.github/workflows/test_setup.yml @@ -3,10 +3,10 @@ name: Run Tests on: push: branches: - - main - - branch-bug - - test-branch7 - - test-branch-pyDict + #- main + #- branch-bug + #- test-branch7 + #- test-branch-pyDict jobs: test: From da5f0dedd662c6db14eaf60f653221ef423a9023 Mon Sep 17 00:00:00 2001 From: Kris Fedorenko <1648280+krisfed@users.noreply.github.com> Date: Fri, 23 May 2025 12:22:37 -0400 Subject: [PATCH 6/6] Address feedback --- .github/workflows/test_setup.yml | 10 ++++------ PythonModule/ZarrPy.py | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test_setup.yml b/.github/workflows/test_setup.yml index 2b828bb..0e23693 100644 --- a/.github/workflows/test_setup.yml +++ b/.github/workflows/test_setup.yml @@ -1,11 +1,9 @@ name: Run Tests -on: - push: - branches: +#on: +# push: +# branches: #- main - #- branch-bug - #- test-branch7 #- test-branch-pyDict jobs: @@ -36,7 +34,7 @@ jobs: pip install numpy==1.26.4 tensorstore==0.1.73 - name: Run tests - uses: matlab-actions/run-tests@v2 + uses: matlab-actions/run-tests@v4 with: select-by-folder: 'test' code-coverage-cobertura: cobertura.xml diff --git a/PythonModule/ZarrPy.py b/PythonModule/ZarrPy.py index f5bbf88..3e51709 100644 --- a/PythonModule/ZarrPy.py +++ b/PythonModule/ZarrPy.py @@ -14,8 +14,8 @@ def createKVStore(isRemote, objPath, bucketName="") -> dict: Parameters: - isRemote (bool): whether the resource to be accessed with this -KV store is remote (s3) or local - - objPath (str): path to local Zarr file or to s3 object +KV store is remote (S3) or local + - objPath (str): path to local Zarr file or to S3 object - bucketName (str): If file is remote, this should be the S3 bucket name