From 8eec486082f4ed38d9889f6ca80017855a0039c9 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 11 Apr 2025 16:01:17 +0200 Subject: [PATCH 1/3] add testMetadataAcceptsStorageTransformer --- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 5 +++ testdata/storage_transformer/zarr.json | 40 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 testdata/storage_transformer/zarr.json diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 7d980ab..ac9cdd4 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -631,4 +631,9 @@ public void testReadL4Sample(String mag) throws IOException, ZarrException { assert MultiArrayUtils.allValuesEqual(httpData2, localData2); } + @Test + public void testMetadataAcceptsStorageTransformer() throws ZarrException, IOException { + StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("storage_transformer"); + Array localArray = Array.open(localStoreHandle); + } } diff --git a/testdata/storage_transformer/zarr.json b/testdata/storage_transformer/zarr.json new file mode 100644 index 0000000..177cc91 --- /dev/null +++ b/testdata/storage_transformer/zarr.json @@ -0,0 +1,40 @@ +{ + "zarr_format": 3, + "node_type": "array", + "shape": [ + 1 + ], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + } + ], + "attributes": {}, + "storage_transformers": [ + { + "name": "chunk-manifest-json", + "configuration": { + "manifest": "./manifest.json" + } + } + ] +} \ No newline at end of file From aec1bb29fd3faf7effc3a6fcc7d7c22f7f3f7a12 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 15 Apr 2025 11:52:25 +0200 Subject: [PATCH 2/3] add storageTransformers attribute --- .../dev/zarr/zarrjava/v3/ArrayMetadata.java | 12 ++++++--- .../zarrjava/v3/ArrayMetadataBuilder.java | 9 ++++++- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 25 ++++++++++++++++++- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index 77940e8..0146f53 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -55,6 +55,9 @@ public final class ArrayMetadata { @Nullable @JsonProperty("dimension_names") public String[] dimensionNames; + @Nullable + @JsonProperty("storage_transformers") + public Map[] storageTransformers; @JsonIgnore public CoreArrayMetadata coreArrayMetadata; @@ -64,11 +67,12 @@ public ArrayMetadata( Object fillValue, @Nonnull Codec[] codecs, @Nullable String[] dimensionNames, - @Nullable Map attributes + @Nullable Map attributes, + @Nullable Map[] storageTransformers ) throws ZarrException { this(ZARR_FORMAT, NODE_TYPE, shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, dimensionNames, - attributes + attributes, storageTransformers ); } @@ -83,7 +87,8 @@ public ArrayMetadata( @JsonProperty(value = "fill_value", required = true) Object fillValue, @Nonnull @JsonProperty(value = "codecs") Codec[] codecs, @Nullable @JsonProperty(value = "dimension_names") String[] dimensionNames, - @Nullable @JsonProperty(value = "attributes") Map attributes + @Nullable @JsonProperty(value = "attributes") Map attributes, + @Nullable @JsonProperty(value = "storage_transformers") Map[] storageTransformers ) throws ZarrException { if (zarrFormat != this.zarrFormat) { throw new ZarrException( @@ -126,6 +131,7 @@ public ArrayMetadata( this.codecs = codecs; this.dimensionNames = dimensionNames; this.attributes = attributes; + this.storageTransformers = storageTransformers; this.coreArrayMetadata = new CoreArrayMetadata(shape, ((RegularChunkGrid) chunkGrid).configuration.chunkShape, dataType, diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index f718327..3d07ad4 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -29,6 +29,7 @@ public class ArrayMetadataBuilder { Object fillValue = 0; Codec[] codecs = new Codec[]{new BytesCodec(Endian.LITTLE)}; Map attributes = new HashMap<>(); + Map[] storageTransformers = new HashMap[]{}; String[] dimensionNames = null; protected ArrayMetadataBuilder() { @@ -44,6 +45,7 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad builder.codecs = arrayMetadata.codecs; builder.attributes = arrayMetadata.attributes; builder.dimensionNames = arrayMetadata.dimensionNames; + builder.storageTransformers = arrayMetadata.storageTransformers; return builder; } @@ -125,6 +127,10 @@ public ArrayMetadataBuilder withAttributes(Map attributes) { this.attributes = attributes; return this; } + public ArrayMetadataBuilder withStorageTransformers(Map[] storageTransformers) { + this.storageTransformers = storageTransformers; + return this; + } public ArrayMetadata build() throws ZarrException { if (shape == null) { @@ -140,7 +146,8 @@ public ArrayMetadata build() throws ZarrException { return new ArrayMetadata(shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, dimensionNames, - attributes + attributes, + storageTransformers ); } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index ac9cdd4..8ceaaf6 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdCompressCtx; +import com.google.common.collect.Maps; import dev.zarr.zarrjava.store.*; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.v3.*; @@ -631,9 +632,31 @@ public void testReadL4Sample(String mag) throws IOException, ZarrException { assert MultiArrayUtils.allValuesEqual(httpData2, localData2); } + @Test public void testMetadataAcceptsStorageTransformer() throws ZarrException, IOException { StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("storage_transformer"); - Array localArray = Array.open(localStoreHandle); + Map[] storageTransformers1 = Array.open(localStoreHandle).metadata.storageTransformers; + + Array array2 = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), + Array.metadataBuilder() + .withShape(1) + .withChunkShape(1) + .withDataType(DataType.UINT8) + .withStorageTransformers(new HashMap[]{new HashMap(){ + { + put("name", "chunk-manifest-json"); + put("configuration", new HashMap(){ + { + put("manifest", "./manifest.json"); + } + }); + } + }}) + .build() + ); + Map[] storageTransformers2 = array2.metadata.storageTransformers; + assert Maps.difference(storageTransformers1[0], storageTransformers2[0]).areEqual(); } } From 8e218e78d36b8f4c06e6837ad0bb0d1539ce44d7 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 15 Apr 2025 12:45:13 +0200 Subject: [PATCH 3/3] fails on non-empty storage_transformers --- .../dev/zarr/zarrjava/v3/ArrayMetadata.java | 5 ++- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 45 ++++++++++--------- testdata/storage_transformer/empty/zarr.json | 33 ++++++++++++++ .../{ => exists}/zarr.json | 0 4 files changed, 60 insertions(+), 23 deletions(-) create mode 100644 testdata/storage_transformer/empty/zarr.json rename testdata/storage_transformer/{ => exists}/zarr.json (100%) diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index 0146f53..f23a9fc 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -98,7 +98,10 @@ public ArrayMetadata( throw new ZarrException( "Expected node type '" + this.nodeType + "', got '" + nodeType + "'."); } - + if (storageTransformers != null && storageTransformers.length > 0) { + throw new ZarrException( + "Storage transformers are not supported in this version of Zarr Java."); + } if (chunkGrid instanceof RegularChunkGrid) { int[] chunkShape = ((RegularChunkGrid) chunkGrid).configuration.chunkShape; if (shape.length != chunkShape.length) { diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 0e7e7b7..b9489a1 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -3,6 +3,7 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.AnonymousAWSCredentials; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdCompressCtx; @@ -657,30 +658,30 @@ public void testParallel(boolean useParallel) throws IOException, ZarrException } @Test - public void testMetadataAcceptsStorageTransformer() throws ZarrException, IOException { - StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("storage_transformer"); - Map[] storageTransformers1 = Array.open(localStoreHandle).metadata.storageTransformers; + public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, IOException { + // non-empty storage transformers are currently not supported - Array array2 = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), - Array.metadataBuilder() - .withShape(1) - .withChunkShape(1) - .withDataType(DataType.UINT8) - .withStorageTransformers(new HashMap[]{new HashMap(){ - { - put("name", "chunk-manifest-json"); - put("configuration", new HashMap(){ - { - put("manifest", "./manifest.json"); - } - }); - } - }}) - .build() + Map[] storageTransformersEmpty = Array.open( + new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty") + ).metadata.storageTransformers; + assert storageTransformersEmpty.length == 0; + + assertThrows(JsonMappingException.class, () -> Array.open( + new FilesystemStore(TESTDATA).resolve("storage_transformer", "exists")) ); - Map[] storageTransformers2 = array2.metadata.storageTransformers; - assert Maps.difference(storageTransformers1[0], storageTransformers2[0]).areEqual(); + + ArrayMetadataBuilder builderWithStorageTransformer = Array.metadataBuilder() + .withShape(1) + .withChunkShape(1) + .withDataType(DataType.UINT8) + .withStorageTransformers(new HashMap[]{new HashMap(){{ + put("some", "value"); + }}}); + + assertThrows(ZarrException.class, () -> Array.create( + new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), + builderWithStorageTransformer.build() + )); } } diff --git a/testdata/storage_transformer/empty/zarr.json b/testdata/storage_transformer/empty/zarr.json new file mode 100644 index 0000000..0621bcd --- /dev/null +++ b/testdata/storage_transformer/empty/zarr.json @@ -0,0 +1,33 @@ +{ + "zarr_format": 3, + "node_type": "array", + "shape": [ + 1 + ], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + } + ], + "attributes": {}, + "storage_transformers": [] +} \ No newline at end of file diff --git a/testdata/storage_transformer/zarr.json b/testdata/storage_transformer/exists/zarr.json similarity index 100% rename from testdata/storage_transformer/zarr.json rename to testdata/storage_transformer/exists/zarr.json