Converter for MDIO dataset spec to Xarray Dataset and serialize it to Zarr

.gitignore

@@ -153,3 +153,5 @@ mdio1/*
 pytest-of-*
 tmp
 debugging/*
+
+test-data/*

_dev/DEVELOPERS_NOTES.md

@@ -0,0 +1,16 @@
+# Wring empty XArray / Zarr to a local storage
+
+src/mdio/schemas/v1/dataset_serializer.py
+
+## Issues encountered
+
+1. FIXED: Non-zero size of the serialized data files
+2. FIXED: Not clear how to properly set `compressor`, `dimension_separator`, and `fill_value`
+3. FIXED: For image_inline chunks[2] are somehow different?
+
+4. `fill_value` for StructuredType is set to null, but "AAAAAAAAAAAAAAAA" is expected
+
+## TO DO:
+
+- Add more unit tests for internal functions
+- Add a trest comparing expected and actual .zmetadata for the serialized dataset

_dev/zmetadata.cpp.json

@@ -0,0 +1,195 @@
+{
+  "metadata": {
+    ".zattrs": {
+      "apiVersion": "1.0.0",
+      "attributes": {
+        "foo": "bar",
+        "textHeader": [
+          "C01 .......................... ",
+          "C02 .......................... ",
+          "C03 .......................... "
+        ]
+      },
+      "createdOn": "2023-12-12T15:02:06.413469-06:00",
+      "name": "campos_3d"
+    },
+    ".zgroup": {
+      "zarr_format": 2
+    },
+    "cdp-x/.zarray": {
+      "chunks": [256, 512],
+      "compressor": null,
+      "dimension_separator": "/",
+      "dtype": "<f4",
+      "fill_value": "NaN",
+      "filters": null,
+      "order": "C",
+      "shape": [256, 512],
+      "zarr_format": 2
+    },
+    "cdp-x/.zattrs": {
+      "_ARRAY_DIMENSIONS": ["inline", "crossline"],
+      "unitsV1": {
+        "length": "m"
+      }
+    },
+    "cdp-y/.zarray": {
+      "chunks": [256, 512],
+      "compressor": null,
+      "dimension_separator": "/",
+      "dtype": "<f4",
+      "fill_value": "NaN",
+      "filters": null,
+      "order": "C",
+      "shape": [256, 512],
+      "zarr_format": 2
+    },
+    "cdp-y/.zattrs": {
+      "_ARRAY_DIMENSIONS": ["inline", "crossline"],
+      "unitsV1": {
+        "length": "m"
+      }
+    },
+    "crossline/.zarray": {
+      "chunks": [512],
+      "compressor": null,
+      "dimension_separator": "/",
+      "dtype": "<u4",
+      "fill_value": null,
+      "filters": null,
+      "order": "C",
+      "shape": [512],
+      "zarr_format": 2
+    },
+    "crossline/.zattrs": {
+      "_ARRAY_DIMENSIONS": ["crossline"]
+    },
+    "depth/.zarray": {
+      "chunks": [384],
+      "compressor": null,
+      "dimension_separator": "/",
+      "dtype": "<u4",
+      "fill_value": null,
+      "filters": null,
+      "order": "C",
+      "shape": [384],
+      "zarr_format": 2
+    },
+    "depth/.zattrs": {
+      "_ARRAY_DIMENSIONS": ["depth"],
+      "unitsV1": {
+        "length": "m"
+      }
+    },
+    "image/.zarray": {
+      "chunks": [128, 128, 128],
+      "compressor": {
+        "blocksize": 0,
+        "clevel": 5,
+        "cname": "zstd",
+        "id": "blosc",
+        "shuffle": 1
+      },
+      "dimension_separator": "/",
+      "dtype": "<f4",
+      "fill_value": "NaN",
+      "filters": null,
+      "order": "C",
+      "shape": [256, 512, 384],
+      "zarr_format": 2
+    },
+    "image/.zattrs": {
+      "_ARRAY_DIMENSIONS": ["inline", "crossline", "depth"],
+      "attributes": {
+        "fizz": "buzz"
+      },
+      "coordinates": "cdp-x cdp-y",
+      "statsV1": {
+        "count": 100,
+        "histogram": {
+          "binCenters": [1, 2],
+          "counts": [10, 15]
+        },
+        "max": 10.84,
+        "min": 5.61,
+        "sum": 1215.1,
+        "sumSquares": 125.12
+      }
+    },
+    "image_headers/.zarray": {
+      "chunks": [128, 128],
+      "compressor": null,
+      "dimension_separator": "/",
+      "dtype": [
+        ["cdp-x", "<i4"],
+        ["cdp-y", "<i4"],
+        ["elevation", "<f2"],
+        ["some_scalar", "<f2"]
+      ],
+      "fill_value": "AAAAAAAAAAAAAAAA",
+      "filters": null,
+      "order": "C",
+      "shape": [256, 512],
+      "zarr_format": 2
+    },
+    "image_headers/.zattrs": {
+      "_ARRAY_DIMENSIONS": ["inline", "crossline"],
+      "coordinates": "cdp-x cdp-y"
+    },
+    "image_inline/.zarray": {
+      "chunks": [4, 512, 512],
+      "compressor": {
+        "blocksize": 0,
+        "clevel": 5,
+        "cname": "zstd",
+        "id": "blosc",
+        "shuffle": 1
+      },
+      "dimension_separator": "/",
+      "dtype": "<f4",
+      "fill_value": "NaN",
+      "filters": null,
+      "order": "C",
+      "shape": [256, 512, 384],
+      "zarr_format": 2
+    },
+    "image_inline/.zattrs": {
+      "_ARRAY_DIMENSIONS": ["inline", "crossline", "depth"],
+      "coordinates": "cdp-x cdp-y",
+      "long_name": "inline optimized version of 3d_stack"
+    },
+    "inline/.zarray": {
+      "chunks": [256],
+      "compressor": null,
+      "dimension_separator": "/",
+      "dtype": "<u4",
+      "fill_value": null,
+      "filters": null,
+      "order": "C",
+      "shape": [256],
+      "zarr_format": 2
+    },
+    "inline/.zattrs": {
+      "_ARRAY_DIMENSIONS": ["inline"]
+    },
+    "velocity/.zarray": {
+      "chunks": [128, 128, 128],
+      "compressor": null,
+      "dimension_separator": "/",
+      "dtype": "<f2",
+      "fill_value": "NaN",
+      "filters": null,
+      "order": "C",
+      "shape": [256, 512, 384],
+      "zarr_format": 2
+    },
+    "velocity/.zattrs": {
+      "_ARRAY_DIMENSIONS": ["inline", "crossline", "depth"],
+      "coordinates": "cdp-x cdp-y",
+      "unitsV1": {
+        "speed": "m/s"
+      }
+    }
+  },
+  "zarr_consolidated_format": 1
+}