Fixes for reading attachments: fill value; hdf5 path (#8727)

Two fixes included in
#8717 as a separate PR
to get them into master sooner.

To distinguish between Box states Failure and Empty for shard loading,
we need to make sure to use `?=>` instead of `?~>` (which only attaches
an error message in Failure case, and passes through Empty)

### Steps to test:
- In a dataset with both registered and scanned agglomerate files, both
should be listed and usable.
- If a zarr array has no shard files, fill_value should be read
(according to shape from zarr.json)

------
- [x] Added changelog entry (create a `$PR_NUMBER.md` file in
`unreleased_changes` or use `./tools/create-changelog-entry.py`)
- [x] Needs datastore update after deployment

Author: fm3

unreleased_changes/8727.md

@@ -0,0 +1,2 @@
+### Fixed
+- Fixed reading chunks from arrays where the whole shard is missing (should return fill_value)

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkReader.scala

@@ -25,17 +25,22 @@ class ChunkReader(header: DatasetHeader) extends FoxImplicits {
       typed <- chunkBytesAndShapeBox.map(_._1) match {
         case Full(chunkBytes) if useSkipTypingShortcut =>
           shortcutChunkTyper.wrapAndType(chunkBytes, chunkShape).toFox ?~> "chunk.shortcutWrapAndType.failed"
-        case Empty if useSkipTypingShortcut =>
-          shortcutChunkTyper.createFromFillValueCached(chunkShape) ?~> "chunk.shortcutCreateFromFillValue.failed"
         case Full(chunkBytes) =>
           chunkTyper.wrapAndType(chunkBytes, chunkShape).toFox ?~> "chunk.wrapAndType.failed"
         case Empty =>
-          chunkTyper.createFromFillValueCached(chunkShape) ?~> "chunk.createFromFillValue.failed"
+          createFromFillValue(chunkShape, useSkipTypingShortcut)
         case f: Failure =>
           f.toFox ?~> s"Reading chunk at $path failed"
       }
     } yield typed
 
+  def createFromFillValue(chunkShape: Array[Int], useSkipTypingShortcut: Boolean)(
+      implicit ec: ExecutionContext): Fox[MultiArray] =
+    if (useSkipTypingShortcut)
+      shortcutChunkTyper.createFromFillValueCached(chunkShape) ?~> "chunk.shortcutCreateFromFillValue.failed"
+    else
+      chunkTyper.createFromFillValueCached(chunkShape) ?~> "chunk.createFromFillValue.failed"
+
   // Returns bytes (optional, Fox.empty may later be replaced with fill value)
   // and chunk shape (optional, only for data formats where each chunk reports its own shape, e.g. N5)
   protected def readChunkBytesAndShape(path: VaultPath, range: Option[NumericRange[Long]])(

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala

@@ -3,7 +3,7 @@ package com.scalableminds.webknossos.datastore.datareaders
 import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.cache.AlfuCache
 import com.scalableminds.util.geometry.Vec3Int
-import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.util.tools.{Empty, Failure, Fox, FoxImplicits, Full}
 import com.scalableminds.webknossos.datastore.datavault.VaultPath
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
 import com.scalableminds.webknossos.datastore.models.AdditionalCoordinate
@@ -248,10 +248,15 @@ class DatasetArray(vaultPath: VaultPath,
       implicit ec: ExecutionContext,
       tc: TokenContext): Fox[MultiArray] =
     if (header.isSharded) {
+      val chunkShape = chunkShapeAtIndex(chunkIndex)
       for {
-        (shardPath, chunkRange) <- getShardedChunkPathAndRange(chunkIndex) ?~> "chunk.getShardedPathAndRange.failed"
-        chunkShape = chunkShapeAtIndex(chunkIndex)
-        multiArray <- chunkReader.read(shardPath, chunkShape, Some(chunkRange), useSkipTypingShortcut)
+        shardPathAndChunkRangeBox <- getShardedChunkPathAndRange(chunkIndex).shiftBox
+        multiArray <- shardPathAndChunkRangeBox match {
+          case Full((shardPath, chunkRange)) =>
+            chunkReader.read(shardPath, chunkShape, Some(chunkRange), useSkipTypingShortcut)
+          case Empty      => chunkReader.createFromFillValue(chunkShape, useSkipTypingShortcut)
+          case f: Failure => f.toFox
+        }
       } yield multiArray
     } else {
       val chunkPath = vaultPath / getChunkFilename(chunkIndex)

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/precomputed/PrecomputedArray.scala

@@ -87,7 +87,7 @@ class PrecomputedArray(vaultPath: VaultPath,
     val shardPath = shardingSpecification.getPathForShard(vaultPath, minishardInfo._1)
     for {
       _ <- Fox.fromBool(minishardInfo._2 <= Int.MaxValue) ?~> "Minishard number is too large"
-      minishardIndex <- getMinishardIndex(shardPath, minishardInfo._2.toInt) ?~> f"Could not get minishard index for chunkIndex ${chunkIndex
+      minishardIndex <- getMinishardIndex(shardPath, minishardInfo._2.toInt) ?=> f"Could not get minishard index for chunkIndex ${chunkIndex
         .mkString(",")}"
       chunkRange: NumericRange.Exclusive[Long] <- getChunkRange(chunkIdentifier, minishardIndex) ?~> s"Could not get chunk range for chunkIndex ${chunkIndex
         .mkString(",")}  with chunkIdentifier $chunkIdentifier in minishard index."

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/zarr3/Zarr3Array.scala

@@ -120,7 +120,7 @@ class Zarr3Array(vaultPath: VaultPath,
   private def readAndParseShardIndex(shardPath: VaultPath)(implicit ec: ExecutionContext,
                                                            tc: TokenContext): Fox[Array[(Long, Long)]] =
     for {
-      shardIndexRaw <- readShardIndex(shardPath) ?~> "zarr.readShardIndex.failed"
+      shardIndexRaw <- readShardIndex(shardPath) ?=> "zarr.readShardIndex.failed"
       parsed = parseShardIndex(shardIndexRaw)
     } yield parsed
 

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/AgglomerateService.scala

@@ -98,7 +98,11 @@ class AgglomerateService @Inject()(config: DataStoreConfig,
       })
       localFallbackAttachment = LayerAttachment(
         mappingName,
-        localDatasetDir.resolve(dataLayer.name).resolve(agglomerateDir).toUri,
+        localDatasetDir
+          .resolve(dataLayer.name)
+          .resolve(agglomerateDir)
+          .resolve(mappingName + "." + hdf5AgglomerateFileExtension)
+          .toUri,
         LayerAttachmentDataformat.hdf5
       )
       selectedAttachment = registeredAttachmentNormalized.getOrElse(localFallbackAttachment)

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshFileService.scala

@@ -74,9 +74,10 @@ class MeshFileService @Inject()(config: DataStoreConfig,
 
   private val dataBaseDir = Paths.get(config.Datastore.baseDirectory)
   private val meshesDir = "meshes"
+  private val hdf5MeshFileExtension = "hdf5"
 
   private val meshFileKeyCache
-    : AlfuCache[(DataSourceId, String, String), MeshFileKey] = AlfuCache() // dataSourceId, layerName, mappingName → MeshFileKey
+    : AlfuCache[(DataSourceId, String, String), MeshFileKey] = AlfuCache() // dataSourceId, layerName, meshFileName → MeshFileKey
 
   def lookUpMeshFileKey(dataSourceId: DataSourceId, dataLayer: DataLayer, meshFileName: String)(
       implicit ec: ExecutionContext): Fox[MeshFileKey] =
@@ -99,7 +100,11 @@ class MeshFileService @Inject()(config: DataStoreConfig,
       })
       localFallbackAttachment = LayerAttachment(
         meshFileName,
-        localDatasetDir.resolve(dataLayer.name).resolve(meshesDir).toUri,
+        localDatasetDir
+          .resolve(dataLayer.name)
+          .resolve(meshesDir)
+          .resolve(meshFileName + "." + hdf5MeshFileExtension)
+          .toUri,
         LayerAttachmentDataformat.hdf5
       )
       selectedAttachment = registeredAttachmentNormalized.getOrElse(localFallbackAttachment)
@@ -119,7 +124,7 @@ class MeshFileService @Inject()(config: DataStoreConfig,
     val layerDir =
       dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName).resolve(dataLayer.name)
     val scannedMeshFileNames = PathUtils
-      .listFiles(layerDir.resolve(meshesDir), silent = true, PathUtils.fileExtensionFilter(hdf5FileExtension))
+      .listFiles(layerDir.resolve(meshesDir), silent = true, PathUtils.fileExtensionFilter(hdf5MeshFileExtension))
       .map { paths =>
         paths.map(path => FilenameUtils.removeExtension(path.getFileName.toString))
       }