Read Zarr Segment Index Files (#8711)

- Introduces attachment lookup for segment index files
- enables reading zarr segment index files
- Fixes bug where segment index file positions were wrongly interpreted
as being target-mag when really they are mag1
- Adds cache clear (including closing hdf5 file handles) for segment
index files
- Note to reviewer: git didn’t catch that a lot of code was moved (from
the old segmentIndexFileService to the new one, and partially to the
hdf5 one).

### Steps to test:
- For a dataset with segment index file (try both hdf5 and zarr, talk to
me for sample data), right-click a segment, context menu should show
plausible segment volume + bbox (not zero)
- Layer reload in the sidebar should log that cached segment index file
handles were cleared.

### TODOs:
- [x] attachment lookup
- [x] make sure hdf5 still works
- [x] implement for zarr
- [x] clear caches

### Issues:
- contributes to #8567 
- contributes to #8618 

------
- [x] Added changelog entry (create a `$PR_NUMBER.md` file in
`unreleased_changes` or use `./tools/create-changelog-entry.py`)
- [x] Removed dev-only changes like prints and application.conf edits
- [x] Considered [common edge
cases](../blob/master/.github/common_edge_cases.md)
- [x] Needs datastore update after deployment

---------

Co-authored-by: MichaelBuessemeyer <39529669+MichaelBuessemeyer@users.noreply.github.com>

Author: fm3

conf/messages

@@ -270,6 +270,8 @@ mesh.file.readVersion.failed=Failed to read format version from file “{0}”
 mesh.file.readMappingName.failed=Failed to read mapping name from mesh file “{0}”
 mesh.meshFileName.required=Trying to load mesh from mesh file, but mesh file name was not supplied.
 
+segmentIndexFile.notFound=Could not find requested segment index file
+
 task.create.noTasks=Zero tasks were requested
 task.create.failed=Failed to create Task
 task.create.limitExceeded=Cannot create more than 1000 tasks in one request.

unreleased_changes/8711.md

@@ -0,0 +1,5 @@
+### Added
+- Segment index files can now also be read from the new zarr3-based format, and from remote object storage.
+
+### Fixed
+- Fixed a bug where segment index files for segmentation layers with no mag1 would be read incorrectly.

webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala

@@ -4,13 +4,24 @@ import org.apache.pekko.actor.ActorSystem
 import com.google.inject.AbstractModule
 import com.google.inject.name.Names
 import com.scalableminds.webknossos.datastore.services._
+import com.scalableminds.webknossos.datastore.services.mapping.{
+  AgglomerateService,
+  Hdf5AgglomerateService,
+  MappingService,
+  ZarrAgglomerateService
+}
 import com.scalableminds.webknossos.datastore.services.mesh.{
   AdHocMeshServiceHolder,
   Hdf5MeshFileService,
   MeshFileService,
   NeuroglancerPrecomputedMeshFileService,
   ZarrMeshFileService
 }
+import com.scalableminds.webknossos.datastore.services.segmentindex.{
+  Hdf5SegmentIndexFileService,
+  SegmentIndexFileService,
+  ZarrSegmentIndexFileService
+}
 import com.scalableminds.webknossos.datastore.services.uploading.UploadService
 import com.scalableminds.webknossos.datastore.storage.{DataVaultService, RemoteSourceDescriptorService}
 
@@ -38,6 +49,9 @@ class DataStoreModule extends AbstractModule {
     bind(classOf[AgglomerateService]).asEagerSingleton()
     bind(classOf[ZarrAgglomerateService]).asEagerSingleton()
     bind(classOf[Hdf5AgglomerateService]).asEagerSingleton()
+    bind(classOf[SegmentIndexFileService]).asEagerSingleton()
+    bind(classOf[ZarrSegmentIndexFileService]).asEagerSingleton()
+    bind(classOf[Hdf5SegmentIndexFileService]).asEagerSingleton()
     bind(classOf[NeuroglancerPrecomputedMeshFileService]).asEagerSingleton()
     bind(classOf[RemoteSourceDescriptorService]).asEagerSingleton()
     bind(classOf[ChunkCacheService]).asEagerSingleton()

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala

@@ -21,6 +21,7 @@ import com.scalableminds.webknossos.datastore.services._
 import com.scalableminds.webknossos.datastore.services.mesh.{AdHocMeshRequest, AdHocMeshService, AdHocMeshServiceHolder}
 import com.scalableminds.webknossos.datastore.slacknotification.DSSlackNotificationService
 import com.scalableminds.util.tools.Box.tryo
+import com.scalableminds.webknossos.datastore.services.mapping.MappingService
 import play.api.i18n.Messages
 import play.api.libs.json.Json
 import play.api.mvc.{AnyContent, _}

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala

@@ -35,8 +35,10 @@ trait SegmentStatistics extends ProtoGeometryImplicits with FoxImplicits {
       implicit ec: ExecutionContext): Fox[Long] =
     for {
       bucketPositionsProtos: Set[Vec3IntProto] <- getBucketPositions(segmentId, mag)
-      bucketPositionsInMag = bucketPositionsProtos.map(vec3IntFromProto)
-      (bucketBoxes, elementClass) <- getDataForBucketPositions(bucketPositionsInMag.toSeq, mag, additionalCoordinates)
+      bucketPositionsInRequestedMag = bucketPositionsProtos.map(vec3IntFromProto)
+      (bucketBoxes, elementClass) <- getDataForBucketPositions(bucketPositionsInRequestedMag.toSeq,
+                                                               mag,
+                                                               additionalCoordinates)
       counts <- Fox.serialCombined(bucketBoxes.toList) {
         case Full(bucketBytes) =>
           tryo(

webknossos-datastore/app/com/scalableminds/webknossos/datastore/helpers/SegmentStatistics.scala

@@ -38,7 +38,16 @@ object LayerAttachmentType extends ExtendedEnumeration {
 case class LayerAttachment(name: String,
                            path: URI,
                            dataFormat: LayerAttachmentDataformat.LayerAttachmentDataformat,
-                           credentialId: Option[String] = None)
+                           credentialId: Option[String] = None) {
+  // Warning: throws! Use inside of tryo
+  def localPath: Path = {
+    if (path.getScheme.nonEmpty && path.getScheme != "file") {
+      throw new Exception(
+        "Trying to open non-local hdf5 file. Hdf5 files are only supported on the datastore-local file system.")
+    }
+    Path.of(path)
+  }
+}
 
 object LayerAttachment {
   implicit val jsonFormat: Format[LayerAttachment] = Json.format[LayerAttachment]

webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DatasetLayerAttachments.scala

@@ -4,14 +4,13 @@ import org.apache.commons.codec.digest.MurmurHash3
 
 import java.nio.ByteBuffer
 
-trait Hdf5HashedArrayUtils {
-
-  val hdf5FileExtension = "hdf5"
+trait ArrayArtifactHashing {
 
   def getHashFunction(name: String): Long => Long = name match {
     case "identity" => identity
     case "murmurhash3_x64_128" =>
       x: Long =>
         Math.abs(MurmurHash3.hash128x64(ByteBuffer.allocate(8).putLong(x).array())(1))
   }
+
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/Hdf5HashedArrayUtils.scala renamed to webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/ArrayArtifactHashing.scala

@@ -14,6 +14,7 @@ import com.typesafe.scalalogging.LazyLogging
 import com.scalableminds.util.tools.{Box, Empty, Full}
 import ucar.ma2.{Array => MultiArray}
 import com.scalableminds.util.tools.Box.tryo
+import com.scalableminds.webknossos.datastore.services.mapping.AgglomerateService
 
 import java.nio.file.Path
 import scala.concurrent.ExecutionContext

webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala

@@ -2,6 +2,7 @@ package com.scalableminds.webknossos.datastore.services
 
 import java.nio.file.Paths
 import com.scalableminds.webknossos.datastore.DataStoreConfig
+import com.scalableminds.webknossos.datastore.services.mapping.AgglomerateService
 import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService
 
 import javax.inject.Inject