Read Zarr Mesh Files (#8682)

- Similar to #8633 we now support reading mesh files in zarr format.
Note that there are also neuroglancerPrecomputed meshes, so this time
there are three services the meshFileService delegates to.

- The frontend no longer needs to specially handle
neuroglancerPrecomputed meshes, since the lookUpMeshFileKey function
abstracts from that. Because of that, I also simplified the frontend
types.

- Note that zarr meshfiles must be registered in the
datasource-properties.json, they are not explored.

- Also fixed a small error in job result status reporting

- Also fixed a small error in mag selection for animation job.

### Steps to test:
- Load some precomputed meshes (both zarr, hdf5,
neuroglancerPrecomputed; talk to me for example datasets), should show
meshes correctly
- Create animations via wk worker, should also work correctly (I tested
this locally)
- Note that neuroglancerPrecomputed meshes now need to be registered as
attachments in the datasource-properties.json. You may need to
re-explore the relevant datasets or edit the json.

### TODOs:
- [x] introduce and look up MeshFileKey
- [x] delegate to zarr, hdf5 or neuroglancer meshfile services
- [x] explore remote neuroglancer meshes
- [x] parse new meshfile attributes zarr group format
- [x] do we still need the null vs none meshfile-mapping attribute?
- [x] test with frontend (zarr,hdf5,neuro)
- [x] adapt frontend to no longer need path + fileType for meshfiles
- [x] clear caches
- [x] unify fullMeshStl route
- [x] unify spelling (meshfile vs meshFile)
- [x] When exploring remote neuroglancer meshes, add credentialId, and
use it? (or create follow-up issue)
- [x] re-add singletons to module after merge of zarr-agglomerate’s
latest changes
- [x] unify function names with zarr-agglomerate’s latest changes

### Issues:
- contributes to #8618
- contributes to #8567

------
- [x] Updated
[changelog](../blob/master/CHANGELOG.unreleased.md#unreleased)
- [x] Considered [common edge
cases](../blob/master/.github/common_edge_cases.md)
- [x] Needs datastore update after deployment

---------

Co-authored-by: MichaelBuessemeyer <39529669+MichaelBuessemeyer@users.noreply.github.com>

Author: fm3

app/controllers/WKRemoteWorkerController.scala

@@ -79,16 +79,18 @@ class WKRemoteWorkerController @Inject()(jobDAO: JobDAO,
       for {
         _ <- workerDAO.findOneByKey(key) ?~> "job.worker.notFound"
         jobBeforeChange <- jobDAO.findOne(id)(GlobalAccessContext)
-        _ <- jobDAO.updateStatus(id, request.body)
+        _ <- jobDAO.updateStatus(id, request.body) ?~> "job.updateStatus.failed"
         jobAfterChange <- jobDAO.findOne(id)(GlobalAccessContext) ?~> "job.notFound"
         _ = jobService.trackStatusChange(jobBeforeChange, jobAfterChange)
         _ <- jobService.cleanUpIfFailed(jobAfterChange) ?~> "job.cleanup.failed"
         _ <- Fox.runIf(request.body.state == JobState.SUCCESS) {
-          creditTransactionService.completeTransactionOfJob(jobAfterChange._id)(GlobalAccessContext)
+          creditTransactionService
+            .completeTransactionOfJob(jobAfterChange._id)(GlobalAccessContext) ?~> "job.creditTransaction.failed"
         }
         _ <- Fox.runIf(
           jobAfterChange.state != request.body.state && (request.body.state == JobState.FAILURE || request.body.state == JobState.CANCELLED)) {
-          creditTransactionService.refundTransactionForJob(jobAfterChange._id)(GlobalAccessContext)
+          creditTransactionService
+            .refundTransactionForJob(jobAfterChange._id)(GlobalAccessContext) ?~> "job.creditTransaction.refund.failed"
         }
       } yield Ok
   }

app/models/organization/CreditTransactionService.scala

@@ -2,7 +2,7 @@ package models.organization
 
 import com.scalableminds.util.accesscontext.DBAccessContext
 import com.scalableminds.util.objectid.ObjectId
-import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.util.tools.{Empty, Failure, Fox, FoxImplicits, Full}
 import com.typesafe.scalalogging.LazyLogging
 import play.api.libs.json.{JsObject, Json}
 
@@ -42,15 +42,30 @@ class CreditTransactionService @Inject()(creditTransactionDAO: CreditTransaction
 
   def completeTransactionOfJob(jobId: ObjectId)(implicit ctx: DBAccessContext): Fox[Unit] =
     for {
-      transaction <- creditTransactionDAO.findTransactionForJob(jobId)
-      _ <- organizationService.assertOrganizationHasPaidPlan(transaction._organization)
-      _ <- creditTransactionDAO.commitTransaction(transaction._id)
+      transactionBox <- creditTransactionDAO.findTransactionForJob(jobId).shiftBox
+      _ <- transactionBox match {
+        case Full(transaction) =>
+          for {
+            _ <- organizationService.assertOrganizationHasPaidPlan(transaction._organization)
+            _ <- creditTransactionDAO.commitTransaction(transaction._id)
+          } yield ()
+        case Empty      => Fox.successful(()) // Assume transaction-less Job
+        case f: Failure => f.toFox
+      }
+
     } yield ()
 
   def refundTransactionForJob(jobId: ObjectId)(implicit ctx: DBAccessContext): Fox[Unit] =
     for {
-      transaction <- creditTransactionDAO.findTransactionForJob(jobId)
-      _ <- refundTransaction(transaction)
+      transactionBox <- creditTransactionDAO.findTransactionForJob(jobId).shiftBox
+      _ <- transactionBox match {
+        case Full(transaction) =>
+          for {
+            _ <- refundTransaction(transaction)
+          } yield ()
+        case Empty      => Fox.successful(()) // Assume transaction-less Job
+        case f: Failure => f.toFox
+      }
     } yield ()
 
   private def refundTransaction(creditTransaction: CreditTransaction)(implicit ctx: DBAccessContext): Fox[Unit] =

conf/evolutions/135-neuroglancer-attachment.sql

@@ -0,0 +1,9 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 134, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+ALTER TYPE webknossos.LAYER_ATTACHMENT_DATAFORMAT ADD VALUE 'neuroglancerPrecomputed';
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 135;
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/135-neuroglancer-attachment.sql

@@ -0,0 +1 @@
+-- Removing enum types directly is not possible so no reversion is available for this.

conf/messages

@@ -265,6 +265,10 @@ mesh.file.listChunks.failed=Failed to load chunk list for segment {0} from mesh
 mesh.file.loadChunk.failed=Failed to load mesh chunk for segment
 mesh.file.open.failed=Failed to open mesh file for reading
 mesh.file.readEncoding.failed=Failed to read encoding from mesh file
+mesh.file.lookup.failed=Failed to look up mesh file “{0}”
+mesh.file.readVersion.failed=Failed to read format version from file “{0}”
+mesh.file.readMappingName.failed=Failed to read mapping name from mesh file “{0}”
+mesh.meshFileName.required=Trying to load mesh from mesh file, but mesh file name was not supplied.
 
 task.create.noTasks=Zero tasks were requested
 task.create.failed=Failed to create Task
@@ -347,7 +351,9 @@ job.trainModel.notAllowed.organization = Training AI models is only allowed for
 job.runInference.notAllowed.organization = Running inference is only allowed for datasets of your own organization.
 job.paidJob.notAllowed.noPaidPlan = You are not allowed to run this job because your organization does not have a paid plan.
 job.notEnoughCredits = Your organization does not have enough WEBKNOSSOS credits to run this job.
-creditTransaction.notPaidPlan = Your organization does not have a paid plan.
+job.updateStatus.failed = Failed to update long-running job’s status
+job.creditTransaction.failed = Failed to perform credit transaction
+job.creditTransaction.refund.failed = Failed to perform credit transaction refund
 
 voxelytics.disabled = Voxelytics workflow reporting and logging are not enabled for this WEBKNOSSOS instance.
 voxelytics.runNotFound = Workflow runs not found
@@ -368,10 +374,10 @@ folder.notFound=Could not find the requested folder
 folder.delete.root=Cannot delete the organization’s root folder
 folder.move.root=Cannot move the organization’s root folder
 folder.update.notAllowed=No write access on this folder
+folder.noWriteAccess=No write access in this folder
 folder.update.name.failed=Failed to update the folder’s name
 folder.update.teams.failed=Failed to update the folder’s allowed teams
 folder.create.failed.teams.failed=Failed to create folder in this location
-folder.noWriteAccess=No write access in this folder
 folder.nameMustNotContainSlash=Folder names cannot contain forward slashes
 
 segmentAnything.notEnabled=AI based quick select is not enabled for this WEBKNOSSOS instance.

frontend/javascripts/admin/api/mesh.ts

@@ -22,7 +22,7 @@ type MeshSegmentInfo = {
 };
 
 type ListMeshChunksRequest = {
-  meshFile: APIMeshFileInfo;
+  meshFileName: string;
   segmentId: number;
 };
 
@@ -52,7 +52,7 @@ export function getMeshfileChunksForSegment(
       params.append("editableMappingTracingId", editableMappingTracingId);
     }
     const payload: ListMeshChunksRequest = {
-      meshFile,
+      meshFileName: meshFile.name,
       segmentId,
     };
     return Request.sendJSONReceiveJSON(
@@ -72,7 +72,7 @@ type MeshChunkDataRequest = {
 };
 
 type MeshChunkDataRequestList = {
-  meshFile: APIMeshFileInfo;
+  meshFileName: string;
   requests: MeshChunkDataRequest[];
 };
 

frontend/javascripts/types/api_types.ts

@@ -1005,12 +1005,10 @@ export type ServerEditableMapping = {
 
 export type APIMeshFileInfo = {
   name: string;
-  path: string | null | undefined;
-  fileType: string | null | undefined;
   mappingName?: string | null | undefined;
-  // 0   - is the first mesh file version
-  // 1-2 - the format should behave as v0 (refer to voxelytics for actual differences)
-  // 3   - is the newer version with draco encoding.
+  // 0   - unsupported (is the first mesh file version)
+  // 1-2 - unsupported (the format should behave as v0; refer to voxelytics for actual differences)
+  // 3+  - is the newer version with draco encoding.
   formatVersion: number;
 };
 export type APIConnectomeFile = {

frontend/javascripts/viewer/model/sagas/meshes/precomputed_mesh_saga.ts

@@ -365,7 +365,7 @@ function* loadPrecomputedMeshesInChunksForLod(
           dataset,
           getBaseSegmentationName(segmentationLayer),
           {
-            meshFile,
+            meshFileName: meshFile.name,
             // Only extract the relevant properties
             requests: chunks.map(({ byteOffset, byteSize }) => ({
               byteOffset,

frontend/javascripts/viewer/view/action-bar/create_animation_modal.tsx

@@ -258,7 +258,7 @@ function CreateAnimationModal(props: Props) {
       const adhocMagIndex = getMagInfo(layer.resolutions).getClosestExistingIndex(
         preferredQualityForMeshAdHocComputation,
       );
-      const adhocMag = layer.resolutions[adhocMagIndex];
+      const adhocMag = getMagInfo(layer.resolutions).getMagByIndexOrThrow(adhocMagIndex);
 
       return Object.values(meshInfos)
         .filter((meshInfo: MeshInformation) => meshInfo.isVisible)

tools/postgres/schema.sql

@@ -21,7 +21,7 @@ CREATE TABLE webknossos.releaseInformation (
   schemaVersion BIGINT NOT NULL
 );
 
-INSERT INTO webknossos.releaseInformation(schemaVersion) values(134);
+INSERT INTO webknossos.releaseInformation(schemaVersion) values(135);
 COMMIT TRANSACTION;
 
 
@@ -163,7 +163,7 @@ CREATE TABLE webknossos.dataset_layer_additionalAxes(
 );
 
 CREATE TYPE webknossos.LAYER_ATTACHMENT_TYPE AS ENUM ('agglomerate', 'connectome', 'segmentIndex', 'mesh', 'cumsum');
-CREATE TYPE webknossos.LAYER_ATTACHMENT_DATAFORMAT AS ENUM ('hdf5', 'zarr3', 'json');
+CREATE TYPE webknossos.LAYER_ATTACHMENT_DATAFORMAT AS ENUM ('hdf5', 'zarr3', 'json', 'neuroglancerPrecomputed');
 CREATE TABLE webknossos.dataset_layer_attachments(
    _dataset TEXT CONSTRAINT _dataset_objectId CHECK (_dataset ~ '^[0-9a-f]{24}$') NOT NULL,
    layerName TEXT NOT NULL,