From 7245fc1bc15d2eb279520c4aeaa8783dd6fb6da1 Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Wed, 14 May 2025 17:00:56 +0200 Subject: [PATCH 01/12] set mesh opacity after mesh was loaded --- .../javascripts/viewer/model/sagas/mesh_saga.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts index e5aeb4d639..c59123c449 100644 --- a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts +++ b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts @@ -50,6 +50,7 @@ import { } from "viewer/model/accessors/volumetracing_accessor"; import type { Action } from "viewer/model/actions/actions"; import { + type FinishedLoadingMeshAction, type MaybeFetchMeshFilesAction, type RefreshMeshAction, type RemoveMeshAction, @@ -1302,6 +1303,17 @@ function* handleSegmentColorChange(action: UpdateSegmentAction): Saga { } } +function* maybeSetMeshOpacity(action: FinishedLoadingMeshAction): Saga { + const { segmentMeshController } = yield* call(getSceneController); + const { layerName, segmentId } = action; + const meshInfo = yield* select((state) => + getMeshInfoForSegment(state, null, layerName, segmentId), + ); + if (meshInfo == null) return; + segmentMeshController.setMeshOpacity(segmentId, layerName, meshInfo.opacity); + console.log(`Set opacity of mesh ${segmentId} in layer ${layerName} to ${meshInfo.opacity}.`); +} + function* handleMeshOpacityChange(action: UpdateMeshOpacityAction): Saga { const { segmentMeshController } = yield* call(getSceneController); segmentMeshController.setMeshOpacity(action.id, action.layerName, action.opacity); @@ -1344,4 +1356,5 @@ export default function* meshSaga(): Saga { yield* takeEvery("UPDATE_SEGMENT", handleSegmentColorChange); yield* takeEvery("UPDATE_MESH_OPACITY", handleMeshOpacityChange); yield* takeEvery("BATCH_UPDATE_GROUPS_AND_SEGMENTS", handleBatchSegmentColorChange); + yield* takeEvery("FINISHED_LOADING_MESH", maybeSetMeshOpacity); } From bb9b5c557c309c638546a753e5a396da291e802e Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Wed, 14 May 2025 18:21:11 +0200 Subject: [PATCH 02/12] WIP: use mesh opacity upon reloading precomputed mesh --- .../controller/segment_mesh_controller.ts | 8 +++-- .../model/actions/segmentation_actions.ts | 2 ++ .../viewer/model/sagas/mesh_saga.ts | 31 ++++++++++--------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/frontend/javascripts/viewer/controller/segment_mesh_controller.ts b/frontend/javascripts/viewer/controller/segment_mesh_controller.ts index 55190b22e8..74ebac18f3 100644 --- a/frontend/javascripts/viewer/controller/segment_mesh_controller.ts +++ b/frontend/javascripts/viewer/controller/segment_mesh_controller.ts @@ -111,6 +111,7 @@ export default class SegmentMeshController { segmentId: number, layerName: string, additionalCoordinates?: AdditionalCoordinate[] | undefined | null, + opacity: number = 1, ): Promise { // Currently, this function is only used by ad hoc meshing. if (vertices.length === 0) return; @@ -129,6 +130,7 @@ export default class SegmentMeshController { NO_LOD_MESH_INDEX, layerName, additionalCoordinates, + opacity, false, ); } @@ -137,6 +139,7 @@ export default class SegmentMeshController { segmentId: number, layerName: string, geometry: BufferGeometryWithInfo, + opacity: number, isMerged: boolean, ): MeshSceneNode { const color = this.getColorObjectForSegment(segmentId, layerName); @@ -169,7 +172,7 @@ export default class SegmentMeshController { tweenAnimation .to( { - opacity: 1, + opacity, }, 100, ) @@ -189,6 +192,7 @@ export default class SegmentMeshController { lod: number, layerName: string, additionalCoordinates: AdditionalCoordinate[] | null | undefined, + opacity: number, isMerged: boolean, ): void { const additionalCoordinatesString = getAdditionalCoordinatesAsString(additionalCoordinates); @@ -232,7 +236,7 @@ export default class SegmentMeshController { ]; targetGroup.scale.copy(new THREE.Vector3(...adaptedScale)); } - const meshChunk = this.constructMesh(segmentId, layerName, geometry, isMerged); + const meshChunk = this.constructMesh(segmentId, layerName, geometry, opacity, isMerged); const group = new THREE.Group() as SceneGroupForMeshes; group.add(meshChunk); diff --git a/frontend/javascripts/viewer/model/actions/segmentation_actions.ts b/frontend/javascripts/viewer/model/actions/segmentation_actions.ts index f6fada6d7f..5f4995b038 100644 --- a/frontend/javascripts/viewer/model/actions/segmentation_actions.ts +++ b/frontend/javascripts/viewer/model/actions/segmentation_actions.ts @@ -34,6 +34,7 @@ export const loadPrecomputedMeshAction = ( seedPosition: Vector3, seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, meshFileName: string, + opacity?: number, layerName?: string | undefined, ) => ({ @@ -42,5 +43,6 @@ export const loadPrecomputedMeshAction = ( seedPosition, seedAdditionalCoordinates, meshFileName, + opacity, layerName, }) as const; diff --git a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts index c59123c449..b58d48fb34 100644 --- a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts +++ b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts @@ -28,7 +28,7 @@ import type { APIDataset, APIMeshFileInfo, APISegmentationLayer } from "types/ap import type { AdditionalCoordinate } from "types/api_types"; import { WkDevFlags } from "viewer/api/wk_dev"; import type { Vector3, Vector4 } from "viewer/constants"; -import { MappingStatusEnum } from "viewer/constants"; +import Constants, { MappingStatusEnum } from "viewer/constants"; import CustomLOD from "viewer/controller/custom_lod"; import { type BufferGeometryWithInfo, @@ -50,7 +50,6 @@ import { } from "viewer/model/accessors/volumetracing_accessor"; import type { Action } from "viewer/model/actions/actions"; import { - type FinishedLoadingMeshAction, type MaybeFetchMeshFilesAction, type RefreshMeshAction, type RemoveMeshAction, @@ -66,6 +65,7 @@ import { startedLoadingMeshAction, updateCurrentMeshFileAction, updateMeshFileListAction, + updateMeshOpacityAction, updateMeshVisibilityAction, } from "viewer/model/actions/annotation_actions"; import { saveNowAction } from "viewer/model/actions/save_actions"; @@ -607,6 +607,7 @@ function* refreshMesh(action: RefreshMeshAction): Saga { const meshInfo = yield* select((state) => getMeshInfoForSegment(state, additionalCoordinates, layerName, segmentId), ); + console.log("opacity", meshInfo?.opacity); if (meshInfo == null) { throw new Error( @@ -622,6 +623,7 @@ function* refreshMesh(action: RefreshMeshAction): Saga { meshInfo.seedPosition, meshInfo.seedAdditionalCoordinates, meshInfo.meshFileName, + meshInfo.opacity, layerName, ), ); @@ -746,7 +748,8 @@ function* maybeFetchMeshFiles(action: MaybeFetchMeshFilesAction): Saga { } function* loadPrecomputedMesh(action: LoadPrecomputedMeshAction) { - const { segmentId, seedPosition, seedAdditionalCoordinates, meshFileName, layerName } = action; + const { segmentId, seedPosition, seedAdditionalCoordinates, meshFileName, layerName, opacity } = + action; const layer = yield* select((state) => layerName != null ? getSegmentationLayerByName(state.dataset, layerName) @@ -769,6 +772,7 @@ function* loadPrecomputedMesh(action: LoadPrecomputedMeshAction) { seedAdditionalCoordinates, meshFileName, layer, + opacity || Constants.DEFAULT_MESH_OPACITY, ), cancel: take( ((otherAction: Action) => @@ -787,6 +791,7 @@ function* loadPrecomputedMeshForSegmentId( seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, meshFileName: string, segmentationLayer: APISegmentationLayer, + opacity: number, ): Saga { const layerName = segmentationLayer.name; const mappingName = yield* call(getMappingName, segmentationLayer); @@ -865,10 +870,14 @@ function* loadPrecomputedMeshForSegmentId( (lod: number) => extractScaleFromMatrix(lods[lod].transform), chunkScale, additionalCoordinates, + opacity, ); } yield* put(finishedLoadingMeshAction(layerName, segmentId)); + if (opacity != null) { + yield* put(updateMeshOpacityAction(layerName, segmentId, opacity)); + } } function* getMappingName(segmentationLayer: APISegmentationLayer) { @@ -973,6 +982,7 @@ function* loadPrecomputedMeshesInChunksForLod( getGlobalScale: (lod: number) => Vector3 | null, chunkScale: Vector3 | null, additionalCoordinates: AdditionalCoordinate[] | null, + opacity: number, ) { const { segmentMeshController } = getSceneController(); const loader = getDracoLoader(); @@ -1048,6 +1058,7 @@ function* loadPrecomputedMeshesInChunksForLod( lod, layerName, additionalCoordinates, + opacity, false, ); @@ -1115,6 +1126,7 @@ function* loadPrecomputedMeshesInChunksForLod( lod, layerName, additionalCoordinates, + opacity, true, ); } @@ -1303,18 +1315,8 @@ function* handleSegmentColorChange(action: UpdateSegmentAction): Saga { } } -function* maybeSetMeshOpacity(action: FinishedLoadingMeshAction): Saga { - const { segmentMeshController } = yield* call(getSceneController); - const { layerName, segmentId } = action; - const meshInfo = yield* select((state) => - getMeshInfoForSegment(state, null, layerName, segmentId), - ); - if (meshInfo == null) return; - segmentMeshController.setMeshOpacity(segmentId, layerName, meshInfo.opacity); - console.log(`Set opacity of mesh ${segmentId} in layer ${layerName} to ${meshInfo.opacity}.`); -} - function* handleMeshOpacityChange(action: UpdateMeshOpacityAction): Saga { + console.log("handleMeshOpacityChange", action); const { segmentMeshController } = yield* call(getSceneController); segmentMeshController.setMeshOpacity(action.id, action.layerName, action.opacity); } @@ -1356,5 +1358,4 @@ export default function* meshSaga(): Saga { yield* takeEvery("UPDATE_SEGMENT", handleSegmentColorChange); yield* takeEvery("UPDATE_MESH_OPACITY", handleMeshOpacityChange); yield* takeEvery("BATCH_UPDATE_GROUPS_AND_SEGMENTS", handleBatchSegmentColorChange); - yield* takeEvery("FINISHED_LOADING_MESH", maybeSetMeshOpacity); } From d8cb7d3427158c98f81d55a5cf9c3e27279cf797 Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Thu, 15 May 2025 21:28:20 +0200 Subject: [PATCH 03/12] use opacity for ad hoc and precomputed meshes --- .../controller/segment_mesh_controller.ts | 1 + .../model/actions/annotation_actions.ts | 3 ++ .../model/actions/segmentation_actions.ts | 1 + .../model/reducers/annotation_reducer.ts | 16 +++++++++- .../viewer/model/sagas/mesh_saga.ts | 29 ++++++++++++++----- 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/frontend/javascripts/viewer/controller/segment_mesh_controller.ts b/frontend/javascripts/viewer/controller/segment_mesh_controller.ts index 74ebac18f3..d1ff049aee 100644 --- a/frontend/javascripts/viewer/controller/segment_mesh_controller.ts +++ b/frontend/javascripts/viewer/controller/segment_mesh_controller.ts @@ -159,6 +159,7 @@ export default class SegmentMeshController { colorBuffer.set(colorArray, i * 3); } geometry.setAttribute("color", new THREE.BufferAttribute(colorBuffer, 3)); + console.log("geometry", geometry, opacity); // mesh.parent is still null at this moment, but when the mesh is // added to the group later, parent will be set. We'll ignore diff --git a/frontend/javascripts/viewer/model/actions/annotation_actions.ts b/frontend/javascripts/viewer/model/actions/annotation_actions.ts index 36912d19c1..73f7546a41 100644 --- a/frontend/javascripts/viewer/model/actions/annotation_actions.ts +++ b/frontend/javascripts/viewer/model/actions/annotation_actions.ts @@ -12,6 +12,7 @@ import type { } from "types/api_types"; import type { AdditionalCoordinate } from "types/api_types"; import type { Vector3 } from "viewer/constants"; +import Constants from "viewer/constants"; import type { Annotation, MappingType, @@ -333,6 +334,7 @@ export const addAdHocMeshAction = ( seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, mappingName: string | null | undefined, mappingType: MappingType | null | undefined, + opacity: number = Constants.DEFAULT_MESH_OPACITY, ) => ({ type: "ADD_AD_HOC_MESH", @@ -342,6 +344,7 @@ export const addAdHocMeshAction = ( seedAdditionalCoordinates, mappingName, mappingType, + opacity, }) as const; export const addPrecomputedMeshAction = ( diff --git a/frontend/javascripts/viewer/model/actions/segmentation_actions.ts b/frontend/javascripts/viewer/model/actions/segmentation_actions.ts index 5f4995b038..2bb054c187 100644 --- a/frontend/javascripts/viewer/model/actions/segmentation_actions.ts +++ b/frontend/javascripts/viewer/model/actions/segmentation_actions.ts @@ -7,6 +7,7 @@ export type AdHocMeshInfo = { mappingType: MappingType | null | undefined; useDataStore?: boolean | null | undefined; preferredQuality?: number | null | undefined; + opacity?: number | null | undefined; }; export type LoadAdHocMeshAction = ReturnType; export type LoadPrecomputedMeshAction = ReturnType; diff --git a/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts b/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts index 617ae3eade..d990e9756a 100644 --- a/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts +++ b/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts @@ -327,6 +327,7 @@ function AnnotationReducer(state: WebknossosState, action: Action): WebknossosSt additionalCoordinates, layerName, ); + console.log("REMOVE_MESH", maybeMeshes, state.localSegmentationData[layerName]); if (maybeMeshes == null || maybeMeshes[segmentId] == null) { // No meshes exist for the segment id. No need to do anything. return state; @@ -353,6 +354,7 @@ function AnnotationReducer(state: WebknossosState, action: Action): WebknossosSt seedAdditionalCoordinates, mappingName, mappingType, + opacity, } = action; const meshInfo: MeshInformation = { segmentId: segmentId, @@ -361,7 +363,7 @@ function AnnotationReducer(state: WebknossosState, action: Action): WebknossosSt isLoading: false, isVisible: true, isPrecomputed: false, - opacity: Constants.DEFAULT_MESH_OPACITY, + opacity, mappingName, mappingType, }; @@ -454,6 +456,12 @@ function AnnotationReducer(state: WebknossosState, action: Action): WebknossosSt }, }, }); + console.log( + "STARTED_LOADING_MESH", + layerName, + segmentId, + state.localSegmentationData[layerName], + ); //TODO_c return updatedKey; } @@ -477,6 +485,12 @@ function AnnotationReducer(state: WebknossosState, action: Action): WebknossosSt }, }, }); + console.log( + "FINISHED_LOADING_MESH", + layerName, + segmentId, + state.localSegmentationData[layerName], + ); //TODO_c return updatedKey; } diff --git a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts index b58d48fb34..c3edadad0a 100644 --- a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts +++ b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts @@ -65,7 +65,6 @@ import { startedLoadingMeshAction, updateCurrentMeshFileAction, updateMeshFileListAction, - updateMeshOpacityAction, updateMeshVisibilityAction, } from "viewer/model/actions/annotation_actions"; import { saveNowAction } from "viewer/model/actions/save_actions"; @@ -344,7 +343,7 @@ function* loadFullAdHocMesh( removeExistingMesh: boolean, ): Saga { let isInitialRequest = true; - const { mappingName, mappingType } = meshExtraInfo; + const { mappingName, mappingType, opacity } = meshExtraInfo; const clippedPosition = clipPositionToCubeBoundary(position, zoomStep, magInfo); yield* put( addAdHocMeshAction( @@ -354,6 +353,7 @@ function* loadFullAdHocMesh( additionalCoordinates, mappingName, mappingType, + opacity || Constants.DEFAULT_MESH_OPACITY, ), ); yield* put(startedLoadingMeshAction(layer.name, segmentId)); @@ -528,6 +528,8 @@ function* maybeLoadMeshChunk( segmentMeshController.removeMeshById(segmentId, layer.name); } + const opacity = meshExtraInfo.opacity || Constants.DEFAULT_MESH_OPACITY; + // We await addMeshFromVerticesAsync here, because the mesh saga will remove // an ad-hoc loaded mesh immediately if it was "empty". Since the check is // done by looking at the scene, we await the population of the scene. @@ -540,6 +542,7 @@ function* maybeLoadMeshChunk( segmentId, layer.name, additionalCoordinates, + opacity, ); return neighbors.map((neighbor) => getNeighborPosition(clippedPosition, neighbor, zoomStep, magInfo), @@ -607,7 +610,6 @@ function* refreshMesh(action: RefreshMeshAction): Saga { const meshInfo = yield* select((state) => getMeshInfoForSegment(state, additionalCoordinates, layerName, segmentId), ); - console.log("opacity", meshInfo?.opacity); if (meshInfo == null) { throw new Error( @@ -633,7 +635,14 @@ function* refreshMesh(action: RefreshMeshAction): Saga { if (threeDMap == null) { return; } - yield* call(_refreshMeshWithMap, segmentId, threeDMap, layerName, additionalCoordinates); + yield* call( + _refreshMeshWithMap, + segmentId, + threeDMap, + layerName, + additionalCoordinates, + meshInfo.opacity, + ); } } @@ -642,6 +651,7 @@ function* _refreshMeshWithMap( threeDMap: ThreeDMap, layerName: string, additionalCoordinates: AdditionalCoordinate[] | null, + opacity: number = Constants.DEFAULT_MESH_OPACITY, ): Saga { const meshInfo = yield* select((state) => getMeshInfoForSegment(state, additionalCoordinates, layerName, segmentId), @@ -668,6 +678,10 @@ function* _refreshMeshWithMap( yield* call(removeMesh, removeMeshAction(layerName, segmentId), false); // The mesh should only be removed once after re-fetching the mesh first position. let shouldBeRemoved = true; + const meshInfo2 = yield* select((state) => + getMeshInfoForSegment(state, additionalCoordinates, layerName, segmentId), + ); + console.log(meshInfo2); for (const [, position] of meshPositions) { // Reload the mesh at the given position if it isn't already loaded there. @@ -682,6 +696,7 @@ function* _refreshMeshWithMap( { mappingName, mappingType, + opacity, }, ); shouldBeRemoved = false; @@ -805,6 +820,9 @@ function* loadPrecomputedMeshForSegmentId( mappingName, ), ); + if (opacity != null) { + //yield* put(updateMeshOpacityAction(layerName, segmentId, opacity)); + } yield* put(startedLoadingMeshAction(layerName, segmentId)); const dataset = yield* select((state) => state.dataset); const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); @@ -875,9 +893,6 @@ function* loadPrecomputedMeshForSegmentId( } yield* put(finishedLoadingMeshAction(layerName, segmentId)); - if (opacity != null) { - yield* put(updateMeshOpacityAction(layerName, segmentId, opacity)); - } } function* getMappingName(segmentationLayer: APISegmentationLayer) { From 7f8c927df413dc6669e8b446891c36c951361e30 Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Thu, 15 May 2025 21:56:47 +0200 Subject: [PATCH 04/12] start cleaning code --- .../javascripts/viewer/controller/segment_mesh_controller.ts | 3 +-- frontend/javascripts/viewer/model/sagas/mesh_saga.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/frontend/javascripts/viewer/controller/segment_mesh_controller.ts b/frontend/javascripts/viewer/controller/segment_mesh_controller.ts index d1ff049aee..5c5de11c38 100644 --- a/frontend/javascripts/viewer/controller/segment_mesh_controller.ts +++ b/frontend/javascripts/viewer/controller/segment_mesh_controller.ts @@ -110,8 +110,8 @@ export default class SegmentMeshController { vertices: Float32Array, segmentId: number, layerName: string, + opacity: number, additionalCoordinates?: AdditionalCoordinate[] | undefined | null, - opacity: number = 1, ): Promise { // Currently, this function is only used by ad hoc meshing. if (vertices.length === 0) return; @@ -159,7 +159,6 @@ export default class SegmentMeshController { colorBuffer.set(colorArray, i * 3); } geometry.setAttribute("color", new THREE.BufferAttribute(colorBuffer, 3)); - console.log("geometry", geometry, opacity); // mesh.parent is still null at this moment, but when the mesh is // added to the group later, parent will be set. We'll ignore diff --git a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts index c3edadad0a..85004840cb 100644 --- a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts +++ b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts @@ -541,8 +541,8 @@ function* maybeLoadMeshChunk( vertices, segmentId, layer.name, - additionalCoordinates, opacity, + additionalCoordinates, ); return neighbors.map((neighbor) => getNeighborPosition(clippedPosition, neighbor, zoomStep, magInfo), From c87cdd2f92efba499ee820bbeacab182ef9cf2bc Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Tue, 20 May 2025 22:21:21 +0200 Subject: [PATCH 05/12] fix opacity reloading for precomputed meshes --- frontend/javascripts/viewer/api/api_latest.ts | 1 + .../viewer/model/actions/annotation_actions.ts | 2 ++ .../viewer/model/actions/segmentation_actions.ts | 2 +- .../viewer/model/reducers/annotation_reducer.ts | 16 ++-------------- .../javascripts/viewer/model/sagas/mesh_saga.ts | 10 ++-------- .../viewer/model/sagas/proofread_saga.ts | 3 ++- .../javascripts/viewer/view/context_menu.tsx | 3 ++- .../segments_tab/segments_view.tsx | 10 ++++++++-- 8 files changed, 20 insertions(+), 27 deletions(-) diff --git a/frontend/javascripts/viewer/api/api_latest.ts b/frontend/javascripts/viewer/api/api_latest.ts index 2f26d77ab4..bc44e579fc 100644 --- a/frontend/javascripts/viewer/api/api_latest.ts +++ b/frontend/javascripts/viewer/api/api_latest.ts @@ -2425,6 +2425,7 @@ class DataApi { seedPosition, seedAdditionalCoordinates, meshFileName, + Constants.DEFAULT_MESH_OPACITY, effectiveLayerName, ), ); diff --git a/frontend/javascripts/viewer/model/actions/annotation_actions.ts b/frontend/javascripts/viewer/model/actions/annotation_actions.ts index 73f7546a41..78b6f15bd3 100644 --- a/frontend/javascripts/viewer/model/actions/annotation_actions.ts +++ b/frontend/javascripts/viewer/model/actions/annotation_actions.ts @@ -354,6 +354,7 @@ export const addPrecomputedMeshAction = ( seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, meshFileName: string, mappingName: string | null | undefined, + opacity: number, ) => ({ type: "ADD_PRECOMPUTED_MESH", @@ -363,6 +364,7 @@ export const addPrecomputedMeshAction = ( seedAdditionalCoordinates, meshFileName, mappingName, + opacity, }) as const; export const setOthersMayEditForAnnotationAction = (othersMayEdit: boolean) => diff --git a/frontend/javascripts/viewer/model/actions/segmentation_actions.ts b/frontend/javascripts/viewer/model/actions/segmentation_actions.ts index 2bb054c187..a157b42277 100644 --- a/frontend/javascripts/viewer/model/actions/segmentation_actions.ts +++ b/frontend/javascripts/viewer/model/actions/segmentation_actions.ts @@ -35,7 +35,7 @@ export const loadPrecomputedMeshAction = ( seedPosition: Vector3, seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, meshFileName: string, - opacity?: number, + opacity: number, layerName?: string | undefined, ) => ({ diff --git a/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts b/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts index d990e9756a..208ac421d9 100644 --- a/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts +++ b/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts @@ -327,7 +327,6 @@ function AnnotationReducer(state: WebknossosState, action: Action): WebknossosSt additionalCoordinates, layerName, ); - console.log("REMOVE_MESH", maybeMeshes, state.localSegmentationData[layerName]); if (maybeMeshes == null || maybeMeshes[segmentId] == null) { // No meshes exist for the segment id. No need to do anything. return state; @@ -400,6 +399,7 @@ function AnnotationReducer(state: WebknossosState, action: Action): WebknossosSt seedAdditionalCoordinates, meshFileName, mappingName, + opacity, } = action; const meshInfo: MeshInformation = { segmentId: segmentId, @@ -408,7 +408,7 @@ function AnnotationReducer(state: WebknossosState, action: Action): WebknossosSt isLoading: false, isVisible: true, isPrecomputed: true, - opacity: Constants.DEFAULT_MESH_OPACITY, + opacity: opacity || Constants.DEFAULT_MESH_OPACITY, meshFileName, mappingName, }; @@ -456,12 +456,6 @@ function AnnotationReducer(state: WebknossosState, action: Action): WebknossosSt }, }, }); - console.log( - "STARTED_LOADING_MESH", - layerName, - segmentId, - state.localSegmentationData[layerName], - ); //TODO_c return updatedKey; } @@ -485,12 +479,6 @@ function AnnotationReducer(state: WebknossosState, action: Action): WebknossosSt }, }, }); - console.log( - "FINISHED_LOADING_MESH", - layerName, - segmentId, - state.localSegmentationData[layerName], - ); //TODO_c return updatedKey; } diff --git a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts index 85004840cb..8b1d3da7a3 100644 --- a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts +++ b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts @@ -678,10 +678,6 @@ function* _refreshMeshWithMap( yield* call(removeMesh, removeMeshAction(layerName, segmentId), false); // The mesh should only be removed once after re-fetching the mesh first position. let shouldBeRemoved = true; - const meshInfo2 = yield* select((state) => - getMeshInfoForSegment(state, additionalCoordinates, layerName, segmentId), - ); - console.log(meshInfo2); for (const [, position] of meshPositions) { // Reload the mesh at the given position if it isn't already loaded there. @@ -774,6 +770,7 @@ function* loadPrecomputedMesh(action: LoadPrecomputedMeshAction) { // Remove older mesh instance if it exists already. yield* put(removeMeshAction(layer.name, action.segmentId)); + console.log("Loading precomputed mesh for segment", segmentId, opacity); // If a REMOVE_MESH action is dispatched and consumed // here before loadPrecomputedMeshForSegmentId is finished, the latter saga @@ -818,11 +815,9 @@ function* loadPrecomputedMeshForSegmentId( seedAdditionalCoordinates, meshFileName, mappingName, + opacity, ), ); - if (opacity != null) { - //yield* put(updateMeshOpacityAction(layerName, segmentId, opacity)); - } yield* put(startedLoadingMeshAction(layerName, segmentId)); const dataset = yield* select((state) => state.dataset); const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); @@ -1331,7 +1326,6 @@ function* handleSegmentColorChange(action: UpdateSegmentAction): Saga { } function* handleMeshOpacityChange(action: UpdateMeshOpacityAction): Saga { - console.log("handleMeshOpacityChange", action); const { segmentMeshController } = yield* call(getSceneController); segmentMeshController.setMeshOpacity(action.id, action.layerName, action.opacity); } diff --git a/frontend/javascripts/viewer/model/sagas/proofread_saga.ts b/frontend/javascripts/viewer/model/sagas/proofread_saga.ts index 63f319fc12..74bbf89712 100644 --- a/frontend/javascripts/viewer/model/sagas/proofread_saga.ts +++ b/frontend/javascripts/viewer/model/sagas/proofread_saga.ts @@ -11,7 +11,7 @@ import { SoftError, isBigInt, isNumberMap } from "libs/utils"; import _ from "lodash"; import { all, call, put, spawn, takeEvery } from "typed-redux-saga"; import type { AdditionalCoordinate, ServerEditableMapping } from "types/api_types"; -import { MappingStatusEnum, TreeTypeEnum, type Vector3 } from "viewer/constants"; +import Constants, { MappingStatusEnum, TreeTypeEnum, type Vector3 } from "viewer/constants"; import { getSegmentIdForPositionAsync } from "viewer/controller/combinations/volume_handlers"; import { getLayerByName, @@ -180,6 +180,7 @@ function* loadCoarseMesh( position, additionalCoordinates, currentMeshFile.name, + Constants.DEFAULT_MESH_OPACITY, undefined, ), ); diff --git a/frontend/javascripts/viewer/view/context_menu.tsx b/frontend/javascripts/viewer/view/context_menu.tsx index c81658bc54..ea182caa62 100644 --- a/frontend/javascripts/viewer/view/context_menu.tsx +++ b/frontend/javascripts/viewer/view/context_menu.tsx @@ -37,7 +37,7 @@ import type { VoxelSize, } from "types/api_types"; import type { AdditionalCoordinate } from "types/api_types"; -import { +import Constants, { AltOrOptionKey, CtrlOrCmdKey, LongUnitToShortUnitMap, @@ -963,6 +963,7 @@ function getNoNodeContextMenuOptions(props: NoNodeContextMenuProps): ItemType[] globalPosition, additionalCoordinates, currentMeshFile.name, + Constants.DEFAULT_MESH_OPACITY, undefined, ), ); diff --git a/frontend/javascripts/viewer/view/right-border-tabs/segments_tab/segments_view.tsx b/frontend/javascripts/viewer/view/right-border-tabs/segments_tab/segments_view.tsx index 5d2adf82bb..322ed51f63 100644 --- a/frontend/javascripts/viewer/view/right-border-tabs/segments_tab/segments_view.tsx +++ b/frontend/javascripts/viewer/view/right-border-tabs/segments_tab/segments_view.tsx @@ -54,7 +54,7 @@ import type { } from "types/api_types"; import { APIJobType, type AdditionalCoordinate } from "types/api_types"; import type { Vector3 } from "viewer/constants"; -import { EMPTY_OBJECT, MappingStatusEnum } from "viewer/constants"; +import Constants, { EMPTY_OBJECT, MappingStatusEnum } from "viewer/constants"; import { getMagInfoOfVisibleSegmentationLayer, getMappingInfo, @@ -261,7 +261,13 @@ const mapDispatchToProps = (dispatch: Dispatch) => ({ meshFileName: string, ) { dispatch( - loadPrecomputedMeshAction(segmentId, seedPosition, seedAdditionalCoordinates, meshFileName), + loadPrecomputedMeshAction( + segmentId, + seedPosition, + seedAdditionalCoordinates, + meshFileName, + Constants.DEFAULT_MESH_OPACITY, + ), ); }, From 90f32d308a68c4f1974921cf6e65975430cc041e Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Tue, 20 May 2025 22:27:55 +0200 Subject: [PATCH 06/12] remove console.log --- frontend/javascripts/viewer/model/sagas/mesh_saga.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts index 8b1d3da7a3..9551cbc6af 100644 --- a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts +++ b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts @@ -770,7 +770,6 @@ function* loadPrecomputedMesh(action: LoadPrecomputedMeshAction) { // Remove older mesh instance if it exists already. yield* put(removeMeshAction(layer.name, action.segmentId)); - console.log("Loading precomputed mesh for segment", segmentId, opacity); // If a REMOVE_MESH action is dispatched and consumed // here before loadPrecomputedMeshForSegmentId is finished, the latter saga From 857debdfeac8d7b99f9b2871804bbef48ad450a2 Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Tue, 27 May 2025 21:55:02 +0200 Subject: [PATCH 07/12] fix color pickers --- frontend/javascripts/components/color_picker.tsx | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/frontend/javascripts/components/color_picker.tsx b/frontend/javascripts/components/color_picker.tsx index dc90a82de3..41a1cb8194 100644 --- a/frontend/javascripts/components/color_picker.tsx +++ b/frontend/javascripts/components/color_picker.tsx @@ -1,7 +1,7 @@ import { InputNumber, Popover } from "antd"; import useThrottledCallback from "beautiful-react-hooks/useThrottledCallback"; import * as Utils from "libs/utils"; -import { useRef, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import type { CSSProperties } from "react"; import { HexColorInput, HexColorPicker, type RgbaColor, RgbaColorPicker } from "react-colorful"; import type { Vector3, Vector4 } from "viewer/constants"; @@ -36,6 +36,12 @@ const ThrottledColorPicker = ({ }) => { const [value, localSetValue] = useState(color); const throttledSetValue = useThrottledCallback(onChange, [onChange], 20); + + // Sync local state when external color prop changes + useEffect(() => { + localSetValue(color); + }, [color]); + const setValue = (newValue: string) => { localSetValue(newValue); throttledSetValue(newValue); @@ -43,7 +49,7 @@ const ThrottledColorPicker = ({ return (
- +
); }; @@ -90,6 +96,12 @@ const ThrottledRGBAColorPicker = ({ }) => { const [value, localSetValue] = useState(color); const throttledSetValue = useThrottledCallback(onChangeColor, [onChangeColor, value], 20); + + // Sync local state when external color prop changes + useEffect(() => { + localSetValue(color); + }, [color]); + const setValue = (newValue: RgbaColor) => { localSetValue(newValue); throttledSetValue(newValue); From 99d8c72e21bd41a9fdc915a7c6e3a655e5174407 Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Wed, 28 May 2025 11:36:41 +0200 Subject: [PATCH 08/12] split mesh saga module --- .../viewer/controller/mesh_helpers.ts | 12 + .../controller/segment_mesh_controller.ts | 2 +- .../model/reducers/annotation_reducer.ts | 2 +- .../viewer/model/sagas/mesh_saga.ts | 1369 ----------------- .../model/sagas/meshes/ad_hoc_mesh_saga.ts | 676 ++++++++ .../model/sagas/meshes/common_mesh_saga.ts | 219 +++ .../sagas/meshes/precomputed_mesh_saga.ts | 501 ++++++ .../viewer/model/sagas/root_saga.ts | 8 +- 8 files changed, 1416 insertions(+), 1373 deletions(-) delete mode 100644 frontend/javascripts/viewer/model/sagas/mesh_saga.ts create mode 100644 frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts create mode 100644 frontend/javascripts/viewer/model/sagas/meshes/common_mesh_saga.ts create mode 100644 frontend/javascripts/viewer/model/sagas/meshes/precomputed_mesh_saga.ts diff --git a/frontend/javascripts/viewer/controller/mesh_helpers.ts b/frontend/javascripts/viewer/controller/mesh_helpers.ts index 5392f61269..733e15dba4 100644 --- a/frontend/javascripts/viewer/controller/mesh_helpers.ts +++ b/frontend/javascripts/viewer/controller/mesh_helpers.ts @@ -1,5 +1,8 @@ +import type { meshApi } from "admin/rest_api"; +import { V3 } from "libs/mjs"; import _ from "lodash"; import type * as THREE from "three"; +import type { Vector3 } from "viewer/constants"; export type BufferGeometryWithInfo = THREE.BufferGeometry & { vertexSegmentMapping?: VertexSegmentMapping; @@ -78,3 +81,12 @@ export class VertexSegmentMapping { return _.sortedIndexOf(this.unmappedSegmentIds, segmentId) !== -1; } } + +export function sortByDistanceTo( + availableChunks: Vector3[] | meshApi.MeshChunk[] | null | undefined, + seedPosition: Vector3, +) { + return _.sortBy(availableChunks, (chunk: Vector3 | meshApi.MeshChunk) => + V3.length(V3.sub(seedPosition, "position" in chunk ? chunk.position : chunk)), + ) as Array | Array; +} diff --git a/frontend/javascripts/viewer/controller/segment_mesh_controller.ts b/frontend/javascripts/viewer/controller/segment_mesh_controller.ts index 5c5de11c38..ff6960afc0 100644 --- a/frontend/javascripts/viewer/controller/segment_mesh_controller.ts +++ b/frontend/javascripts/viewer/controller/segment_mesh_controller.ts @@ -13,11 +13,11 @@ import { getActiveSegmentationTracing, getSegmentColorAsHSLA, } from "viewer/model/accessors/volumetracing_accessor"; -import { NO_LOD_MESH_INDEX } from "viewer/model/sagas/mesh_saga"; import Store from "viewer/store"; import { computeBvhAsync } from "libs/compute_bvh_async"; import type { BufferAttribute } from "three"; +import { NO_LOD_MESH_INDEX } from "viewer/model/sagas/meshes/common_mesh_saga"; import type { BufferGeometryWithInfo } from "./mesh_helpers"; // Add the raycast function. Assumes the BVH is available on diff --git a/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts b/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts index 208ac421d9..b8f12ef8bf 100644 --- a/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts +++ b/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts @@ -344,7 +344,7 @@ function AnnotationReducer(state: WebknossosState, action: Action): WebknossosSt }); } - // Mesh information is stored in three places: the state in the store, segment_view_controller and within the mesh_saga. + // Mesh information is stored in three places: the state in the store, segment_view_controller and within the mesh sagas. case "ADD_AD_HOC_MESH": { const { layerName, diff --git a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/mesh_saga.ts deleted file mode 100644 index 9551cbc6af..0000000000 --- a/frontend/javascripts/viewer/model/sagas/mesh_saga.ts +++ /dev/null @@ -1,1369 +0,0 @@ -import type { MeshLodInfo } from "admin/api/mesh"; -import { - computeAdHocMesh, - getBucketPositionsForAdHocMesh, - getMeshfilesForDatasetLayer, - meshApi, - sendAnalyticsEvent, -} from "admin/rest_api"; -import { saveAs } from "file-saver"; -import { mergeGeometries } from "libs/BufferGeometryUtils"; -import ThreeDMap from "libs/ThreeDMap"; -import Deferred from "libs/async/deferred"; -import processTaskWithPool from "libs/async/task_pool"; -import { computeBvhAsync } from "libs/compute_bvh_async"; -import { getDracoLoader } from "libs/draco"; -import ErrorHandling from "libs/error_handling"; -import { V3 } from "libs/mjs"; -import exportToStl from "libs/stl_exporter"; -import Toast from "libs/toast"; -import { chunkDynamically, sleep } from "libs/utils"; -import Zip from "libs/zipjs_wrapper"; -import _ from "lodash"; -import messages from "messages"; -import type { ActionPattern } from "redux-saga/effects"; -import type * as THREE from "three"; -import { actionChannel, all, call, put, race, take, takeEvery } from "typed-redux-saga"; -import type { APIDataset, APIMeshFileInfo, APISegmentationLayer } from "types/api_types"; -import type { AdditionalCoordinate } from "types/api_types"; -import { WkDevFlags } from "viewer/api/wk_dev"; -import type { Vector3, Vector4 } from "viewer/constants"; -import Constants, { MappingStatusEnum } from "viewer/constants"; -import CustomLOD from "viewer/controller/custom_lod"; -import { - type BufferGeometryWithInfo, - type UnmergedBufferGeometryWithInfo, - VertexSegmentMapping, -} from "viewer/controller/mesh_helpers"; -import getSceneController from "viewer/controller/scene_controller_provider"; -import { - getMagInfo, - getMappingInfo, - getSegmentationLayerByName, - getVisibleSegmentationLayer, -} from "viewer/model/accessors/dataset_accessor"; -import { - getActiveSegmentationTracing, - getEditableMappingForVolumeTracingId, - getMeshInfoForSegment, - getTracingForSegmentationLayer, -} from "viewer/model/accessors/volumetracing_accessor"; -import type { Action } from "viewer/model/actions/actions"; -import { - type MaybeFetchMeshFilesAction, - type RefreshMeshAction, - type RemoveMeshAction, - type TriggerMeshDownloadAction, - type TriggerMeshesDownloadAction, - type UpdateMeshOpacityAction, - type UpdateMeshVisibilityAction, - addAdHocMeshAction, - addPrecomputedMeshAction, - dispatchMaybeFetchMeshFilesAsync, - finishedLoadingMeshAction, - removeMeshAction, - startedLoadingMeshAction, - updateCurrentMeshFileAction, - updateMeshFileListAction, - updateMeshVisibilityAction, -} from "viewer/model/actions/annotation_actions"; -import { saveNowAction } from "viewer/model/actions/save_actions"; -import { - type AdHocMeshInfo, - type LoadAdHocMeshAction, - type LoadPrecomputedMeshAction, - loadPrecomputedMeshAction, -} from "viewer/model/actions/segmentation_actions"; -import type DataLayer from "viewer/model/data_layer"; -import { zoomedAddressToAnotherZoomStepWithInfo } from "viewer/model/helpers/position_converter"; -import type { Saga } from "viewer/model/sagas/effect-generators"; -import { select } from "viewer/model/sagas/effect-generators"; -import { Model } from "viewer/singletons"; -import Store from "viewer/store"; -import { stlMeshConstants } from "viewer/view/right-border-tabs/segments_tab/segments_view"; -import { getBaseSegmentationName } from "viewer/view/right-border-tabs/segments_tab/segments_view_helper"; -import { getAdditionalCoordinatesAsString } from "../accessors/flycam_accessor"; -import type { FlycamAction } from "../actions/flycam_actions"; -import type { - BatchUpdateGroupsAndSegmentsAction, - RemoveSegmentAction, - UpdateSegmentAction, -} from "../actions/volumetracing_actions"; -import type { MagInfo } from "../helpers/mag_info"; -import { ensureSceneControllerReady, ensureWkReady } from "./ready_sagas"; - -export const NO_LOD_MESH_INDEX = -1; -const MAX_RETRY_COUNT = 5; -const RETRY_WAIT_TIME = 5000; -const MESH_CHUNK_THROTTLE_DELAY = 500; -const PARALLEL_PRECOMPUTED_MESH_LOADING_COUNT = 32; -const MIN_BATCH_SIZE_IN_BYTES = 2 ** 16; - -// The calculation of a mesh is spread across multiple requests. -// In order to avoid, that a huge amount of chunks is downloaded at full speed, -// we artificially throttle the download speed after the first MESH_CHUNK_THROTTLE_LIMIT -// requests for each segment. -const batchCounterPerSegment: Record = {}; -const MESH_CHUNK_THROTTLE_LIMIT = 50; - -/* - * - * Ad-Hoc Meshes - * - */ -// Maps from additional coordinates, layerName and segmentId to a ThreeDMap that stores for each chunk -// (at x, y, z) position whether the mesh chunk was loaded. -const adhocMeshesMapByLayer: Record>>> = {}; - -function marchingCubeSizeInTargetMag(): Vector3 { - return WkDevFlags.meshing.marchingCubeSizeInTargetMag; -} -const modifiedCells: Set = new Set(); -export function isMeshSTL(buffer: ArrayBuffer): boolean { - const dataView = new DataView(buffer); - const isMesh = stlMeshConstants.meshMarker.every( - (marker, index) => dataView.getUint8(index) === marker, - ); - return isMesh; -} - -function getOrAddMapForSegment( - layerName: string, - segmentId: number, - additionalCoordinates?: AdditionalCoordinate[] | null, -): ThreeDMap { - const additionalCoordKey = getAdditionalCoordinatesAsString(additionalCoordinates); - - const keys = [additionalCoordKey, layerName]; - // create new map if adhocMeshesMapByLayer[additionalCoordinatesString][layerName] doesn't exist yet. - _.set(adhocMeshesMapByLayer, keys, _.get(adhocMeshesMapByLayer, keys, new Map())); - const meshesMap = adhocMeshesMapByLayer[additionalCoordKey][layerName]; - const maybeMap = meshesMap.get(segmentId); - - if (maybeMap == null) { - const newMap = new ThreeDMap(); - meshesMap.set(segmentId, newMap); - return newMap; - } - - return maybeMap; -} - -function removeMapForSegment( - layerName: string, - segmentId: number, - additionalCoordinateKey: string, -): void { - if ( - adhocMeshesMapByLayer[additionalCoordinateKey] == null || - adhocMeshesMapByLayer[additionalCoordinateKey][layerName] == null - ) { - return; - } - - adhocMeshesMapByLayer[additionalCoordinateKey][layerName].delete(segmentId); -} - -function getCubeSizeInMag1(zoomStep: number, magInfo: MagInfo): Vector3 { - // Convert marchingCubeSizeInTargetMag to mag1 via zoomStep - // Drop the last element of the Vector4; - const [x, y, z] = zoomedAddressToAnotherZoomStepWithInfo( - [...marchingCubeSizeInTargetMag(), zoomStep], - magInfo, - 0, - ); - return [x, y, z]; -} - -function clipPositionToCubeBoundary( - position: Vector3, - zoomStep: number, - magInfo: MagInfo, -): Vector3 { - const cubeSizeInMag1 = getCubeSizeInMag1(zoomStep, magInfo); - const currentCube = V3.floor(V3.divide3(position, cubeSizeInMag1)); - const clippedPosition = V3.scale3(currentCube, cubeSizeInMag1); - return clippedPosition; -} - -// front_xy, front_xz, front_yz, back_xy, back_xz, back_yz -const NEIGHBOR_LOOKUP = [ - [0, 0, -1], - [0, -1, 0], - [-1, 0, 0], - [0, 0, 1], - [0, 1, 0], - [1, 0, 0], -]; - -function getNeighborPosition( - clippedPosition: Vector3, - neighborId: number, - zoomStep: number, - magInfo: MagInfo, -): Vector3 { - const neighborMultiplier = NEIGHBOR_LOOKUP[neighborId]; - const cubeSizeInMag1 = getCubeSizeInMag1(zoomStep, magInfo); - const neighboringPosition: Vector3 = [ - clippedPosition[0] + neighborMultiplier[0] * cubeSizeInMag1[0], - clippedPosition[1] + neighborMultiplier[1] * cubeSizeInMag1[1], - clippedPosition[2] + neighborMultiplier[2] * cubeSizeInMag1[2], - ]; - return neighboringPosition; -} - -function* loadAdHocMeshFromAction(action: LoadAdHocMeshAction): Saga { - const { layerName } = action; - const layer = - layerName != null ? Model.getLayerByName(layerName) : Model.getVisibleSegmentationLayer(); - if (layer == null) { - return; - } - // Remove older mesh instance if it exists already. - yield* put(removeMeshAction(layer.name, action.segmentId)); - - yield* call( - loadAdHocMesh, - action.seedPosition, - action.seedAdditionalCoordinates, - action.segmentId, - false, - layer.name, - action.extraInfo, - ); -} - -function* getMeshExtraInfo( - layerName: string, - maybeExtraInfo: AdHocMeshInfo | null | undefined, -): Saga { - const activeMappingByLayer = yield* select( - (state) => state.temporaryConfiguration.activeMappingByLayer, - ); - if (maybeExtraInfo != null) return maybeExtraInfo; - const mappingInfo = getMappingInfo(activeMappingByLayer, layerName); - const isMappingActive = mappingInfo.mappingStatus === MappingStatusEnum.ENABLED; - const mappingName = isMappingActive ? mappingInfo.mappingName : null; - const mappingType = isMappingActive ? mappingInfo.mappingType : null; - return { - mappingName, - mappingType, - }; -} - -function* getInfoForMeshLoading( - layer: DataLayer, - meshExtraInfo: AdHocMeshInfo, -): Saga<{ - zoomStep: number; - magInfo: MagInfo; -}> { - const magInfo = getMagInfo(layer.mags); - const preferredZoomStep = - meshExtraInfo.preferredQuality != null - ? meshExtraInfo.preferredQuality - : yield* select( - (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation, - ); - const zoomStep = magInfo.getClosestExistingIndex(preferredZoomStep); - return { - zoomStep, - magInfo: magInfo, - }; -} - -function* loadAdHocMesh( - seedPosition: Vector3, - seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, - segmentId: number, - removeExistingMesh: boolean = false, - layerName: string, - maybeExtraInfo?: AdHocMeshInfo, -): Saga { - const layer = Model.getLayerByName(layerName); - - if (segmentId === 0) { - return; - } - - yield* call([Model, Model.ensureSavedState]); - - const meshExtraInfo = yield* call(getMeshExtraInfo, layer.name, maybeExtraInfo); - - const { zoomStep, magInfo } = yield* call(getInfoForMeshLoading, layer, meshExtraInfo); - batchCounterPerSegment[segmentId] = 0; - - // If a REMOVE_MESH action is dispatched and consumed - // here before loadFullAdHocMesh is finished, the latter saga - // should be canceled automatically to avoid populating mesh data even though - // the mesh was removed. This is accomplished by redux-saga's race effect. - yield* race({ - loadFullAdHocMesh: call( - loadFullAdHocMesh, - layer, - segmentId, - seedPosition, - seedAdditionalCoordinates, - zoomStep, - meshExtraInfo, - magInfo, - removeExistingMesh, - ), - cancel: take( - ((action: Action) => - action.type === "REMOVE_MESH" && - action.segmentId === segmentId && - action.layerName === layer.name) as ActionPattern, - ), - }); - removeMeshWithoutVoxels(segmentId, layer.name, seedAdditionalCoordinates); -} - -function removeMeshWithoutVoxels( - segmentId: number, - layerName: string, - additionalCoordinates: AdditionalCoordinate[] | undefined | null, -) { - // If no voxels were added to the scene (e.g. because the segment doesn't have any voxels in this n-dimension), - // remove it from the store's state as well. - const { segmentMeshController } = getSceneController(); - if (!segmentMeshController.hasMesh(segmentId, layerName, additionalCoordinates)) { - Store.dispatch(removeMeshAction(layerName, segmentId)); - } -} - -function* loadFullAdHocMesh( - layer: DataLayer, - segmentId: number, - position: Vector3, - additionalCoordinates: AdditionalCoordinate[] | undefined | null, - zoomStep: number, - meshExtraInfo: AdHocMeshInfo, - magInfo: MagInfo, - removeExistingMesh: boolean, -): Saga { - let isInitialRequest = true; - const { mappingName, mappingType, opacity } = meshExtraInfo; - const clippedPosition = clipPositionToCubeBoundary(position, zoomStep, magInfo); - yield* put( - addAdHocMeshAction( - layer.name, - segmentId, - position, - additionalCoordinates, - mappingName, - mappingType, - opacity || Constants.DEFAULT_MESH_OPACITY, - ), - ); - yield* put(startedLoadingMeshAction(layer.name, segmentId)); - - const cubeSize = marchingCubeSizeInTargetMag(); - const tracingStoreHost = yield* select((state) => state.annotation.tracingStore.url); - const mag = magInfo.getMagByIndexOrThrow(zoomStep); - - const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state)); - const visibleSegmentationLayer = yield* select((state) => getVisibleSegmentationLayer(state)); - // Fetch from datastore if no volumetracing ... - let useDataStore = volumeTracing == null || visibleSegmentationLayer?.tracingId == null; - if (meshExtraInfo.useDataStore != null) { - // ... except if the caller specified whether to use the data store ... - useDataStore = meshExtraInfo.useDataStore; - } else if (volumeTracing?.hasEditableMapping) { - // ... or if an editable mapping is active. - useDataStore = false; - } - - // Segment stats can only be used for volume tracings that have a segment index - // and that don't have editable mappings. - const usePositionsFromSegmentIndex = - volumeTracing?.hasSegmentIndex && - !volumeTracing.hasEditableMapping && - visibleSegmentationLayer?.tracingId != null; - let positionsToRequest = usePositionsFromSegmentIndex - ? yield* getChunkPositionsFromSegmentIndex( - tracingStoreHost, - layer, - segmentId, - cubeSize, - mag, - clippedPosition, - additionalCoordinates, - ) - : [clippedPosition]; - - if (positionsToRequest.length === 0) { - //if no positions are requested, remove the mesh, - //so that the old one isn't displayed anymore - yield* put(removeMeshAction(layer.name, segmentId)); - } - while (positionsToRequest.length > 0) { - const currentPosition = positionsToRequest.shift(); - if (currentPosition == null) { - throw new Error("Satisfy typescript"); - } - const neighbors = yield* call( - maybeLoadMeshChunk, - layer, - segmentId, - currentPosition, - zoomStep, - meshExtraInfo, - magInfo, - isInitialRequest, - removeExistingMesh && isInitialRequest, - useDataStore, - !usePositionsFromSegmentIndex, - ); - isInitialRequest = false; - - // If we are using the positions from the segment index, the backend will - // send an empty neighbors array, as it's not necessary to have them. - if (usePositionsFromSegmentIndex && neighbors.length > 0) { - throw new Error("Retrieved neighbor positions even though these were not requested."); - } - positionsToRequest = positionsToRequest.concat(neighbors); - } - - yield* put(finishedLoadingMeshAction(layer.name, segmentId)); -} - -function* getChunkPositionsFromSegmentIndex( - tracingStoreHost: string, - layer: DataLayer, - segmentId: number, - cubeSize: Vector3, - mag: Vector3, - clippedPosition: Vector3, - additionalCoordinates: AdditionalCoordinate[] | null | undefined, -) { - const targetMagPositions = yield* call( - getBucketPositionsForAdHocMesh, - tracingStoreHost, - layer.name, - segmentId, - cubeSize, - mag, - additionalCoordinates, - ); - const mag1Positions = targetMagPositions.map((pos) => V3.scale3(pos, mag)); - return sortByDistanceTo(mag1Positions, clippedPosition) as Vector3[]; -} - -function hasMeshChunkExceededThrottleLimit(segmentId: number): boolean { - return batchCounterPerSegment[segmentId] > MESH_CHUNK_THROTTLE_LIMIT; -} - -function* maybeLoadMeshChunk( - layer: DataLayer, - segmentId: number, - clippedPosition: Vector3, - zoomStep: number, - meshExtraInfo: AdHocMeshInfo, - magInfo: MagInfo, - isInitialRequest: boolean, - removeExistingMesh: boolean, - useDataStore: boolean, - findNeighbors: boolean, -): Saga { - const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); - const threeDMap = getOrAddMapForSegment(layer.name, segmentId, additionalCoordinates); - - if (threeDMap.get(clippedPosition)) { - return []; - } - - if (hasMeshChunkExceededThrottleLimit(segmentId)) { - yield* call(sleep, MESH_CHUNK_THROTTLE_DELAY); - } - - batchCounterPerSegment[segmentId]++; - threeDMap.set(clippedPosition, true); - const scaleFactor = yield* select((state) => state.dataset.dataSource.scale.factor); - const dataStoreHost = yield* select((state) => state.dataset.dataStore.url); - const owningOrganization = yield* select((state) => state.dataset.owningOrganization); - const datasetDirectoryName = yield* select((state) => state.dataset.directoryName); - const tracingStoreHost = yield* select((state) => state.annotation.tracingStore.url); - const dataStoreUrl = `${dataStoreHost}/data/datasets/${owningOrganization}/${datasetDirectoryName}/layers/${ - layer.fallbackLayer != null ? layer.fallbackLayer : layer.name - }`; - const tracingStoreUrl = `${tracingStoreHost}/tracings/volume/${layer.name}`; - - const mag = magInfo.getMagByIndexOrThrow(zoomStep); - - if (isInitialRequest) { - sendAnalyticsEvent("request_isosurface", { - mode: useDataStore ? "view" : "annotation", - }); - } - - let retryCount = 0; - - const { segmentMeshController } = getSceneController(); - - const cubeSize = marchingCubeSizeInTargetMag(); - - while (retryCount < MAX_RETRY_COUNT) { - try { - const { buffer: responseBuffer, neighbors } = yield* call( - { - context: null, - fn: computeAdHocMesh, - }, - useDataStore ? dataStoreUrl : tracingStoreUrl, - { - position: clippedPosition, - additionalCoordinates: additionalCoordinates || undefined, - mag, - segmentId, - cubeSize, - scaleFactor, - findNeighbors, - ...meshExtraInfo, - }, - ); - const vertices = new Float32Array(responseBuffer); - - if (removeExistingMesh) { - segmentMeshController.removeMeshById(segmentId, layer.name); - } - - const opacity = meshExtraInfo.opacity || Constants.DEFAULT_MESH_OPACITY; - - // We await addMeshFromVerticesAsync here, because the mesh saga will remove - // an ad-hoc loaded mesh immediately if it was "empty". Since the check is - // done by looking at the scene, we await the population of the scene. - // Theoretically, this could be built differently so that other ad-hoc chunks - // can be loaded in parallel to addMeshFromVerticesAsync. However, it's unclear - // how big the bottleneck really is. - yield* call( - { fn: segmentMeshController.addMeshFromVerticesAsync, context: segmentMeshController }, - vertices, - segmentId, - layer.name, - opacity, - additionalCoordinates, - ); - return neighbors.map((neighbor) => - getNeighborPosition(clippedPosition, neighbor, zoomStep, magInfo), - ); - } catch (exception) { - retryCount++; - ErrorHandling.notify(exception as Error); - console.warn("Retrying mesh generation due to", exception); - yield* call(sleep, RETRY_WAIT_TIME * 2 ** retryCount); - } - } - - return []; -} - -function* markEditedCellAsDirty(): Saga { - const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state)); - - if (volumeTracing != null && volumeTracing.fallbackLayer == null) { - const activeCellId = volumeTracing.activeCellId; - modifiedCells.add(activeCellId); - } -} - -function* refreshMeshes(): Saga { - yield* put(saveNowAction()); - // We reload all cells that got modified till the start of reloading. - // By that we avoid to remove cells that got annotated during reloading from the modifiedCells set. - const currentlyModifiedCells = new Set(modifiedCells); - modifiedCells.clear(); - - const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); - const additionalCoordKey = getAdditionalCoordinatesAsString(additionalCoordinates); - const segmentationLayer = Model.getVisibleSegmentationLayer(); - - if (!segmentationLayer) { - return; - } - - adhocMeshesMapByLayer[additionalCoordKey][segmentationLayer.name] = - adhocMeshesMapByLayer[additionalCoordKey][segmentationLayer.name] || new Map(); - const meshesMapForLayer = adhocMeshesMapByLayer[additionalCoordKey][segmentationLayer.name]; - - for (const [segmentId, threeDMap] of Array.from(meshesMapForLayer.entries())) { - if (!currentlyModifiedCells.has(segmentId)) { - continue; - } - - yield* call( - _refreshMeshWithMap, - segmentId, - threeDMap, - segmentationLayer.name, - additionalCoordinates, - ); - } -} - -function* refreshMesh(action: RefreshMeshAction): Saga { - const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); - const additionalCoordKey = getAdditionalCoordinatesAsString(additionalCoordinates); - - const { segmentId, layerName } = action; - - const meshInfo = yield* select((state) => - getMeshInfoForSegment(state, additionalCoordinates, layerName, segmentId), - ); - - if (meshInfo == null) { - throw new Error( - `Mesh refreshing failed due to lack of mesh info for segment ${segmentId} in store.`, - ); - } - - if (meshInfo.isPrecomputed) { - yield* put(removeMeshAction(layerName, meshInfo.segmentId)); - yield* put( - loadPrecomputedMeshAction( - meshInfo.segmentId, - meshInfo.seedPosition, - meshInfo.seedAdditionalCoordinates, - meshInfo.meshFileName, - meshInfo.opacity, - layerName, - ), - ); - } else { - if (adhocMeshesMapByLayer[additionalCoordKey] == null) return; - const threeDMap = adhocMeshesMapByLayer[additionalCoordKey][action.layerName].get(segmentId); - if (threeDMap == null) { - return; - } - yield* call( - _refreshMeshWithMap, - segmentId, - threeDMap, - layerName, - additionalCoordinates, - meshInfo.opacity, - ); - } -} - -function* _refreshMeshWithMap( - segmentId: number, - threeDMap: ThreeDMap, - layerName: string, - additionalCoordinates: AdditionalCoordinate[] | null, - opacity: number = Constants.DEFAULT_MESH_OPACITY, -): Saga { - const meshInfo = yield* select((state) => - getMeshInfoForSegment(state, additionalCoordinates, layerName, segmentId), - ); - if (meshInfo == null) { - throw new Error( - `Mesh refreshing failed due to lack of mesh info for segment ${segmentId} in store.`, - ); - } - yield* call( - [ErrorHandling, ErrorHandling.assert], - !meshInfo.isPrecomputed, - "_refreshMeshWithMap was called for a precomputed mesh.", - ); - if (meshInfo.isPrecomputed) return; - const { mappingName, mappingType } = meshInfo; - const meshPositions = threeDMap.entries().filter(([value, _position]) => value); - - if (meshPositions.length === 0) { - return; - } - - // Remove mesh from cache. - yield* call(removeMesh, removeMeshAction(layerName, segmentId), false); - // The mesh should only be removed once after re-fetching the mesh first position. - let shouldBeRemoved = true; - - for (const [, position] of meshPositions) { - // Reload the mesh at the given position if it isn't already loaded there. - // This is done to ensure that every voxel of the mesh is reloaded. - yield* call( - loadAdHocMesh, - position, - additionalCoordinates, - segmentId, - shouldBeRemoved, - layerName, - { - mappingName, - mappingType, - opacity, - }, - ); - shouldBeRemoved = false; - } -} - -/* - * - * Precomputed Meshes - * - */ - -// Avoid redundant fetches of mesh files for the same layer by -// storing Deferreds per layer lazily. -let fetchDeferredsPerLayer: Record, unknown>> = {}; -function* maybeFetchMeshFiles(action: MaybeFetchMeshFilesAction): Saga { - const { segmentationLayer, dataset, mustRequest, autoActivate, callback } = action; - - if (!segmentationLayer) { - callback([]); - return; - } - - const layerName = segmentationLayer.name; - - function* maybeActivateMeshFile(availableMeshFiles: APIMeshFileInfo[]) { - const currentMeshFile = yield* select( - (state) => state.localSegmentationData[layerName].currentMeshFile, - ); - if (!currentMeshFile && availableMeshFiles.length > 0 && autoActivate) { - yield* put(updateCurrentMeshFileAction(layerName, availableMeshFiles[0].name)); - } - } - - // If a deferred already exists (and mustRequest is not true), the deferred - // can be awaited (regardless of whether it's finished or not) and its - // content used to call the callback. - if (fetchDeferredsPerLayer[layerName] && !mustRequest) { - const availableMeshFiles = yield* call(() => fetchDeferredsPerLayer[layerName].promise()); - yield* maybeActivateMeshFile(availableMeshFiles); - callback(availableMeshFiles); - return; - } - // A request has to be made (either because none was made before or because - // it is enforced by mustRequest). - // If mustRequest is true and an old deferred exists, a new deferred will be created which - // replaces the old one (old references to the first Deferred will still - // work and will be resolved by the corresponding saga execution). - const deferred = new Deferred, unknown>(); - fetchDeferredsPerLayer[layerName] = deferred; - - const availableMeshFiles = yield* call( - getMeshfilesForDatasetLayer, - dataset.dataStore.url, - dataset, - getBaseSegmentationName(segmentationLayer), - ); - yield* put(updateMeshFileListAction(layerName, availableMeshFiles)); - deferred.resolve(availableMeshFiles); - - yield* maybeActivateMeshFile(availableMeshFiles); - - callback(availableMeshFiles); -} - -function* loadPrecomputedMesh(action: LoadPrecomputedMeshAction) { - const { segmentId, seedPosition, seedAdditionalCoordinates, meshFileName, layerName, opacity } = - action; - const layer = yield* select((state) => - layerName != null - ? getSegmentationLayerByName(state.dataset, layerName) - : getVisibleSegmentationLayer(state), - ); - if (layer == null) return; - - // Remove older mesh instance if it exists already. - yield* put(removeMeshAction(layer.name, action.segmentId)); - - // If a REMOVE_MESH action is dispatched and consumed - // here before loadPrecomputedMeshForSegmentId is finished, the latter saga - // should be canceled automatically to avoid populating mesh data even though - // the mesh was removed. This is accomplished by redux-saga's race effect. - yield* race({ - loadPrecomputedMeshForSegmentId: call( - loadPrecomputedMeshForSegmentId, - segmentId, - seedPosition, - seedAdditionalCoordinates, - meshFileName, - layer, - opacity || Constants.DEFAULT_MESH_OPACITY, - ), - cancel: take( - ((otherAction: Action) => - otherAction.type === "REMOVE_MESH" && - otherAction.segmentId === segmentId && - otherAction.layerName === layer.name) as ActionPattern, - ), - }); -} - -type ChunksMap = Record; - -function* loadPrecomputedMeshForSegmentId( - segmentId: number, - seedPosition: Vector3, - seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, - meshFileName: string, - segmentationLayer: APISegmentationLayer, - opacity: number, -): Saga { - const layerName = segmentationLayer.name; - const mappingName = yield* call(getMappingName, segmentationLayer); - yield* put( - addPrecomputedMeshAction( - layerName, - segmentId, - seedPosition, - seedAdditionalCoordinates, - meshFileName, - mappingName, - opacity, - ), - ); - yield* put(startedLoadingMeshAction(layerName, segmentId)); - const dataset = yield* select((state) => state.dataset); - const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); - - const availableMeshFiles = yield* call( - dispatchMaybeFetchMeshFilesAsync, - Store.dispatch, - segmentationLayer, - dataset, - false, - false, - ); - - const meshFile = availableMeshFiles.find((file) => file.name === meshFileName); - if (!meshFile) { - Toast.error("Could not load mesh, since the requested mesh file was not found."); - return; - } - if (segmentId === 0) { - Toast.error("Could not load mesh, since the clicked segment ID is 0."); - return; - } - - let availableChunksMap: ChunksMap = {}; - let chunkScale: Vector3 | null = null; - let loadingOrder: number[] | null = null; - let lods: MeshLodInfo[] | null = null; - try { - const chunkDescriptors = yield* call( - _getChunkLoadingDescriptors, - segmentId, - dataset, - segmentationLayer, - meshFile, - ); - lods = chunkDescriptors.segmentInfo.lods; - availableChunksMap = chunkDescriptors.availableChunksMap; - chunkScale = chunkDescriptors.segmentInfo.chunkScale; - loadingOrder = chunkDescriptors.loadingOrder; - } catch (exception) { - Toast.warning(messages["tracing.mesh_listing_failed"](segmentId)); - console.warn( - `Mesh chunks for segment ${segmentId} couldn't be loaded due to`, - exception, - "\nOne possible explanation could be that the segment was not included in the mesh file because it's smaller than the dust threshold that was specified for the mesh computation.", - ); - yield* put(finishedLoadingMeshAction(layerName, segmentId)); - yield* put(removeMeshAction(layerName, segmentId)); - return; - } - - for (const lod of loadingOrder) { - yield* call( - loadPrecomputedMeshesInChunksForLod, - dataset, - layerName, - meshFile, - segmentationLayer, - segmentId, - seedPosition, - availableChunksMap, - lod, - (lod: number) => extractScaleFromMatrix(lods[lod].transform), - chunkScale, - additionalCoordinates, - opacity, - ); - } - - yield* put(finishedLoadingMeshAction(layerName, segmentId)); -} - -function* getMappingName(segmentationLayer: APISegmentationLayer) { - const meshExtraInfo = yield* call(getMeshExtraInfo, segmentationLayer.name, null); - const editableMapping = yield* select((state) => - getEditableMappingForVolumeTracingId(state, segmentationLayer.tracingId), - ); - - // meshExtraInfo.mappingName contains the currently active mapping - // (can be the id of an editable mapping). However, we always need to - // use the mapping name of the on-disk mapping. - return editableMapping != null ? editableMapping.baseMappingName : meshExtraInfo.mappingName; -} - -function* _getChunkLoadingDescriptors( - segmentId: number, - dataset: APIDataset, - segmentationLayer: APISegmentationLayer, - meshFile: APIMeshFileInfo, -) { - const availableChunksMap: ChunksMap = {}; - let loadingOrder: number[] = []; - - const { segmentMeshController } = getSceneController(); - const version = meshFile.formatVersion; - - const editableMapping = yield* select((state) => - getEditableMappingForVolumeTracingId(state, segmentationLayer.tracingId), - ); - const tracing = yield* select((state) => - getTracingForSegmentationLayer(state, segmentationLayer), - ); - const mappingName = yield* call(getMappingName, segmentationLayer); - - if (version < 3) { - console.warn("The active mesh file uses a version lower than 3, which is not supported"); - } - - // mappingName only exists for versions >= 3 - if (meshFile.mappingName != null && meshFile.mappingName !== mappingName) { - throw Error( - `Trying to use a mesh file that was computed for mapping ${meshFile.mappingName} for a requested mapping of ${mappingName}.`, - ); - } - - const segmentInfo = yield* call( - meshApi.getMeshfileChunksForSegment, - dataset.dataStore.url, - dataset, - getBaseSegmentationName(segmentationLayer), - meshFile, - segmentId, - // The back-end should only receive a non-null mapping name, - // if it should perform extra (reverse) look ups to compute a mesh - // with a specific mapping from a mesh file that was computed - // without a mapping. - meshFile.mappingName == null ? mappingName : null, - editableMapping != null && tracing ? tracing.tracingId : null, - ); - segmentInfo.lods.forEach((meshLodInfo, lodIndex) => { - availableChunksMap[lodIndex] = meshLodInfo?.chunks; - loadingOrder.push(lodIndex); - meshLodInfo.transform; - }); - const currentLODGroup: CustomLOD = - (yield* call( - { - context: segmentMeshController, - fn: segmentMeshController.getLODGroupOfLayer, - }, - segmentationLayer.name, - )) ?? new CustomLOD(); - const currentLODIndex = yield* call( - { - context: currentLODGroup, - fn: currentLODGroup.getCurrentLOD, - }, - Math.max(...loadingOrder), - ); - // Load the chunks closest to the current LOD first. - loadingOrder.sort((a, b) => Math.abs(a - currentLODIndex) - Math.abs(b - currentLODIndex)); - - return { - availableChunksMap, - loadingOrder, - segmentInfo, - }; -} -function extractScaleFromMatrix(transform: [Vector4, Vector4, Vector4]): Vector3 { - return [transform[0][0], transform[1][1], transform[2][2]]; -} - -function* loadPrecomputedMeshesInChunksForLod( - dataset: APIDataset, - layerName: string, - meshFile: APIMeshFileInfo, - segmentationLayer: APISegmentationLayer, - segmentId: number, - seedPosition: Vector3, - availableChunksMap: ChunksMap, - lod: number, - getGlobalScale: (lod: number) => Vector3 | null, - chunkScale: Vector3 | null, - additionalCoordinates: AdditionalCoordinate[] | null, - opacity: number, -) { - const { segmentMeshController } = getSceneController(); - const loader = getDracoLoader(); - if (availableChunksMap[lod] == null) { - return; - } - const availableChunks = availableChunksMap[lod]; - // Sort the chunks by distance to the seedPosition, so that the mesh loads from the inside out - const sortedAvailableChunks = sortByDistanceTo(availableChunks, seedPosition); - - const batches = chunkDynamically( - sortedAvailableChunks as meshApi.MeshChunk[], - MIN_BATCH_SIZE_IN_BYTES, - (chunk) => chunk.byteSize, - ); - - let bufferGeometries: UnmergedBufferGeometryWithInfo[] = []; - const tasks = batches.map( - (chunks) => - function* loadChunks(): Saga { - const dataForChunks = yield* call( - meshApi.getMeshfileChunkData, - dataset.dataStore.url, - dataset, - getBaseSegmentationName(segmentationLayer), - { - meshFile, - // Only extract the relevant properties - requests: chunks.map(({ byteOffset, byteSize }) => ({ - byteOffset, - byteSize, - segmentId, - })), - }, - ); - - const errorsWithDetails = []; - - for (const [chunk, data] of _.zip(chunks, dataForChunks)) { - try { - if (chunk == null || data == null) { - throw new Error("Unexpected null value."); - } - const position = chunk.position; - const bufferGeometry = (yield* call( - loader.decodeDracoFileAsync, - data, - )) as UnmergedBufferGeometryWithInfo; - bufferGeometry.unmappedSegmentId = chunk.unmappedSegmentId; - if (chunkScale != null) { - bufferGeometry.scale(...chunkScale); - } - - bufferGeometry.translate(position[0], position[1], position[2]); - // Compute vertex normals to achieve smooth shading. We do this here - // within the chunk-specific code (instead of after all chunks are merged) - // to distribute the workload a bit over time. - bufferGeometry.computeVertexNormals(); - - // Eagerly add the chunk geometry so that they will be rendered - // as soon as possible. These chunks will be removed later and then - // replaced by a merged geometry so that we have better performance - // for large meshes. - yield* call( - { - context: segmentMeshController, - fn: segmentMeshController.addMeshFromGeometry, - }, - bufferGeometry, - segmentId, - // Apply the scale from the segment info, which includes dataset scale and mag - getGlobalScale(lod), - lod, - layerName, - additionalCoordinates, - opacity, - false, - ); - - bufferGeometries.push(bufferGeometry); - } catch (error) { - errorsWithDetails.push({ error, chunk }); - } - } - - if (errorsWithDetails.length > 0) { - console.warn("Errors occurred while decoding mesh chunks:", errorsWithDetails); - // Use first error as representative - throw errorsWithDetails[0].error; - } - }, - ); - - try { - yield* call(processTaskWithPool, tasks, PARALLEL_PRECOMPUTED_MESH_LOADING_COUNT); - } catch (exception) { - Toast.warning(`Some mesh chunks could not be loaded for segment ${segmentId}.`); - console.error(exception); - } - - // Merge Chunks - const sortedBufferGeometries = _.sortBy( - bufferGeometries, - (geometryWithInfo) => geometryWithInfo.unmappedSegmentId, - ); - - // mergeGeometries will crash if the array is empty. Even if it's not empty, - // the function might return null in case of another error. - const mergedGeometry = ( - sortedBufferGeometries.length > 0 ? mergeGeometries(sortedBufferGeometries, false) : null - ) as BufferGeometryWithInfo | null; - - if (mergedGeometry == null) { - console.error("Merged geometry is null. Look at error above."); - return; - } - mergedGeometry.vertexSegmentMapping = new VertexSegmentMapping(sortedBufferGeometries); - mergedGeometry.boundsTree = yield* call(computeBvhAsync, mergedGeometry); - - // Remove the eagerly added chunks (see above). - yield* call( - { - context: segmentMeshController, - fn: segmentMeshController.removeMeshById, - }, - segmentId, - layerName, - { lod }, - ); - - // Add the final merged geometry. - yield* call( - { - context: segmentMeshController, - fn: segmentMeshController.addMeshFromGeometry, - }, - mergedGeometry, - segmentId, - // Apply the scale from the segment info, which includes dataset scale and mag - getGlobalScale(lod), - lod, - layerName, - additionalCoordinates, - opacity, - true, - ); -} - -function sortByDistanceTo( - availableChunks: Vector3[] | meshApi.MeshChunk[] | null | undefined, - seedPosition: Vector3, -) { - return _.sortBy(availableChunks, (chunk: Vector3 | meshApi.MeshChunk) => - V3.length(V3.sub(seedPosition, "position" in chunk ? chunk.position : chunk)), - ) as Array | Array; -} - -/* - * - * Ad-Hoc and Precomputed Meshes - * - */ -function* downloadMeshCellById(cellName: string, segmentId: number, layerName: string): Saga { - const { segmentMeshController } = getSceneController(); - const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); - const geometry = segmentMeshController.getMeshGeometryInBestLOD( - segmentId, - layerName, - additionalCoordinates, - ); - - if (geometry == null) { - const errorMessage = messages["tracing.not_mesh_available_to_download"]; - Toast.error(errorMessage, { - sticky: false, - }); - return; - } - - try { - const blob = getSTLBlob(geometry, segmentId); - yield* call(saveAs, blob, `${cellName}-${segmentId}.stl`); - } catch (exception) { - ErrorHandling.notify(exception as Error); - Toast.error("Could not export to STL. See console for details"); - console.error(exception); - } -} - -function* downloadMeshCellsAsZIP( - segments: Array<{ segmentName: string; segmentId: number; layerName: string }>, -): Saga { - const { segmentMeshController } = getSceneController(); - const zipWriter = new Zip.ZipWriter(new Zip.BlobWriter("application/zip")); - const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); - try { - const addFileToZipWriterPromises = segments.map((element) => { - const geometry = segmentMeshController.getMeshGeometryInBestLOD( - element.segmentId, - element.layerName, - additionalCoordinates, - ); - - if (geometry == null) { - const errorMessage = messages["tracing.not_mesh_available_to_download"]; - Toast.error(errorMessage, { - sticky: false, - }); - return; - } - const stlDataReader = new Zip.BlobReader(getSTLBlob(geometry, element.segmentId)); - return zipWriter.add(`${element.segmentName}-${element.segmentId}.stl`, stlDataReader); - }); - yield all(addFileToZipWriterPromises); - const result = yield* call([zipWriter, zipWriter.close]); - yield* call(saveAs, result as Blob, "mesh-export.zip"); - } catch (exception) { - ErrorHandling.notify(exception as Error); - Toast.error("Could not export meshes as STL files. See console for details"); - console.error(exception); - } -} - -const getSTLBlob = (geometry: THREE.Group, segmentId: number): Blob => { - const stlDataViews = exportToStl(geometry); - // Encode mesh and cell id property - const { meshMarker, segmentIdIndex } = stlMeshConstants; - meshMarker.forEach((marker, index) => { - stlDataViews[0].setUint8(index, marker); - }); - stlDataViews[0].setUint32(segmentIdIndex, segmentId, true); - return new Blob(stlDataViews); -}; - -function* downloadMeshCell(action: TriggerMeshDownloadAction): Saga { - yield* call(downloadMeshCellById, action.segmentName, action.segmentId, action.layerName); -} - -function* downloadMeshCells(action: TriggerMeshesDownloadAction): Saga { - yield* call(downloadMeshCellsAsZIP, action.segmentsArray); -} - -function* handleRemoveSegment(action: RemoveSegmentAction) { - // The dispatched action will make sure that the mesh entry is removed from the - // store and from the scene. - yield* put(removeMeshAction(action.layerName, action.segmentId)); -} - -function* removeMesh(action: RemoveMeshAction, removeFromScene: boolean = true): Saga { - const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); - const additionalCoordKey = getAdditionalCoordinatesAsString(additionalCoordinates); - const { layerName } = action; - const segmentId = action.segmentId; - - if (removeFromScene) { - getSceneController().segmentMeshController.removeMeshById(segmentId, layerName); - } - removeMapForSegment(layerName, segmentId, additionalCoordKey); -} - -function* handleMeshVisibilityChange(action: UpdateMeshVisibilityAction): Saga { - const { id, visibility, layerName, additionalCoordinates } = action; - const { segmentMeshController } = yield* call(getSceneController); - segmentMeshController.setMeshVisibility(id, visibility, layerName, additionalCoordinates); -} - -export function* handleAdditionalCoordinateUpdate(): Saga { - // We want to prevent iterating through all additional coordinates to adjust the mesh visibility, so we store the - // previous additional coordinates in this method. Thus we have to catch SET_ADDITIONAL_COORDINATES actions in a - // while-true loop and register this saga in the root saga instead of calling from the mesh saga. - yield* call(ensureWkReady); - - let previousAdditionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); - const { segmentMeshController } = yield* call(getSceneController); - - while (true) { - const action = (yield* take(["SET_ADDITIONAL_COORDINATES"]) as any) as FlycamAction; - // Satisfy TS - if (action.type !== "SET_ADDITIONAL_COORDINATES") { - // Don't throw as this would interfere with the never return type - console.error("Unexpected action.type. Ignoring SET_ADDITIONAL_COORDINATES action..."); - continue; - } - const meshRecords = segmentMeshController.meshesGroupsPerSegmentId; - - if (action.values == null || action.values.length === 0) continue; - const newAdditionalCoordKey = getAdditionalCoordinatesAsString(action.values); - - for (const additionalCoordinates of [action.values, previousAdditionalCoordinates]) { - const currentAdditionalCoordKey = getAdditionalCoordinatesAsString(additionalCoordinates); - const shouldBeVisible = currentAdditionalCoordKey === newAdditionalCoordKey; - const recordsOfLayers = meshRecords[currentAdditionalCoordKey] || {}; - for (const [layerName, recordsForOneLayer] of Object.entries(recordsOfLayers)) { - const segmentIds = Object.keys(recordsForOneLayer); - for (const segmentIdAsString of segmentIds) { - const segmentId = Number.parseInt(segmentIdAsString); - yield* put( - updateMeshVisibilityAction( - layerName, - segmentId, - shouldBeVisible, - additionalCoordinates, - ), - ); - yield* call( - { - context: segmentMeshController, - fn: segmentMeshController.setMeshVisibility, - }, - segmentId, - shouldBeVisible, - layerName, - additionalCoordinates, - ); - } - } - } - previousAdditionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); - } -} - -function* handleSegmentColorChange(action: UpdateSegmentAction): Saga { - const { segmentMeshController } = yield* call(getSceneController); - const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); - if ( - "color" in action.segment && - segmentMeshController.hasMesh(action.segmentId, action.layerName, additionalCoordinates) - ) { - segmentMeshController.setMeshColor(action.segmentId, action.layerName); - } -} - -function* handleMeshOpacityChange(action: UpdateMeshOpacityAction): Saga { - const { segmentMeshController } = yield* call(getSceneController); - segmentMeshController.setMeshOpacity(action.id, action.layerName, action.opacity); -} - -function* handleBatchSegmentColorChange( - batchAction: BatchUpdateGroupsAndSegmentsAction, -): Saga { - // Manually unpack batched actions and handle these. - // In theory, this could happen automatically. See this issue in the corresponding (rather unmaintained) package: https://github.com/tshelburne/redux-batched-actions/pull/18 - // However, there seem to be some problems with that approach (e.g., too many updates, infinite recursion) and the discussion there didn't really reach a consensus - // about the correct solution. - // This is why we stick to the manual unpacking for now. - const updateSegmentActions = batchAction.payload - .filter((action) => action.type === "UPDATE_SEGMENT") - .map((action) => call(handleSegmentColorChange, action as UpdateSegmentAction)); - yield* all(updateSegmentActions); -} - -export default function* meshSaga(): Saga { - fetchDeferredsPerLayer = {}; - // Buffer actions since they might be dispatched before WK_READY - const loadAdHocMeshActionChannel = yield* actionChannel("LOAD_AD_HOC_MESH_ACTION"); - const loadPrecomputedMeshActionChannel = yield* actionChannel("LOAD_PRECOMPUTED_MESH_ACTION"); - const maybeFetchMeshFilesActionChannel = yield* actionChannel("MAYBE_FETCH_MESH_FILES"); - - yield* call(ensureSceneControllerReady); - yield* call(ensureWkReady); - yield* takeEvery(maybeFetchMeshFilesActionChannel, maybeFetchMeshFiles); - yield* takeEvery(loadAdHocMeshActionChannel, loadAdHocMeshFromAction); - yield* takeEvery(loadPrecomputedMeshActionChannel, loadPrecomputedMesh); - yield* takeEvery("TRIGGER_MESH_DOWNLOAD", downloadMeshCell); - yield* takeEvery("TRIGGER_MESHES_DOWNLOAD", downloadMeshCells); - yield* takeEvery("REMOVE_MESH", removeMesh); - yield* takeEvery("REMOVE_SEGMENT", handleRemoveSegment); - yield* takeEvery("REFRESH_MESHES", refreshMeshes); - yield* takeEvery("REFRESH_MESH", refreshMesh); - yield* takeEvery("UPDATE_MESH_VISIBILITY", handleMeshVisibilityChange); - yield* takeEvery(["START_EDITING", "COPY_SEGMENTATION_LAYER"], markEditedCellAsDirty); - yield* takeEvery("UPDATE_SEGMENT", handleSegmentColorChange); - yield* takeEvery("UPDATE_MESH_OPACITY", handleMeshOpacityChange); - yield* takeEvery("BATCH_UPDATE_GROUPS_AND_SEGMENTS", handleBatchSegmentColorChange); -} diff --git a/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts new file mode 100644 index 0000000000..c6b7e0c950 --- /dev/null +++ b/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts @@ -0,0 +1,676 @@ +import { + computeAdHocMesh, + getBucketPositionsForAdHocMesh, + sendAnalyticsEvent, +} from "admin/rest_api"; +import ThreeDMap from "libs/ThreeDMap"; +import ErrorHandling from "libs/error_handling"; +import { V3 } from "libs/mjs"; +import { sleep } from "libs/utils"; +import _ from "lodash"; +import type { ActionPattern } from "redux-saga/effects"; +import { actionChannel, call, put, race, take, takeEvery } from "typed-redux-saga"; +import type { AdditionalCoordinate } from "types/api_types"; +import { WkDevFlags } from "viewer/api/wk_dev"; +import type { Vector3 } from "viewer/constants"; +import Constants, { MappingStatusEnum } from "viewer/constants"; +import { sortByDistanceTo } from "viewer/controller/mesh_helpers"; +import getSceneController from "viewer/controller/scene_controller_provider"; +import { + getMagInfo, + getMappingInfo, + getVisibleSegmentationLayer, +} from "viewer/model/accessors/dataset_accessor"; +import { + getActiveSegmentationTracing, + getMeshInfoForSegment, +} from "viewer/model/accessors/volumetracing_accessor"; +import type { Action } from "viewer/model/actions/actions"; +import { + type RefreshMeshAction, + type RemoveMeshAction, + addAdHocMeshAction, + finishedLoadingMeshAction, + removeMeshAction, + startedLoadingMeshAction, +} from "viewer/model/actions/annotation_actions"; +import { saveNowAction } from "viewer/model/actions/save_actions"; +import { + type AdHocMeshInfo, + type LoadAdHocMeshAction, + loadPrecomputedMeshAction, +} from "viewer/model/actions/segmentation_actions"; +import type DataLayer from "viewer/model/data_layer"; +import type { MagInfo } from "viewer/model/helpers/mag_info"; +import { zoomedAddressToAnotherZoomStepWithInfo } from "viewer/model/helpers/position_converter"; +import type { Saga } from "viewer/model/sagas/effect-generators"; +import { select } from "viewer/model/sagas/effect-generators"; +import { Model } from "viewer/singletons"; +import Store from "viewer/store"; +import { stlMeshConstants } from "viewer/view/right-border-tabs/segments_tab/segments_view"; +import { getAdditionalCoordinatesAsString } from "../../accessors/flycam_accessor"; +import { ensureSceneControllerReady, ensureWkReady } from "../ready_sagas"; + +const MAX_RETRY_COUNT = 5; +const RETRY_WAIT_TIME = 5000; +const MESH_CHUNK_THROTTLE_DELAY = 500; + +// The calculation of a mesh is spread across multiple requests. +// In order to avoid, that a huge amount of chunks is downloaded at full speed, +// we artificially throttle the download speed after the first MESH_CHUNK_THROTTLE_LIMIT +// requests for each segment. +const batchCounterPerSegment: Record = {}; +const MESH_CHUNK_THROTTLE_LIMIT = 50; + +// Maps from additional coordinates, layerName and segmentId to a ThreeDMap that stores for each chunk +// (at x, y, z) position whether the mesh chunk was loaded. +const adhocMeshesMapByLayer: Record>>> = {}; + +function marchingCubeSizeInTargetMag(): Vector3 { + return WkDevFlags.meshing.marchingCubeSizeInTargetMag; +} +const modifiedCells: Set = new Set(); +export function isMeshSTL(buffer: ArrayBuffer): boolean { + const dataView = new DataView(buffer); + const isMesh = stlMeshConstants.meshMarker.every( + (marker, index) => dataView.getUint8(index) === marker, + ); + return isMesh; +} + +function getOrAddMapForSegment( + layerName: string, + segmentId: number, + additionalCoordinates?: AdditionalCoordinate[] | null, +): ThreeDMap { + const additionalCoordKey = getAdditionalCoordinatesAsString(additionalCoordinates); + + const keys = [additionalCoordKey, layerName]; + // create new map if adhocMeshesMapByLayer[additionalCoordinatesString][layerName] doesn't exist yet. + _.set(adhocMeshesMapByLayer, keys, _.get(adhocMeshesMapByLayer, keys, new Map())); + const meshesMap = adhocMeshesMapByLayer[additionalCoordKey][layerName]; + const maybeMap = meshesMap.get(segmentId); + + if (maybeMap == null) { + const newMap = new ThreeDMap(); + meshesMap.set(segmentId, newMap); + return newMap; + } + + return maybeMap; +} + +export function* removeMesh(action: RemoveMeshAction, removeFromScene: boolean = true): Saga { + const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); + const additionalCoordKey = getAdditionalCoordinatesAsString(additionalCoordinates); + const { layerName } = action; + const segmentId = action.segmentId; + + if (removeFromScene) { + getSceneController().segmentMeshController.removeMeshById(segmentId, layerName); + } + removeMapForSegment(layerName, segmentId, additionalCoordKey); +} + +function removeMapForSegment( + layerName: string, + segmentId: number, + additionalCoordinateKey: string, +): void { + if ( + adhocMeshesMapByLayer[additionalCoordinateKey] == null || + adhocMeshesMapByLayer[additionalCoordinateKey][layerName] == null + ) { + return; + } + + adhocMeshesMapByLayer[additionalCoordinateKey][layerName].delete(segmentId); +} + +function getCubeSizeInMag1(zoomStep: number, magInfo: MagInfo): Vector3 { + // Convert marchingCubeSizeInTargetMag to mag1 via zoomStep + // Drop the last element of the Vector4; + const [x, y, z] = zoomedAddressToAnotherZoomStepWithInfo( + [...marchingCubeSizeInTargetMag(), zoomStep], + magInfo, + 0, + ); + return [x, y, z]; +} + +function clipPositionToCubeBoundary( + position: Vector3, + zoomStep: number, + magInfo: MagInfo, +): Vector3 { + const cubeSizeInMag1 = getCubeSizeInMag1(zoomStep, magInfo); + const currentCube = V3.floor(V3.divide3(position, cubeSizeInMag1)); + const clippedPosition = V3.scale3(currentCube, cubeSizeInMag1); + return clippedPosition; +} + +// front_xy, front_xz, front_yz, back_xy, back_xz, back_yz +const NEIGHBOR_LOOKUP = [ + [0, 0, -1], + [0, -1, 0], + [-1, 0, 0], + [0, 0, 1], + [0, 1, 0], + [1, 0, 0], +]; + +function getNeighborPosition( + clippedPosition: Vector3, + neighborId: number, + zoomStep: number, + magInfo: MagInfo, +): Vector3 { + const neighborMultiplier = NEIGHBOR_LOOKUP[neighborId]; + const cubeSizeInMag1 = getCubeSizeInMag1(zoomStep, magInfo); + const neighboringPosition: Vector3 = [ + clippedPosition[0] + neighborMultiplier[0] * cubeSizeInMag1[0], + clippedPosition[1] + neighborMultiplier[1] * cubeSizeInMag1[1], + clippedPosition[2] + neighborMultiplier[2] * cubeSizeInMag1[2], + ]; + return neighboringPosition; +} + +function* loadAdHocMeshFromAction(action: LoadAdHocMeshAction): Saga { + const { layerName } = action; + const layer = + layerName != null ? Model.getLayerByName(layerName) : Model.getVisibleSegmentationLayer(); + if (layer == null) { + return; + } + // Remove older mesh instance if it exists already. + yield* put(removeMeshAction(layer.name, action.segmentId)); + + yield* call( + loadAdHocMesh, + action.seedPosition, + action.seedAdditionalCoordinates, + action.segmentId, + false, + layer.name, + action.extraInfo, + ); +} + +export function* getMeshExtraInfo( + layerName: string, + maybeExtraInfo: AdHocMeshInfo | null | undefined, +): Saga { + const activeMappingByLayer = yield* select( + (state) => state.temporaryConfiguration.activeMappingByLayer, + ); + if (maybeExtraInfo != null) return maybeExtraInfo; + const mappingInfo = getMappingInfo(activeMappingByLayer, layerName); + const isMappingActive = mappingInfo.mappingStatus === MappingStatusEnum.ENABLED; + const mappingName = isMappingActive ? mappingInfo.mappingName : null; + const mappingType = isMappingActive ? mappingInfo.mappingType : null; + return { + mappingName, + mappingType, + }; +} + +function* getInfoForMeshLoading( + layer: DataLayer, + meshExtraInfo: AdHocMeshInfo, +): Saga<{ + zoomStep: number; + magInfo: MagInfo; +}> { + const magInfo = getMagInfo(layer.mags); + const preferredZoomStep = + meshExtraInfo.preferredQuality != null + ? meshExtraInfo.preferredQuality + : yield* select( + (state) => state.temporaryConfiguration.preferredQualityForMeshAdHocComputation, + ); + const zoomStep = magInfo.getClosestExistingIndex(preferredZoomStep); + return { + zoomStep, + magInfo: magInfo, + }; +} + +function* loadAdHocMesh( + seedPosition: Vector3, + seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, + segmentId: number, + removeExistingMesh: boolean = false, + layerName: string, + maybeExtraInfo?: AdHocMeshInfo, +): Saga { + const layer = Model.getLayerByName(layerName); + + if (segmentId === 0) { + return; + } + + yield* call([Model, Model.ensureSavedState]); + + const meshExtraInfo = yield* call(getMeshExtraInfo, layer.name, maybeExtraInfo); + + const { zoomStep, magInfo } = yield* call(getInfoForMeshLoading, layer, meshExtraInfo); + batchCounterPerSegment[segmentId] = 0; + + // If a REMOVE_MESH action is dispatched and consumed + // here before loadFullAdHocMesh is finished, the latter saga + // should be canceled automatically to avoid populating mesh data even though + // the mesh was removed. This is accomplished by redux-saga's race effect. + yield* race({ + loadFullAdHocMesh: call( + loadFullAdHocMesh, + layer, + segmentId, + seedPosition, + seedAdditionalCoordinates, + zoomStep, + meshExtraInfo, + magInfo, + removeExistingMesh, + ), + cancel: take( + ((action: Action) => + action.type === "REMOVE_MESH" && + action.segmentId === segmentId && + action.layerName === layer.name) as ActionPattern, + ), + }); + removeMeshWithoutVoxels(segmentId, layer.name, seedAdditionalCoordinates); +} + +function removeMeshWithoutVoxels( + segmentId: number, + layerName: string, + additionalCoordinates: AdditionalCoordinate[] | undefined | null, +) { + // If no voxels were added to the scene (e.g. because the segment doesn't have any voxels in this n-dimension), + // remove it from the store's state as well. + const { segmentMeshController } = getSceneController(); + if (!segmentMeshController.hasMesh(segmentId, layerName, additionalCoordinates)) { + Store.dispatch(removeMeshAction(layerName, segmentId)); + } +} + +function* loadFullAdHocMesh( + layer: DataLayer, + segmentId: number, + position: Vector3, + additionalCoordinates: AdditionalCoordinate[] | undefined | null, + zoomStep: number, + meshExtraInfo: AdHocMeshInfo, + magInfo: MagInfo, + removeExistingMesh: boolean, +): Saga { + let isInitialRequest = true; + const { mappingName, mappingType, opacity } = meshExtraInfo; + const clippedPosition = clipPositionToCubeBoundary(position, zoomStep, magInfo); + yield* put( + addAdHocMeshAction( + layer.name, + segmentId, + position, + additionalCoordinates, + mappingName, + mappingType, + opacity || Constants.DEFAULT_MESH_OPACITY, + ), + ); + yield* put(startedLoadingMeshAction(layer.name, segmentId)); + + const cubeSize = marchingCubeSizeInTargetMag(); + const tracingStoreHost = yield* select((state) => state.annotation.tracingStore.url); + const mag = magInfo.getMagByIndexOrThrow(zoomStep); + + const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state)); + const visibleSegmentationLayer = yield* select((state) => getVisibleSegmentationLayer(state)); + // Fetch from datastore if no volumetracing ... + let useDataStore = volumeTracing == null || visibleSegmentationLayer?.tracingId == null; + if (meshExtraInfo.useDataStore != null) { + // ... except if the caller specified whether to use the data store ... + useDataStore = meshExtraInfo.useDataStore; + } else if (volumeTracing?.hasEditableMapping) { + // ... or if an editable mapping is active. + useDataStore = false; + } + + // Segment stats can only be used for volume tracings that have a segment index + // and that don't have editable mappings. + const usePositionsFromSegmentIndex = + volumeTracing?.hasSegmentIndex && + !volumeTracing.hasEditableMapping && + visibleSegmentationLayer?.tracingId != null; + let positionsToRequest = usePositionsFromSegmentIndex + ? yield* getChunkPositionsFromSegmentIndex( + tracingStoreHost, + layer, + segmentId, + cubeSize, + mag, + clippedPosition, + additionalCoordinates, + ) + : [clippedPosition]; + + if (positionsToRequest.length === 0) { + //if no positions are requested, remove the mesh, + //so that the old one isn't displayed anymore + yield* put(removeMeshAction(layer.name, segmentId)); + } + while (positionsToRequest.length > 0) { + const currentPosition = positionsToRequest.shift(); + if (currentPosition == null) { + throw new Error("Satisfy typescript"); + } + const neighbors = yield* call( + maybeLoadMeshChunk, + layer, + segmentId, + currentPosition, + zoomStep, + meshExtraInfo, + magInfo, + isInitialRequest, + removeExistingMesh && isInitialRequest, + useDataStore, + !usePositionsFromSegmentIndex, + ); + isInitialRequest = false; + + // If we are using the positions from the segment index, the backend will + // send an empty neighbors array, as it's not necessary to have them. + if (usePositionsFromSegmentIndex && neighbors.length > 0) { + throw new Error("Retrieved neighbor positions even though these were not requested."); + } + positionsToRequest = positionsToRequest.concat(neighbors); + } + + yield* put(finishedLoadingMeshAction(layer.name, segmentId)); +} + +function* getChunkPositionsFromSegmentIndex( + tracingStoreHost: string, + layer: DataLayer, + segmentId: number, + cubeSize: Vector3, + mag: Vector3, + clippedPosition: Vector3, + additionalCoordinates: AdditionalCoordinate[] | null | undefined, +) { + const targetMagPositions = yield* call( + getBucketPositionsForAdHocMesh, + tracingStoreHost, + layer.name, + segmentId, + cubeSize, + mag, + additionalCoordinates, + ); + const mag1Positions = targetMagPositions.map((pos) => V3.scale3(pos, mag)); + return sortByDistanceTo(mag1Positions, clippedPosition) as Vector3[]; +} + +function hasMeshChunkExceededThrottleLimit(segmentId: number): boolean { + return batchCounterPerSegment[segmentId] > MESH_CHUNK_THROTTLE_LIMIT; +} + +function* maybeLoadMeshChunk( + layer: DataLayer, + segmentId: number, + clippedPosition: Vector3, + zoomStep: number, + meshExtraInfo: AdHocMeshInfo, + magInfo: MagInfo, + isInitialRequest: boolean, + removeExistingMesh: boolean, + useDataStore: boolean, + findNeighbors: boolean, +): Saga { + const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); + const threeDMap = getOrAddMapForSegment(layer.name, segmentId, additionalCoordinates); + + if (threeDMap.get(clippedPosition)) { + return []; + } + + if (hasMeshChunkExceededThrottleLimit(segmentId)) { + yield* call(sleep, MESH_CHUNK_THROTTLE_DELAY); + } + + batchCounterPerSegment[segmentId]++; + threeDMap.set(clippedPosition, true); + const scaleFactor = yield* select((state) => state.dataset.dataSource.scale.factor); + const dataStoreHost = yield* select((state) => state.dataset.dataStore.url); + const owningOrganization = yield* select((state) => state.dataset.owningOrganization); + const datasetDirectoryName = yield* select((state) => state.dataset.directoryName); + const tracingStoreHost = yield* select((state) => state.annotation.tracingStore.url); + const dataStoreUrl = `${dataStoreHost}/data/datasets/${owningOrganization}/${datasetDirectoryName}/layers/${ + layer.fallbackLayer != null ? layer.fallbackLayer : layer.name + }`; + const tracingStoreUrl = `${tracingStoreHost}/tracings/volume/${layer.name}`; + + const mag = magInfo.getMagByIndexOrThrow(zoomStep); + + if (isInitialRequest) { + sendAnalyticsEvent("request_isosurface", { + mode: useDataStore ? "view" : "annotation", + }); + } + + let retryCount = 0; + + const { segmentMeshController } = getSceneController(); + + const cubeSize = marchingCubeSizeInTargetMag(); + + while (retryCount < MAX_RETRY_COUNT) { + try { + const { buffer: responseBuffer, neighbors } = yield* call( + { + context: null, + fn: computeAdHocMesh, + }, + useDataStore ? dataStoreUrl : tracingStoreUrl, + { + position: clippedPosition, + additionalCoordinates: additionalCoordinates || undefined, + mag, + segmentId, + cubeSize, + scaleFactor, + findNeighbors, + ...meshExtraInfo, + }, + ); + const vertices = new Float32Array(responseBuffer); + + if (removeExistingMesh) { + segmentMeshController.removeMeshById(segmentId, layer.name); + } + + const opacity = meshExtraInfo.opacity || Constants.DEFAULT_MESH_OPACITY; + + // We await addMeshFromVerticesAsync here, because the mesh saga will remove + // an ad-hoc loaded mesh immediately if it was "empty". Since the check is + // done by looking at the scene, we await the population of the scene. + // Theoretically, this could be built differently so that other ad-hoc chunks + // can be loaded in parallel to addMeshFromVerticesAsync. However, it's unclear + // how big the bottleneck really is. + yield* call( + { fn: segmentMeshController.addMeshFromVerticesAsync, context: segmentMeshController }, + vertices, + segmentId, + layer.name, + opacity, + additionalCoordinates, + ); + return neighbors.map((neighbor) => + getNeighborPosition(clippedPosition, neighbor, zoomStep, magInfo), + ); + } catch (exception) { + retryCount++; + ErrorHandling.notify(exception as Error); + console.warn("Retrying mesh generation due to", exception); + yield* call(sleep, RETRY_WAIT_TIME * 2 ** retryCount); + } + } + + return []; +} + +function* markEditedCellAsDirty(): Saga { + const volumeTracing = yield* select((state) => getActiveSegmentationTracing(state)); + + if (volumeTracing != null && volumeTracing.fallbackLayer == null) { + const activeCellId = volumeTracing.activeCellId; + modifiedCells.add(activeCellId); + } +} + +function* refreshMeshes(): Saga { + yield* put(saveNowAction()); + // We reload all cells that got modified till the start of reloading. + // By that we avoid to remove cells that got annotated during reloading from the modifiedCells set. + const currentlyModifiedCells = new Set(modifiedCells); + modifiedCells.clear(); + + const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); + const additionalCoordKey = getAdditionalCoordinatesAsString(additionalCoordinates); + const segmentationLayer = Model.getVisibleSegmentationLayer(); + + if (!segmentationLayer) { + return; + } + + adhocMeshesMapByLayer[additionalCoordKey][segmentationLayer.name] = + adhocMeshesMapByLayer[additionalCoordKey][segmentationLayer.name] || new Map(); + const meshesMapForLayer = adhocMeshesMapByLayer[additionalCoordKey][segmentationLayer.name]; + + for (const [segmentId, threeDMap] of Array.from(meshesMapForLayer.entries())) { + if (!currentlyModifiedCells.has(segmentId)) { + continue; + } + + yield* call( + refreshMeshWithMap, + segmentId, + threeDMap, + segmentationLayer.name, + additionalCoordinates, + ); + } +} + +function* refreshMesh(action: RefreshMeshAction): Saga { + const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); + const additionalCoordKey = getAdditionalCoordinatesAsString(additionalCoordinates); + + const { segmentId, layerName } = action; + + const meshInfo = yield* select((state) => + getMeshInfoForSegment(state, additionalCoordinates, layerName, segmentId), + ); + + if (meshInfo == null) { + throw new Error( + `Mesh refreshing failed due to lack of mesh info for segment ${segmentId} in store.`, + ); + } + + if (meshInfo.isPrecomputed) { + yield* put(removeMeshAction(layerName, meshInfo.segmentId)); + yield* put( + loadPrecomputedMeshAction( + meshInfo.segmentId, + meshInfo.seedPosition, + meshInfo.seedAdditionalCoordinates, + meshInfo.meshFileName, + meshInfo.opacity, + layerName, + ), + ); + } else { + if (adhocMeshesMapByLayer[additionalCoordKey] == null) return; + const threeDMap = adhocMeshesMapByLayer[additionalCoordKey][action.layerName].get(segmentId); + if (threeDMap == null) { + return; + } + yield* call( + refreshMeshWithMap, + segmentId, + threeDMap, + layerName, + additionalCoordinates, + meshInfo.opacity, + ); + } +} + +function* refreshMeshWithMap( + segmentId: number, + threeDMap: ThreeDMap, + layerName: string, + additionalCoordinates: AdditionalCoordinate[] | null, + opacity: number = Constants.DEFAULT_MESH_OPACITY, +): Saga { + const meshInfo = yield* select((state) => + getMeshInfoForSegment(state, additionalCoordinates, layerName, segmentId), + ); + if (meshInfo == null) { + throw new Error( + `Mesh refreshing failed due to lack of mesh info for segment ${segmentId} in store.`, + ); + } + yield* call( + [ErrorHandling, ErrorHandling.assert], + !meshInfo.isPrecomputed, + "_refreshMeshWithMap was called for a precomputed mesh.", + ); + if (meshInfo.isPrecomputed) return; + const { mappingName, mappingType } = meshInfo; + const meshPositions = threeDMap.entries().filter(([value, _position]) => value); + + if (meshPositions.length === 0) { + return; + } + + // Remove mesh from cache. + yield* call(removeMesh, removeMeshAction(layerName, segmentId), false); + // The mesh should only be removed once after re-fetching the mesh first position. + let shouldBeRemoved = true; + + for (const [, position] of meshPositions) { + // Reload the mesh at the given position if it isn't already loaded there. + // This is done to ensure that every voxel of the mesh is reloaded. + yield* call( + loadAdHocMesh, + position, + additionalCoordinates, + segmentId, + shouldBeRemoved, + layerName, + { + mappingName, + mappingType, + opacity, + }, + ); + shouldBeRemoved = false; + } +} + +export default function* adHocMeshSaga(): Saga { + // Buffer actions since they might be dispatched before WK_READY + const loadAdHocMeshActionChannel = yield* actionChannel("LOAD_AD_HOC_MESH_ACTION"); + + yield* call(ensureSceneControllerReady); + yield* call(ensureWkReady); + yield* takeEvery(loadAdHocMeshActionChannel, loadAdHocMeshFromAction); + yield* takeEvery("REMOVE_MESH", removeMesh); + yield* takeEvery("REFRESH_MESHES", refreshMeshes); + yield* takeEvery("REFRESH_MESH", refreshMesh); + yield* takeEvery(["START_EDITING", "COPY_SEGMENTATION_LAYER"], markEditedCellAsDirty); +} diff --git a/frontend/javascripts/viewer/model/sagas/meshes/common_mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/meshes/common_mesh_saga.ts new file mode 100644 index 0000000000..1073bb7294 --- /dev/null +++ b/frontend/javascripts/viewer/model/sagas/meshes/common_mesh_saga.ts @@ -0,0 +1,219 @@ +import { saveAs } from "file-saver"; +import ErrorHandling from "libs/error_handling"; +import exportToStl from "libs/stl_exporter"; +import Toast from "libs/toast"; +import Zip from "libs/zipjs_wrapper"; +import messages from "messages"; +import type * as THREE from "three"; +import { all, call, put, take, takeEvery } from "typed-redux-saga"; +import getSceneController from "viewer/controller/scene_controller_provider"; +import { + type TriggerMeshDownloadAction, + type TriggerMeshesDownloadAction, + type UpdateMeshOpacityAction, + type UpdateMeshVisibilityAction, + removeMeshAction, + updateMeshVisibilityAction, +} from "viewer/model/actions/annotation_actions"; +import type { Saga } from "viewer/model/sagas/effect-generators"; +import { select } from "viewer/model/sagas/effect-generators"; +import { stlMeshConstants } from "viewer/view/right-border-tabs/segments_tab/segments_view"; +import { getAdditionalCoordinatesAsString } from "../../accessors/flycam_accessor"; +import type { FlycamAction } from "../../actions/flycam_actions"; +import type { + BatchUpdateGroupsAndSegmentsAction, + RemoveSegmentAction, + UpdateSegmentAction, +} from "../../actions/volumetracing_actions"; +import { ensureSceneControllerReady, ensureWkReady } from "../ready_sagas"; + +export const NO_LOD_MESH_INDEX = -1; + +function* downloadMeshCellById(cellName: string, segmentId: number, layerName: string): Saga { + const { segmentMeshController } = getSceneController(); + const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); + const geometry = segmentMeshController.getMeshGeometryInBestLOD( + segmentId, + layerName, + additionalCoordinates, + ); + + if (geometry == null) { + const errorMessage = messages["tracing.not_mesh_available_to_download"]; + Toast.error(errorMessage, { + sticky: false, + }); + return; + } + + try { + const blob = getSTLBlob(geometry, segmentId); + yield* call(saveAs, blob, `${cellName}-${segmentId}.stl`); + } catch (exception) { + ErrorHandling.notify(exception as Error); + Toast.error("Could not export to STL. See console for details"); + console.error(exception); + } +} + +function* downloadMeshCellsAsZIP( + segments: Array<{ segmentName: string; segmentId: number; layerName: string }>, +): Saga { + const { segmentMeshController } = getSceneController(); + const zipWriter = new Zip.ZipWriter(new Zip.BlobWriter("application/zip")); + const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); + try { + const addFileToZipWriterPromises = segments.map((element) => { + const geometry = segmentMeshController.getMeshGeometryInBestLOD( + element.segmentId, + element.layerName, + additionalCoordinates, + ); + + if (geometry == null) { + const errorMessage = messages["tracing.not_mesh_available_to_download"]; + Toast.error(errorMessage, { + sticky: false, + }); + return; + } + const stlDataReader = new Zip.BlobReader(getSTLBlob(geometry, element.segmentId)); + return zipWriter.add(`${element.segmentName}-${element.segmentId}.stl`, stlDataReader); + }); + yield all(addFileToZipWriterPromises); + const result = yield* call([zipWriter, zipWriter.close]); + yield* call(saveAs, result as Blob, "mesh-export.zip"); + } catch (exception) { + ErrorHandling.notify(exception as Error); + Toast.error("Could not export meshes as STL files. See console for details"); + console.error(exception); + } +} + +const getSTLBlob = (geometry: THREE.Group, segmentId: number): Blob => { + const stlDataViews = exportToStl(geometry); + // Encode mesh and cell id property + const { meshMarker, segmentIdIndex } = stlMeshConstants; + meshMarker.forEach((marker, index) => { + stlDataViews[0].setUint8(index, marker); + }); + stlDataViews[0].setUint32(segmentIdIndex, segmentId, true); + return new Blob(stlDataViews); +}; + +function* downloadMeshCell(action: TriggerMeshDownloadAction): Saga { + yield* call(downloadMeshCellById, action.segmentName, action.segmentId, action.layerName); +} + +function* downloadMeshCells(action: TriggerMeshesDownloadAction): Saga { + yield* call(downloadMeshCellsAsZIP, action.segmentsArray); +} + +function* handleRemoveSegment(action: RemoveSegmentAction) { + // The dispatched action will make sure that the mesh entry is removed from the + // store and from the scene. + yield* put(removeMeshAction(action.layerName, action.segmentId)); +} + +function* handleMeshVisibilityChange(action: UpdateMeshVisibilityAction): Saga { + const { id, visibility, layerName, additionalCoordinates } = action; + const { segmentMeshController } = yield* call(getSceneController); + segmentMeshController.setMeshVisibility(id, visibility, layerName, additionalCoordinates); +} + +export function* handleAdditionalCoordinateUpdate(): Saga { + // We want to prevent iterating through all additional coordinates to adjust the mesh visibility, so we store the + // previous additional coordinates in this method. Thus we have to catch SET_ADDITIONAL_COORDINATES actions in a + // while-true loop and register this saga in the root saga instead of calling from the mesh saga. + yield* call(ensureWkReady); + + let previousAdditionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); + const { segmentMeshController } = yield* call(getSceneController); + + while (true) { + const action = (yield* take(["SET_ADDITIONAL_COORDINATES"]) as any) as FlycamAction; + // Satisfy TS + if (action.type !== "SET_ADDITIONAL_COORDINATES") { + // Don't throw as this would interfere with the never return type + console.error("Unexpected action.type. Ignoring SET_ADDITIONAL_COORDINATES action..."); + continue; + } + const meshRecords = segmentMeshController.meshesGroupsPerSegmentId; + + if (action.values == null || action.values.length === 0) continue; + const newAdditionalCoordKey = getAdditionalCoordinatesAsString(action.values); + + for (const additionalCoordinates of [action.values, previousAdditionalCoordinates]) { + const currentAdditionalCoordKey = getAdditionalCoordinatesAsString(additionalCoordinates); + const shouldBeVisible = currentAdditionalCoordKey === newAdditionalCoordKey; + const recordsOfLayers = meshRecords[currentAdditionalCoordKey] || {}; + for (const [layerName, recordsForOneLayer] of Object.entries(recordsOfLayers)) { + const segmentIds = Object.keys(recordsForOneLayer); + for (const segmentIdAsString of segmentIds) { + const segmentId = Number.parseInt(segmentIdAsString); + yield* put( + updateMeshVisibilityAction( + layerName, + segmentId, + shouldBeVisible, + additionalCoordinates, + ), + ); + yield* call( + { + context: segmentMeshController, + fn: segmentMeshController.setMeshVisibility, + }, + segmentId, + shouldBeVisible, + layerName, + additionalCoordinates, + ); + } + } + } + previousAdditionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); + } +} + +function* handleSegmentColorChange(action: UpdateSegmentAction): Saga { + const { segmentMeshController } = yield* call(getSceneController); + const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); + if ( + "color" in action.segment && + segmentMeshController.hasMesh(action.segmentId, action.layerName, additionalCoordinates) + ) { + segmentMeshController.setMeshColor(action.segmentId, action.layerName); + } +} + +function* handleMeshOpacityChange(action: UpdateMeshOpacityAction): Saga { + const { segmentMeshController } = yield* call(getSceneController); + segmentMeshController.setMeshOpacity(action.id, action.layerName, action.opacity); +} + +function* handleBatchSegmentColorChange( + batchAction: BatchUpdateGroupsAndSegmentsAction, +): Saga { + // Manually unpack batched actions and handle these. + // In theory, this could happen automatically. See this issue in the corresponding (rather unmaintained) package: https://github.com/tshelburne/redux-batched-actions/pull/18 + // However, there seem to be some problems with that approach (e.g., too many updates, infinite recursion) and the discussion there didn't really reach a consensus + // about the correct solution. + // This is why we stick to the manual unpacking for now. + const updateSegmentActions = batchAction.payload + .filter((action) => action.type === "UPDATE_SEGMENT") + .map((action) => call(handleSegmentColorChange, action as UpdateSegmentAction)); + yield* all(updateSegmentActions); +} + +export default function* commonMeshSaga(): Saga { + yield* call(ensureSceneControllerReady); + yield* call(ensureWkReady); + yield* takeEvery("TRIGGER_MESH_DOWNLOAD", downloadMeshCell); + yield* takeEvery("TRIGGER_MESHES_DOWNLOAD", downloadMeshCells); + yield* takeEvery("REMOVE_SEGMENT", handleRemoveSegment); + yield* takeEvery("UPDATE_MESH_VISIBILITY", handleMeshVisibilityChange); + yield* takeEvery("UPDATE_SEGMENT", handleSegmentColorChange); + yield* takeEvery("UPDATE_MESH_OPACITY", handleMeshOpacityChange); + yield* takeEvery("BATCH_UPDATE_GROUPS_AND_SEGMENTS", handleBatchSegmentColorChange); +} diff --git a/frontend/javascripts/viewer/model/sagas/meshes/precomputed_mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/meshes/precomputed_mesh_saga.ts new file mode 100644 index 0000000000..ac8829701a --- /dev/null +++ b/frontend/javascripts/viewer/model/sagas/meshes/precomputed_mesh_saga.ts @@ -0,0 +1,501 @@ +import type { MeshLodInfo } from "admin/api/mesh"; +import { getMeshfilesForDatasetLayer, meshApi } from "admin/rest_api"; +import { mergeGeometries } from "libs/BufferGeometryUtils"; +import Deferred from "libs/async/deferred"; +import processTaskWithPool from "libs/async/task_pool"; +import { computeBvhAsync } from "libs/compute_bvh_async"; +import { getDracoLoader } from "libs/draco"; +import Toast from "libs/toast"; +import { chunkDynamically } from "libs/utils"; +import _ from "lodash"; +import messages from "messages"; +import type { ActionPattern } from "redux-saga/effects"; +import { actionChannel, call, put, race, take, takeEvery } from "typed-redux-saga"; +import type { APIDataset, APIMeshFileInfo, APISegmentationLayer } from "types/api_types"; +import type { AdditionalCoordinate } from "types/api_types"; +import type { Vector3, Vector4 } from "viewer/constants"; +import Constants from "viewer/constants"; +import CustomLOD from "viewer/controller/custom_lod"; +import { + type BufferGeometryWithInfo, + type UnmergedBufferGeometryWithInfo, + VertexSegmentMapping, + sortByDistanceTo, +} from "viewer/controller/mesh_helpers"; +import getSceneController from "viewer/controller/scene_controller_provider"; +import { + getSegmentationLayerByName, + getVisibleSegmentationLayer, +} from "viewer/model/accessors/dataset_accessor"; +import { + getEditableMappingForVolumeTracingId, + getTracingForSegmentationLayer, +} from "viewer/model/accessors/volumetracing_accessor"; +import type { Action } from "viewer/model/actions/actions"; +import { + type MaybeFetchMeshFilesAction, + addPrecomputedMeshAction, + dispatchMaybeFetchMeshFilesAsync, + finishedLoadingMeshAction, + removeMeshAction, + startedLoadingMeshAction, + updateCurrentMeshFileAction, + updateMeshFileListAction, +} from "viewer/model/actions/annotation_actions"; +import type { LoadPrecomputedMeshAction } from "viewer/model/actions/segmentation_actions"; +import type { Saga } from "viewer/model/sagas/effect-generators"; +import { select } from "viewer/model/sagas/effect-generators"; +import Store from "viewer/store"; +import { getBaseSegmentationName } from "viewer/view/right-border-tabs/segments_tab/segments_view_helper"; +import { ensureSceneControllerReady, ensureWkReady } from "../ready_sagas"; +import { getMeshExtraInfo } from "./ad_hoc_mesh_saga"; + +const PARALLEL_PRECOMPUTED_MESH_LOADING_COUNT = 32; +const MIN_BATCH_SIZE_IN_BYTES = 2 ** 16; + +// Avoid redundant fetches of mesh files for the same layer by +// storing Deferreds per layer lazily. +let fetchDeferredsPerLayer: Record, unknown>> = {}; +function* maybeFetchMeshFiles(action: MaybeFetchMeshFilesAction): Saga { + const { segmentationLayer, dataset, mustRequest, autoActivate, callback } = action; + + if (!segmentationLayer) { + callback([]); + return; + } + + const layerName = segmentationLayer.name; + + function* maybeActivateMeshFile(availableMeshFiles: APIMeshFileInfo[]) { + const currentMeshFile = yield* select( + (state) => state.localSegmentationData[layerName].currentMeshFile, + ); + if (!currentMeshFile && availableMeshFiles.length > 0 && autoActivate) { + yield* put(updateCurrentMeshFileAction(layerName, availableMeshFiles[0].name)); + } + } + + // If a deferred already exists (and mustRequest is not true), the deferred + // can be awaited (regardless of whether it's finished or not) and its + // content used to call the callback. + if (fetchDeferredsPerLayer[layerName] && !mustRequest) { + const availableMeshFiles = yield* call(() => fetchDeferredsPerLayer[layerName].promise()); + yield* maybeActivateMeshFile(availableMeshFiles); + callback(availableMeshFiles); + return; + } + // A request has to be made (either because none was made before or because + // it is enforced by mustRequest). + // If mustRequest is true and an old deferred exists, a new deferred will be created which + // replaces the old one (old references to the first Deferred will still + // work and will be resolved by the corresponding saga execution). + const deferred = new Deferred, unknown>(); + fetchDeferredsPerLayer[layerName] = deferred; + + const availableMeshFiles = yield* call( + getMeshfilesForDatasetLayer, + dataset.dataStore.url, + dataset, + getBaseSegmentationName(segmentationLayer), + ); + yield* put(updateMeshFileListAction(layerName, availableMeshFiles)); + deferred.resolve(availableMeshFiles); + + yield* maybeActivateMeshFile(availableMeshFiles); + + callback(availableMeshFiles); +} + +function* loadPrecomputedMesh(action: LoadPrecomputedMeshAction) { + const { segmentId, seedPosition, seedAdditionalCoordinates, meshFileName, layerName, opacity } = + action; + const layer = yield* select((state) => + layerName != null + ? getSegmentationLayerByName(state.dataset, layerName) + : getVisibleSegmentationLayer(state), + ); + if (layer == null) return; + + // Remove older mesh instance if it exists already. + yield* put(removeMeshAction(layer.name, action.segmentId)); + + // If a REMOVE_MESH action is dispatched and consumed + // here before loadPrecomputedMeshForSegmentId is finished, the latter saga + // should be canceled automatically to avoid populating mesh data even though + // the mesh was removed. This is accomplished by redux-saga's race effect. + yield* race({ + loadPrecomputedMeshForSegmentId: call( + loadPrecomputedMeshForSegmentId, + segmentId, + seedPosition, + seedAdditionalCoordinates, + meshFileName, + layer, + opacity || Constants.DEFAULT_MESH_OPACITY, + ), + cancel: take( + ((otherAction: Action) => + otherAction.type === "REMOVE_MESH" && + otherAction.segmentId === segmentId && + otherAction.layerName === layer.name) as ActionPattern, + ), + }); +} + +type ChunksMap = Record; + +function* loadPrecomputedMeshForSegmentId( + segmentId: number, + seedPosition: Vector3, + seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, + meshFileName: string, + segmentationLayer: APISegmentationLayer, + opacity: number, +): Saga { + const layerName = segmentationLayer.name; + const mappingName = yield* call(getMappingName, segmentationLayer); + yield* put( + addPrecomputedMeshAction( + layerName, + segmentId, + seedPosition, + seedAdditionalCoordinates, + meshFileName, + mappingName, + opacity, + ), + ); + yield* put(startedLoadingMeshAction(layerName, segmentId)); + const dataset = yield* select((state) => state.dataset); + const additionalCoordinates = yield* select((state) => state.flycam.additionalCoordinates); + + const availableMeshFiles = yield* call( + dispatchMaybeFetchMeshFilesAsync, + Store.dispatch, + segmentationLayer, + dataset, + false, + false, + ); + + const meshFile = availableMeshFiles.find((file) => file.name === meshFileName); + if (!meshFile) { + Toast.error("Could not load mesh, since the requested mesh file was not found."); + return; + } + if (segmentId === 0) { + Toast.error("Could not load mesh, since the clicked segment ID is 0."); + return; + } + + let availableChunksMap: ChunksMap = {}; + let chunkScale: Vector3 | null = null; + let loadingOrder: number[] | null = null; + let lods: MeshLodInfo[] | null = null; + try { + const chunkDescriptors = yield* call( + _getChunkLoadingDescriptors, + segmentId, + dataset, + segmentationLayer, + meshFile, + ); + lods = chunkDescriptors.segmentInfo.lods; + availableChunksMap = chunkDescriptors.availableChunksMap; + chunkScale = chunkDescriptors.segmentInfo.chunkScale; + loadingOrder = chunkDescriptors.loadingOrder; + } catch (exception) { + Toast.warning(messages["tracing.mesh_listing_failed"](segmentId)); + console.warn( + `Mesh chunks for segment ${segmentId} couldn't be loaded due to`, + exception, + "\nOne possible explanation could be that the segment was not included in the mesh file because it's smaller than the dust threshold that was specified for the mesh computation.", + ); + yield* put(finishedLoadingMeshAction(layerName, segmentId)); + yield* put(removeMeshAction(layerName, segmentId)); + return; + } + + for (const lod of loadingOrder) { + yield* call( + loadPrecomputedMeshesInChunksForLod, + dataset, + layerName, + meshFile, + segmentationLayer, + segmentId, + seedPosition, + availableChunksMap, + lod, + (lod: number) => extractScaleFromMatrix(lods[lod].transform), + chunkScale, + additionalCoordinates, + opacity, + ); + } + + yield* put(finishedLoadingMeshAction(layerName, segmentId)); +} + +function* getMappingName(segmentationLayer: APISegmentationLayer) { + const meshExtraInfo = yield* call(getMeshExtraInfo, segmentationLayer.name, null); + const editableMapping = yield* select((state) => + getEditableMappingForVolumeTracingId(state, segmentationLayer.tracingId), + ); + + // meshExtraInfo.mappingName contains the currently active mapping + // (can be the id of an editable mapping). However, we always need to + // use the mapping name of the on-disk mapping. + return editableMapping != null ? editableMapping.baseMappingName : meshExtraInfo.mappingName; +} + +function* _getChunkLoadingDescriptors( + segmentId: number, + dataset: APIDataset, + segmentationLayer: APISegmentationLayer, + meshFile: APIMeshFileInfo, +) { + const availableChunksMap: ChunksMap = {}; + let loadingOrder: number[] = []; + + const { segmentMeshController } = getSceneController(); + const version = meshFile.formatVersion; + + const editableMapping = yield* select((state) => + getEditableMappingForVolumeTracingId(state, segmentationLayer.tracingId), + ); + const tracing = yield* select((state) => + getTracingForSegmentationLayer(state, segmentationLayer), + ); + const mappingName = yield* call(getMappingName, segmentationLayer); + + if (version < 3) { + console.warn("The active mesh file uses a version lower than 3, which is not supported"); + } + + // mappingName only exists for versions >= 3 + if (meshFile.mappingName != null && meshFile.mappingName !== mappingName) { + throw Error( + `Trying to use a mesh file that was computed for mapping ${meshFile.mappingName} for a requested mapping of ${mappingName}.`, + ); + } + + const segmentInfo = yield* call( + meshApi.getMeshfileChunksForSegment, + dataset.dataStore.url, + dataset, + getBaseSegmentationName(segmentationLayer), + meshFile, + segmentId, + // The back-end should only receive a non-null mapping name, + // if it should perform extra (reverse) look ups to compute a mesh + // with a specific mapping from a mesh file that was computed + // without a mapping. + meshFile.mappingName == null ? mappingName : null, + editableMapping != null && tracing ? tracing.tracingId : null, + ); + segmentInfo.lods.forEach((meshLodInfo, lodIndex) => { + availableChunksMap[lodIndex] = meshLodInfo?.chunks; + loadingOrder.push(lodIndex); + meshLodInfo.transform; + }); + const currentLODGroup: CustomLOD = + (yield* call( + { + context: segmentMeshController, + fn: segmentMeshController.getLODGroupOfLayer, + }, + segmentationLayer.name, + )) ?? new CustomLOD(); + const currentLODIndex = yield* call( + { + context: currentLODGroup, + fn: currentLODGroup.getCurrentLOD, + }, + Math.max(...loadingOrder), + ); + // Load the chunks closest to the current LOD first. + loadingOrder.sort((a, b) => Math.abs(a - currentLODIndex) - Math.abs(b - currentLODIndex)); + + return { + availableChunksMap, + loadingOrder, + segmentInfo, + }; +} +function extractScaleFromMatrix(transform: [Vector4, Vector4, Vector4]): Vector3 { + return [transform[0][0], transform[1][1], transform[2][2]]; +} + +function* loadPrecomputedMeshesInChunksForLod( + dataset: APIDataset, + layerName: string, + meshFile: APIMeshFileInfo, + segmentationLayer: APISegmentationLayer, + segmentId: number, + seedPosition: Vector3, + availableChunksMap: ChunksMap, + lod: number, + getGlobalScale: (lod: number) => Vector3 | null, + chunkScale: Vector3 | null, + additionalCoordinates: AdditionalCoordinate[] | null, + opacity: number, +) { + const { segmentMeshController } = getSceneController(); + const loader = getDracoLoader(); + if (availableChunksMap[lod] == null) { + return; + } + const availableChunks = availableChunksMap[lod]; + // Sort the chunks by distance to the seedPosition, so that the mesh loads from the inside out + const sortedAvailableChunks = sortByDistanceTo(availableChunks, seedPosition); + + const batches = chunkDynamically( + sortedAvailableChunks as meshApi.MeshChunk[], + MIN_BATCH_SIZE_IN_BYTES, + (chunk) => chunk.byteSize, + ); + + let bufferGeometries: UnmergedBufferGeometryWithInfo[] = []; + const tasks = batches.map( + (chunks) => + function* loadChunks(): Saga { + const dataForChunks = yield* call( + meshApi.getMeshfileChunkData, + dataset.dataStore.url, + dataset, + getBaseSegmentationName(segmentationLayer), + { + meshFile, + // Only extract the relevant properties + requests: chunks.map(({ byteOffset, byteSize }) => ({ + byteOffset, + byteSize, + segmentId, + })), + }, + ); + + const errorsWithDetails = []; + + for (const [chunk, data] of _.zip(chunks, dataForChunks)) { + try { + if (chunk == null || data == null) { + throw new Error("Unexpected null value."); + } + const position = chunk.position; + const bufferGeometry = (yield* call( + loader.decodeDracoFileAsync, + data, + )) as UnmergedBufferGeometryWithInfo; + bufferGeometry.unmappedSegmentId = chunk.unmappedSegmentId; + if (chunkScale != null) { + bufferGeometry.scale(...chunkScale); + } + + bufferGeometry.translate(position[0], position[1], position[2]); + // Compute vertex normals to achieve smooth shading. We do this here + // within the chunk-specific code (instead of after all chunks are merged) + // to distribute the workload a bit over time. + bufferGeometry.computeVertexNormals(); + + // Eagerly add the chunk geometry so that they will be rendered + // as soon as possible. These chunks will be removed later and then + // replaced by a merged geometry so that we have better performance + // for large meshes. + yield* call( + { + context: segmentMeshController, + fn: segmentMeshController.addMeshFromGeometry, + }, + bufferGeometry, + segmentId, + // Apply the scale from the segment info, which includes dataset scale and mag + getGlobalScale(lod), + lod, + layerName, + additionalCoordinates, + opacity, + false, + ); + + bufferGeometries.push(bufferGeometry); + } catch (error) { + errorsWithDetails.push({ error, chunk }); + } + } + + if (errorsWithDetails.length > 0) { + console.warn("Errors occurred while decoding mesh chunks:", errorsWithDetails); + // Use first error as representative + throw errorsWithDetails[0].error; + } + }, + ); + + try { + yield* call(processTaskWithPool, tasks, PARALLEL_PRECOMPUTED_MESH_LOADING_COUNT); + } catch (exception) { + Toast.warning(`Some mesh chunks could not be loaded for segment ${segmentId}.`); + console.error(exception); + } + + // Merge Chunks + const sortedBufferGeometries = _.sortBy( + bufferGeometries, + (geometryWithInfo) => geometryWithInfo.unmappedSegmentId, + ); + + // mergeGeometries will crash if the array is empty. Even if it's not empty, + // the function might return null in case of another error. + const mergedGeometry = ( + sortedBufferGeometries.length > 0 ? mergeGeometries(sortedBufferGeometries, false) : null + ) as BufferGeometryWithInfo | null; + + if (mergedGeometry == null) { + console.error("Merged geometry is null. Look at error above."); + return; + } + mergedGeometry.vertexSegmentMapping = new VertexSegmentMapping(sortedBufferGeometries); + mergedGeometry.boundsTree = yield* call(computeBvhAsync, mergedGeometry); + + // Remove the eagerly added chunks (see above). + yield* call( + { + context: segmentMeshController, + fn: segmentMeshController.removeMeshById, + }, + segmentId, + layerName, + { lod }, + ); + + // Add the final merged geometry. + yield* call( + { + context: segmentMeshController, + fn: segmentMeshController.addMeshFromGeometry, + }, + mergedGeometry, + segmentId, + // Apply the scale from the segment info, which includes dataset scale and mag + getGlobalScale(lod), + lod, + layerName, + additionalCoordinates, + opacity, + true, + ); +} + +export default function* precomputedMeshSaga(): Saga { + // Buffer actions since they might be dispatched before WK_READY + fetchDeferredsPerLayer = {}; + const loadPrecomputedMeshActionChannel = yield* actionChannel("LOAD_PRECOMPUTED_MESH_ACTION"); + const maybeFetchMeshFilesActionChannel = yield* actionChannel("MAYBE_FETCH_MESH_FILES"); + + yield* call(ensureSceneControllerReady); + yield* call(ensureWkReady); + yield* takeEvery(maybeFetchMeshFilesActionChannel, maybeFetchMeshFiles); + yield* takeEvery(loadPrecomputedMeshActionChannel, loadPrecomputedMesh); +} diff --git a/frontend/javascripts/viewer/model/sagas/root_saga.ts b/frontend/javascripts/viewer/model/sagas/root_saga.ts index d70bfe6623..8fbf573c4b 100644 --- a/frontend/javascripts/viewer/model/sagas/root_saga.ts +++ b/frontend/javascripts/viewer/model/sagas/root_saga.ts @@ -9,7 +9,6 @@ import DatasetSagas from "viewer/model/sagas/dataset_saga"; import type { Saga } from "viewer/model/sagas/effect-generators"; import loadHistogramDataSaga from "viewer/model/sagas/load_histogram_data_saga"; import MappingSaga from "viewer/model/sagas/mapping_saga"; -import meshSaga, { handleAdditionalCoordinateUpdate } from "viewer/model/sagas/mesh_saga"; import { watchDataRelevantChanges } from "viewer/model/sagas/prefetch_saga"; import ProofreadSaga from "viewer/model/sagas/proofread_saga"; import ReadySagas from "viewer/model/sagas/ready_sagas"; @@ -22,6 +21,9 @@ import VolumetracingSagas from "viewer/model/sagas/volumetracing_saga"; import type { EscalateErrorAction } from "../actions/actions"; import { setIsWkReadyAction } from "../actions/ui_actions"; import maintainMaximumZoomForAllMagsSaga from "./flycam_info_cache_saga"; +import adHocMeshSaga from "./meshes/ad_hoc_mesh_saga"; +import commonMeshSaga, { handleAdditionalCoordinateUpdate } from "./meshes/common_mesh_saga"; +import precomputedMeshSaga from "./meshes/precomputed_mesh_saga"; import splitBoundaryMeshSaga from "./split_boundary_mesh_saga"; import { warnIfEmailIsUnverified } from "./user_saga"; @@ -68,7 +70,9 @@ function* restartableSaga(): Saga { call(listenToClipHistogramSaga), call(loadHistogramDataSaga), call(watchDataRelevantChanges), - call(meshSaga), + call(adHocMeshSaga), + call(precomputedMeshSaga), + call(commonMeshSaga), call(watchTasksAsync), call(MappingSaga), call(ProofreadSaga), From 70156e1410a62ab0148c5a0d0d6fc787b06f2a4e Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Wed, 28 May 2025 14:46:28 +0200 Subject: [PATCH 09/12] fix function call --- frontend/javascripts/viewer/model_initialization.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/javascripts/viewer/model_initialization.ts b/frontend/javascripts/viewer/model_initialization.ts index 9e3359a1d2..0d5a3ed3c9 100644 --- a/frontend/javascripts/viewer/model_initialization.ts +++ b/frontend/javascripts/viewer/model_initialization.ts @@ -34,6 +34,7 @@ import type { } from "types/api_types"; import type { Mutable } from "types/globals"; import constants, { ControlModeEnum, type Vector3 } from "viewer/constants"; +import Constants from "viewer/constants"; import type { DirectLayerSpecificProps, PartialUrlManagerState, @@ -891,6 +892,7 @@ async function applyLayerState(stateByLayer: UrlStateByLayer) { seedPosition, seedAdditionalCoordinates, meshFileName, + Constants.DEFAULT_MESH_OPACITY, effectiveLayerName, ), ); From f897b895e9cdb964571321bdd2c4557321a7f048 Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Wed, 28 May 2025 15:02:17 +0200 Subject: [PATCH 10/12] changelog --- CHANGELOG.unreleased.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index eb457a6171..d2a014043f 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -18,6 +18,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Added the possibility for super users to retry manually cancelled jobs from the jobs list. [#8629](https://github.com/scalableminds/webknossos/pull/8629) - Added checkboxes to the segments tab that allow to show/hide individual segments. The visibility of segments that are not listed in the segments list can be controlled with a new "Hide unlisted segments" toggle in the layer settings. Additionally, you can right-click a segment in a data viewport and select "Only show this segment" (and similar functionality). [#8546](https://github.com/scalableminds/webknossos/pull/8546) - Instead of pasting a dataset position from the clipboard to the position input box, you can simply paste it without focussing the position input first. Furthermore, you can also paste a "hash string", such as `#1406,1794,1560,0,0.234,186`, as it can be found in WK URLs. Pasting such a string will also set the encoded zoom, rotation, viewport etc. Note that the `#` has to be included in the pasted text. You can also copy and paste an entire link, but note that the dataset or annotation id in the link will be ignored. [#8652](https://github.com/scalableminds/webknossos/pull/8652) +- Meshes are now reloaded using their previous opacity value. [#8622](https://github.com/scalableminds/webknossos/pull/8622) ### Changed - Remove `data.maybe` dependency and replaced with regular Typescript types. [#8563](https://github.com/scalableminds/webknossos/pull/8563) From c4f41c3a796d50706e4e3c32652cbeda7de24044 Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Wed, 28 May 2025 15:48:14 +0200 Subject: [PATCH 11/12] apply coderabbit suggestions --- .../viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts index c6b7e0c950..cebb491ca1 100644 --- a/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts +++ b/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts @@ -47,7 +47,6 @@ import type { Saga } from "viewer/model/sagas/effect-generators"; import { select } from "viewer/model/sagas/effect-generators"; import { Model } from "viewer/singletons"; import Store from "viewer/store"; -import { stlMeshConstants } from "viewer/view/right-border-tabs/segments_tab/segments_view"; import { getAdditionalCoordinatesAsString } from "../../accessors/flycam_accessor"; import { ensureSceneControllerReady, ensureWkReady } from "../ready_sagas"; @@ -70,13 +69,6 @@ function marchingCubeSizeInTargetMag(): Vector3 { return WkDevFlags.meshing.marchingCubeSizeInTargetMag; } const modifiedCells: Set = new Set(); -export function isMeshSTL(buffer: ArrayBuffer): boolean { - const dataView = new DataView(buffer); - const isMesh = stlMeshConstants.meshMarker.every( - (marker, index) => dataView.getUint8(index) === marker, - ); - return isMesh; -} function getOrAddMapForSegment( layerName: string, @@ -554,12 +546,17 @@ function* refreshMeshes(): Saga { continue; } + const meshInfo = yield* select((state) => + getMeshInfoForSegment(state, additionalCoordinates, segmentationLayer.name, segmentId), + ); + yield* call( refreshMeshWithMap, segmentId, threeDMap, segmentationLayer.name, additionalCoordinates, + meshInfo?.opacity || Constants.DEFAULT_MESH_OPACITY, ); } } From 38cdef22faca04a740ac41604728e9b0ed1e54f5 Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Fri, 30 May 2025 23:48:41 +0200 Subject: [PATCH 12/12] address review: pass undefined as opacity and only set default opacity in some central places --- frontend/javascripts/viewer/api/api_latest.ts | 2 +- .../viewer/controller/segment_mesh_controller.ts | 9 +++++---- .../viewer/model/actions/annotation_actions.ts | 8 ++++---- .../viewer/model/actions/segmentation_actions.ts | 4 ++-- .../viewer/model/reducers/annotation_reducer.ts | 3 +-- .../viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts | 13 ++++++------- .../model/sagas/meshes/precomputed_mesh_saga.ts | 7 +++---- .../viewer/model/sagas/proofread_saga.ts | 4 ++-- frontend/javascripts/viewer/model_initialization.ts | 3 +-- frontend/javascripts/viewer/view/context_menu.tsx | 4 ++-- .../segments_tab/segments_view.tsx | 4 ++-- 11 files changed, 29 insertions(+), 32 deletions(-) diff --git a/frontend/javascripts/viewer/api/api_latest.ts b/frontend/javascripts/viewer/api/api_latest.ts index b6ff2ef06b..509db1f150 100644 --- a/frontend/javascripts/viewer/api/api_latest.ts +++ b/frontend/javascripts/viewer/api/api_latest.ts @@ -2521,7 +2521,7 @@ class DataApi { seedPosition, seedAdditionalCoordinates, meshFileName, - Constants.DEFAULT_MESH_OPACITY, + undefined, effectiveLayerName, ), ); diff --git a/frontend/javascripts/viewer/controller/segment_mesh_controller.ts b/frontend/javascripts/viewer/controller/segment_mesh_controller.ts index ff6960afc0..032afb21c4 100644 --- a/frontend/javascripts/viewer/controller/segment_mesh_controller.ts +++ b/frontend/javascripts/viewer/controller/segment_mesh_controller.ts @@ -17,6 +17,7 @@ import Store from "viewer/store"; import { computeBvhAsync } from "libs/compute_bvh_async"; import type { BufferAttribute } from "three"; +import Constants from "viewer/constants"; import { NO_LOD_MESH_INDEX } from "viewer/model/sagas/meshes/common_mesh_saga"; import type { BufferGeometryWithInfo } from "./mesh_helpers"; @@ -110,7 +111,7 @@ export default class SegmentMeshController { vertices: Float32Array, segmentId: number, layerName: string, - opacity: number, + opacity: number | undefined, additionalCoordinates?: AdditionalCoordinate[] | undefined | null, ): Promise { // Currently, this function is only used by ad hoc meshing. @@ -139,7 +140,7 @@ export default class SegmentMeshController { segmentId: number, layerName: string, geometry: BufferGeometryWithInfo, - opacity: number, + opacity: number | undefined, isMerged: boolean, ): MeshSceneNode { const color = this.getColorObjectForSegment(segmentId, layerName); @@ -172,7 +173,7 @@ export default class SegmentMeshController { tweenAnimation .to( { - opacity, + opacity: opacity ?? Constants.DEFAULT_MESH_OPACITY, }, 100, ) @@ -192,7 +193,7 @@ export default class SegmentMeshController { lod: number, layerName: string, additionalCoordinates: AdditionalCoordinate[] | null | undefined, - opacity: number, + opacity: number | undefined, isMerged: boolean, ): void { const additionalCoordinatesString = getAdditionalCoordinatesAsString(additionalCoordinates); diff --git a/frontend/javascripts/viewer/model/actions/annotation_actions.ts b/frontend/javascripts/viewer/model/actions/annotation_actions.ts index 78b6f15bd3..32dbf01b6b 100644 --- a/frontend/javascripts/viewer/model/actions/annotation_actions.ts +++ b/frontend/javascripts/viewer/model/actions/annotation_actions.ts @@ -334,7 +334,7 @@ export const addAdHocMeshAction = ( seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, mappingName: string | null | undefined, mappingType: MappingType | null | undefined, - opacity: number = Constants.DEFAULT_MESH_OPACITY, + opacity: number | undefined, ) => ({ type: "ADD_AD_HOC_MESH", @@ -344,7 +344,7 @@ export const addAdHocMeshAction = ( seedAdditionalCoordinates, mappingName, mappingType, - opacity, + opacity: opacity ?? Constants.DEFAULT_MESH_OPACITY, }) as const; export const addPrecomputedMeshAction = ( @@ -354,7 +354,7 @@ export const addPrecomputedMeshAction = ( seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, meshFileName: string, mappingName: string | null | undefined, - opacity: number, + opacity: number | undefined, ) => ({ type: "ADD_PRECOMPUTED_MESH", @@ -364,7 +364,7 @@ export const addPrecomputedMeshAction = ( seedAdditionalCoordinates, meshFileName, mappingName, - opacity, + opacity: opacity ?? Constants.DEFAULT_MESH_OPACITY, }) as const; export const setOthersMayEditForAnnotationAction = (othersMayEdit: boolean) => diff --git a/frontend/javascripts/viewer/model/actions/segmentation_actions.ts b/frontend/javascripts/viewer/model/actions/segmentation_actions.ts index a157b42277..8dcc6b2519 100644 --- a/frontend/javascripts/viewer/model/actions/segmentation_actions.ts +++ b/frontend/javascripts/viewer/model/actions/segmentation_actions.ts @@ -7,7 +7,7 @@ export type AdHocMeshInfo = { mappingType: MappingType | null | undefined; useDataStore?: boolean | null | undefined; preferredQuality?: number | null | undefined; - opacity?: number | null | undefined; + opacity?: number | undefined; }; export type LoadAdHocMeshAction = ReturnType; export type LoadPrecomputedMeshAction = ReturnType; @@ -35,7 +35,7 @@ export const loadPrecomputedMeshAction = ( seedPosition: Vector3, seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, meshFileName: string, - opacity: number, + opacity: number | undefined, layerName?: string | undefined, ) => ({ diff --git a/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts b/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts index b8f12ef8bf..40ae372612 100644 --- a/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts +++ b/frontend/javascripts/viewer/model/reducers/annotation_reducer.ts @@ -3,7 +3,6 @@ import { V3 } from "libs/mjs"; import * as Utils from "libs/utils"; import _ from "lodash"; import type { AdditionalCoordinate } from "types/api_types"; -import Constants from "viewer/constants"; import { maybeGetSomeTracing } from "viewer/model/accessors/tracing_accessor"; import { getDisplayedDataExtentInPlaneMode } from "viewer/model/accessors/view_mode_accessor"; import type { Action } from "viewer/model/actions/actions"; @@ -408,7 +407,7 @@ function AnnotationReducer(state: WebknossosState, action: Action): WebknossosSt isLoading: false, isVisible: true, isPrecomputed: true, - opacity: opacity || Constants.DEFAULT_MESH_OPACITY, + opacity, meshFileName, mappingName, }; diff --git a/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts index cebb491ca1..3e618ec83c 100644 --- a/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts +++ b/frontend/javascripts/viewer/model/sagas/meshes/ad_hoc_mesh_saga.ts @@ -13,7 +13,7 @@ import { actionChannel, call, put, race, take, takeEvery } from "typed-redux-sag import type { AdditionalCoordinate } from "types/api_types"; import { WkDevFlags } from "viewer/api/wk_dev"; import type { Vector3 } from "viewer/constants"; -import Constants, { MappingStatusEnum } from "viewer/constants"; +import { MappingStatusEnum } from "viewer/constants"; import { sortByDistanceTo } from "viewer/controller/mesh_helpers"; import getSceneController from "viewer/controller/scene_controller_provider"; import { @@ -308,7 +308,7 @@ function* loadFullAdHocMesh( additionalCoordinates, mappingName, mappingType, - opacity || Constants.DEFAULT_MESH_OPACITY, + opacity, ), ); yield* put(startedLoadingMeshAction(layer.name, segmentId)); @@ -483,8 +483,6 @@ function* maybeLoadMeshChunk( segmentMeshController.removeMeshById(segmentId, layer.name); } - const opacity = meshExtraInfo.opacity || Constants.DEFAULT_MESH_OPACITY; - // We await addMeshFromVerticesAsync here, because the mesh saga will remove // an ad-hoc loaded mesh immediately if it was "empty". Since the check is // done by looking at the scene, we await the population of the scene. @@ -496,7 +494,7 @@ function* maybeLoadMeshChunk( vertices, segmentId, layer.name, - opacity, + meshExtraInfo.opacity, additionalCoordinates, ); return neighbors.map((neighbor) => @@ -522,6 +520,7 @@ function* markEditedCellAsDirty(): Saga { } } +// This function is only called when the front-end api.data.refreshMeshes is used. function* refreshMeshes(): Saga { yield* put(saveNowAction()); // We reload all cells that got modified till the start of reloading. @@ -556,7 +555,7 @@ function* refreshMeshes(): Saga { threeDMap, segmentationLayer.name, additionalCoordinates, - meshInfo?.opacity || Constants.DEFAULT_MESH_OPACITY, + meshInfo?.opacity, ); } } @@ -611,7 +610,7 @@ function* refreshMeshWithMap( threeDMap: ThreeDMap, layerName: string, additionalCoordinates: AdditionalCoordinate[] | null, - opacity: number = Constants.DEFAULT_MESH_OPACITY, + opacity: number | undefined, ): Saga { const meshInfo = yield* select((state) => getMeshInfoForSegment(state, additionalCoordinates, layerName, segmentId), diff --git a/frontend/javascripts/viewer/model/sagas/meshes/precomputed_mesh_saga.ts b/frontend/javascripts/viewer/model/sagas/meshes/precomputed_mesh_saga.ts index ac8829701a..49b5b47943 100644 --- a/frontend/javascripts/viewer/model/sagas/meshes/precomputed_mesh_saga.ts +++ b/frontend/javascripts/viewer/model/sagas/meshes/precomputed_mesh_saga.ts @@ -14,7 +14,6 @@ import { actionChannel, call, put, race, take, takeEvery } from "typed-redux-sag import type { APIDataset, APIMeshFileInfo, APISegmentationLayer } from "types/api_types"; import type { AdditionalCoordinate } from "types/api_types"; import type { Vector3, Vector4 } from "viewer/constants"; -import Constants from "viewer/constants"; import CustomLOD from "viewer/controller/custom_lod"; import { type BufferGeometryWithInfo, @@ -131,7 +130,7 @@ function* loadPrecomputedMesh(action: LoadPrecomputedMeshAction) { seedAdditionalCoordinates, meshFileName, layer, - opacity || Constants.DEFAULT_MESH_OPACITY, + opacity, ), cancel: take( ((otherAction: Action) => @@ -150,7 +149,7 @@ function* loadPrecomputedMeshForSegmentId( seedAdditionalCoordinates: AdditionalCoordinate[] | undefined | null, meshFileName: string, segmentationLayer: APISegmentationLayer, - opacity: number, + opacity: number | undefined, ): Saga { const layerName = segmentationLayer.name; const mappingName = yield* call(getMappingName, segmentationLayer); @@ -339,7 +338,7 @@ function* loadPrecomputedMeshesInChunksForLod( getGlobalScale: (lod: number) => Vector3 | null, chunkScale: Vector3 | null, additionalCoordinates: AdditionalCoordinate[] | null, - opacity: number, + opacity: number | undefined, ) { const { segmentMeshController } = getSceneController(); const loader = getDracoLoader(); diff --git a/frontend/javascripts/viewer/model/sagas/proofread_saga.ts b/frontend/javascripts/viewer/model/sagas/proofread_saga.ts index 96190e4a2c..edd25a2780 100644 --- a/frontend/javascripts/viewer/model/sagas/proofread_saga.ts +++ b/frontend/javascripts/viewer/model/sagas/proofread_saga.ts @@ -11,7 +11,7 @@ import { SoftError, isBigInt, isNumberMap } from "libs/utils"; import _ from "lodash"; import { all, call, put, spawn, takeEvery } from "typed-redux-saga"; import type { AdditionalCoordinate, ServerEditableMapping } from "types/api_types"; -import Constants, { MappingStatusEnum, TreeTypeEnum, type Vector3 } from "viewer/constants"; +import { MappingStatusEnum, TreeTypeEnum, type Vector3 } from "viewer/constants"; import { getSegmentIdForPositionAsync } from "viewer/controller/combinations/volume_handlers"; import { getLayerByName, @@ -181,7 +181,7 @@ function* loadCoarseMesh( position, additionalCoordinates, currentMeshFile.name, - Constants.DEFAULT_MESH_OPACITY, + undefined, undefined, ), ); diff --git a/frontend/javascripts/viewer/model_initialization.ts b/frontend/javascripts/viewer/model_initialization.ts index 8aa7ae5952..67962ee328 100644 --- a/frontend/javascripts/viewer/model_initialization.ts +++ b/frontend/javascripts/viewer/model_initialization.ts @@ -34,7 +34,6 @@ import type { } from "types/api_types"; import type { Mutable } from "types/globals"; import constants, { ControlModeEnum, type Vector3 } from "viewer/constants"; -import Constants from "viewer/constants"; import type { DirectLayerSpecificProps, PartialUrlManagerState, @@ -891,7 +890,7 @@ async function applyLayerState(stateByLayer: UrlStateByLayer) { seedPosition, seedAdditionalCoordinates, meshFileName, - Constants.DEFAULT_MESH_OPACITY, + undefined, effectiveLayerName, ), ); diff --git a/frontend/javascripts/viewer/view/context_menu.tsx b/frontend/javascripts/viewer/view/context_menu.tsx index 824bfa1dce..8647780c34 100644 --- a/frontend/javascripts/viewer/view/context_menu.tsx +++ b/frontend/javascripts/viewer/view/context_menu.tsx @@ -36,7 +36,7 @@ import type { VoxelSize, } from "types/api_types"; import type { AdditionalCoordinate } from "types/api_types"; -import Constants, { +import { AltOrOptionKey, CtrlOrCmdKey, LongUnitToShortUnitMap, @@ -962,7 +962,7 @@ function getNoNodeContextMenuOptions(props: NoNodeContextMenuProps): ItemType[] globalPosition, additionalCoordinates, currentMeshFile.name, - Constants.DEFAULT_MESH_OPACITY, + undefined, undefined, ), ); diff --git a/frontend/javascripts/viewer/view/right-border-tabs/segments_tab/segments_view.tsx b/frontend/javascripts/viewer/view/right-border-tabs/segments_tab/segments_view.tsx index 82b1b4957d..41935a94ba 100644 --- a/frontend/javascripts/viewer/view/right-border-tabs/segments_tab/segments_view.tsx +++ b/frontend/javascripts/viewer/view/right-border-tabs/segments_tab/segments_view.tsx @@ -51,7 +51,7 @@ import type { Dispatch } from "redux"; import type { APIMeshFileInfo, MetadataEntryProto } from "types/api_types"; import { APIJobType, type AdditionalCoordinate } from "types/api_types"; import type { Vector3 } from "viewer/constants"; -import Constants, { EMPTY_OBJECT, MappingStatusEnum } from "viewer/constants"; +import { EMPTY_OBJECT, MappingStatusEnum } from "viewer/constants"; import { getMagInfoOfVisibleSegmentationLayer, getMappingInfo, @@ -247,7 +247,7 @@ const mapDispatchToProps = (dispatch: Dispatch) => ({ seedPosition, seedAdditionalCoordinates, meshFileName, - Constants.DEFAULT_MESH_OPACITY, + undefined, ), ); },