From 833b707c3f4259ec7f0e045de23cef809e614758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= <39529669+MichaelBuessemeyer@users.noreply.github.com> Date: Fri, 4 Jul 2025 19:24:24 +0200 Subject: [PATCH 1/3] WIP: truncate action log to max 1000 items per action group --- .../tracingstore/annotation/TSAnnotationService.scala | 8 ++++++-- .../tracingstore/controllers/TSAnnotationController.scala | 6 ++++-- webknossos-tracingstore/conf/tracingstore.latest.routes | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala index 13393d42ed9..25c9c787aee 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala @@ -60,6 +60,8 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss private lazy val materializedAnnotationWithTracingCache = AlfuCache[ObjectId, AlfuCache[Long, AnnotationWithTracings]](maxCapacity = 1000) + private val maxUpdateActionLogBatchSize = 1000 + private def newInnerCache(implicit ec: ExecutionContext): Fox[AlfuCache[Long, AnnotationWithTracings]] = Fox.successful(AlfuCache[Long, AnnotationWithTracings](maxCapacity = 2)) @@ -240,7 +242,7 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss else tracingDataStore.annotations.put(annotationId.toString, version, annotationProto) - def updateActionLog(annotationId: ObjectId, newestVersion: Long, oldestVersion: Long)( + def updateActionLog(annotationId: ObjectId, newestVersion: Long, oldestVersion: Long, truncate: Boolean)( implicit ec: ExecutionContext): Fox[JsValue] = { def versionedTupleToJson(tuple: (Long, List[UpdateAction])): JsObject = Json.obj( @@ -258,7 +260,9 @@ class TSAnnotationService @Inject()(val remoteWebknossosClient: TSRemoteWebknoss Some(batchTo), Some(batchFrom))(fromJsonBytes[List[UpdateAction]]) } - } yield Json.toJson(updateActionBatches.flatten.map(versionedTupleToJson)) + truncatedUpdateActionBatches = if (truncate) updateActionBatches.map(_.take(maxUpdateActionLogBatchSize)) + else updateActionBatches + } yield Json.toJson(truncatedUpdateActionBatches.flatten.map(versionedTupleToJson)) } def findEditableMappingInfo(annotationId: ObjectId, tracingId: String, version: Option[Long] = None)( diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala index 3f1a05cdeb0..faeb9173165 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala @@ -78,14 +78,16 @@ class TSAnnotationController @Inject()( def updateActionLog(annotationId: ObjectId, newestVersion: Option[Long] = None, - oldestVersion: Option[Long] = None): Action[AnyContent] = Action.async { implicit request => + oldestVersion: Option[Long] = None, + truncate: Option[Boolean] = None): Action[AnyContent] = Action.async { implicit request => log() { accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readAnnotation(annotationId)) { for { newestMaterializableVersion <- annotationService.currentMaterializableVersion(annotationId) updateLog <- annotationService.updateActionLog(annotationId, newestVersion.getOrElse(newestMaterializableVersion), - oldestVersion.getOrElse(0)) + oldestVersion.getOrElse(0), + truncate.getOrElse(false)) } yield Ok(updateLog) } } diff --git a/webknossos-tracingstore/conf/tracingstore.latest.routes b/webknossos-tracingstore/conf/tracingstore.latest.routes index 85a5a73b3f4..5b991d1414c 100644 --- a/webknossos-tracingstore/conf/tracingstore.latest.routes +++ b/webknossos-tracingstore/conf/tracingstore.latest.routes @@ -7,7 +7,7 @@ GET /health POST /annotation/save @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.save(annotationId: ObjectId) GET /annotation/:annotationId @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.get(annotationId: ObjectId, version: Option[Long]) POST /annotation/:annotationId/update @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.update(annotationId: ObjectId) -GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(annotationId: ObjectId, newestVersion: Option[Long], oldestVersion: Option[Long]) +GET /annotation/:annotationId/updateActionLog @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.updateActionLog(annotationId: ObjectId, newestVersion: Option[Long], oldestVersion: Option[Long], truncate: Option[Boolean]) GET /annotation/:annotationId/newestVersion @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.newestVersion(annotationId: ObjectId) POST /annotation/:annotationId/duplicate @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.duplicate(annotationId: ObjectId, newAnnotationId: ObjectId, ownerId: ObjectId, requestingUserId: ObjectId, version: Option[Long], isFromTask: Boolean, datasetBoundingBox: Option[String]) POST /annotation/:annotationId/resetToBase @com.scalableminds.webknossos.tracingstore.controllers.TSAnnotationController.resetToBase(annotationId: ObjectId) From 3b289d3a72894c0a455ba7c50394a95a65931e24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= <39529669+MichaelBuessemeyer@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:00:38 +0200 Subject: [PATCH 2/3] make frontend request action log only in truncated form --- frontend/javascripts/admin/rest_api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/javascripts/admin/rest_api.ts b/frontend/javascripts/admin/rest_api.ts index 396fe6a88cb..02dad4c9371 100644 --- a/frontend/javascripts/admin/rest_api.ts +++ b/frontend/javascripts/admin/rest_api.ts @@ -779,7 +779,7 @@ export function getUpdateActionLog( newestVersion?: number, ): Promise> { return doWithToken((token) => { - const params = new URLSearchParams(); + const params = new URLSearchParams([["truncate", "true"]]); params.set("token", token); if (oldestVersion != null) { params.set("oldestVersion", oldestVersion.toString()); From 3cd88f8d1f786cfd2582f84396f6f98aff1e8f84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20B=C3=BC=C3=9Femeyer?= <39529669+MichaelBuessemeyer@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:11:19 +0200 Subject: [PATCH 3/3] make fetching update action log per default not truncate but truncate in version restore view --- frontend/javascripts/admin/rest_api.ts | 3 ++- frontend/javascripts/viewer/view/version_list.tsx | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/javascripts/admin/rest_api.ts b/frontend/javascripts/admin/rest_api.ts index 02dad4c9371..5e07adcfe37 100644 --- a/frontend/javascripts/admin/rest_api.ts +++ b/frontend/javascripts/admin/rest_api.ts @@ -777,9 +777,10 @@ export function getUpdateActionLog( annotationId: string, oldestVersion?: number, newestVersion?: number, + truncateActionLog: boolean = false, ): Promise> { return doWithToken((token) => { - const params = new URLSearchParams([["truncate", "true"]]); + const params = new URLSearchParams([["truncate", truncateActionLog.toString()]]); params.set("token", token); if (oldestVersion != null) { params.set("oldestVersion", oldestVersion.toString()); diff --git a/frontend/javascripts/viewer/view/version_list.tsx b/frontend/javascripts/viewer/view/version_list.tsx index 21f319e8c30..205bfe61f97 100644 --- a/frontend/javascripts/viewer/view/version_list.tsx +++ b/frontend/javascripts/viewer/view/version_list.tsx @@ -168,6 +168,7 @@ async function getUpdateActionLogPage( annotationId, oldestVersionInPage, newestVersionInPage, + true, ); // The backend won't send the version 0 as that does not exist. The frontend however