Add HTTP API versioning for DataStore and TracingStore (#8705)

fm3 · web-flow · commit bd505f0ceaf7 · 2025-06-25T10:51:13.000+02:00
Introducing versioning for the HTTP APIs of datastore and tracingstore, analogous to the existing one on the wk side. That means that e.g. http://localhost:9000/data/v9/health should now reply the same as http://localhost:9000/data/health – and we now have the option to support legacy variants of datastore and tracingstore routes (for a limited time). As discussed in [slack](https://scm.slack.com/archives/C5AKLAV0B/p1750402140167989) this is now designed such that the versions of wk, datastore and tracingstore have to be identical. That means than when we introduce a new version in one of them, we have to create version redirects for the other two as well. I tried to unify the formatting of the routes files a little and moved the collections package in util to unify package structure there. ### URL of deployed dev instance (used for testing): - https://dstsversioning.webknossos.xyz ### Steps to test: - Trace some, should work normally - Try some routes with explicit version. Should work as expected (5 to 9 should be equal to latest, newer and older than that should be rejected) ### Issues: - fixes #8235 - Corresponding libs issue (blocked by this PR): scalableminds/webknossos-libs#1327 ------ - [x] Added changelog entry (create a `$PR_NUMBER.md` file in `unreleased_changes` or use `./tools/create-changelog-entry.py`) - [x] Considered [common edge cases](../blob/master/.github/common_edge_cases.md) - [x] Needs datastore update after deployment
diff --git a/app/RequestHandler.scala b/app/RequestHandler.scala
@@ -1,4 +1,4 @@
-import com.scalableminds.util.mvc.{CspHeaders, ExtendedController}
+import com.scalableminds.util.mvc.{ApiVersioning, CspHeaders, ExtendedController}
 import com.typesafe.scalalogging.LazyLogging
 import controllers.{Assets, SitemapController, AboutPageRedirectController}
 
@@ -9,7 +9,7 @@ import play.api.mvc.{Handler, InjectedController, RequestHeader}
 import play.api.routing.Router
 import play.core.WebCommands
 import play.filters.csp.CSPConfig
-import utils.{ApiVersioning, WkConf}
+import utils.WkConf
 
 import scala.concurrent.ExecutionContext
 
diff --git a/app/controllers/AnalyticsController.scala b/app/controllers/AnalyticsController.scala
@@ -6,16 +6,15 @@ import play.api.libs.json.JsObject
 import play.api.mvc.{Action, PlayBodyParsers}
 import play.silhouette.api.Silhouette
 import security.WkEnv
-import utils.{ApiVersioning, WkConf}
+import utils.WkConf
 
 import javax.inject.Inject
 import scala.concurrent.ExecutionContext
 
 class AnalyticsController @Inject()(analyticsService: AnalyticsService, conf: WkConf, sil: Silhouette[WkEnv])(
     implicit ec: ExecutionContext,
     bodyParsers: PlayBodyParsers)
-    extends Controller
-    with ApiVersioning {
+    extends Controller {
 
   def ingestAnalyticsEvents: Action[AnalyticsEventsIngestJson] = Action.async(validateJson[AnalyticsEventsIngestJson]) {
     implicit request =>
diff --git a/app/controllers/AnnotationIOController.scala b/app/controllers/AnnotationIOController.scala
@@ -1,7 +1,7 @@
 package controllers
 
-import collections.SequenceUtils
 import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
+import com.scalableminds.util.collections.SequenceUtils
 import com.scalableminds.util.io.ZipIO
 import com.scalableminds.util.objectid.ObjectId
 import com.scalableminds.util.tools.{Fox, FoxImplicits, TextUtils}
diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala
@@ -1,5 +1,6 @@
 package controllers
 
+import com.scalableminds.util.mvc.ApiVersioning
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.typesafe.config.ConfigRenderOptions
 import mail.{DefaultMails, Send}
@@ -11,7 +12,7 @@ import play.api.mvc.{Action, AnyContent, Result}
 import play.silhouette.api.Silhouette
 import security.{CertificateValidationService, WkEnv}
 import utils.sql.{SimpleSQLDAO, SqlClient}
-import utils.{ApiVersioning, StoreModules, WkConf}
+import utils.{StoreModules, WkConf}
 
 import javax.inject.Inject
 import scala.concurrent.ExecutionContext
@@ -42,10 +43,7 @@ class Application @Inject()(actorSystem: ActorSystem,
             "webknossos" -> Json.toJson(
               webknossos.BuildInfo.toMap.view.mapValues(_.toString).filterKeys(_ != "certificatePublicKey").toMap),
             "schemaVersion" -> schemaVersion.toOption,
-            "httpApiVersioning" -> Json.obj(
-              "currentApiVersion" -> CURRENT_API_VERSION,
-              "oldestSupportedApiVersion" -> OLDEST_SUPPORTED_API_VERSION
-            ),
+            "httpApiVersioning" -> apiVersioningInfo,
             "localDataStoreEnabled" -> storeModules.localDataStoreEnabled,
             "localTracingStoreEnabled" -> storeModules.localTracingStoreEnabled
           ))
diff --git a/app/controllers/TaskController.scala b/app/controllers/TaskController.scala
@@ -1,10 +1,9 @@
 package controllers
 
-import collections.SequenceUtils
-
 import java.io.File
 import play.silhouette.api.Silhouette
 import com.scalableminds.util.accesscontext.GlobalAccessContext
+import com.scalableminds.util.collections.SequenceUtils
 import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper}
 import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing
 import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
diff --git a/app/models/dataset/explore/WKExploreRemoteLayerService.scala b/app/models/dataset/explore/WKExploreRemoteLayerService.scala
@@ -1,7 +1,7 @@
 package models.dataset.explore
 
-import collections.SequenceUtils
 import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
+import com.scalableminds.util.collections.SequenceUtils
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.explore.{
   ExploreLayerUtils,
diff --git a/app/models/task/TaskCreationService.scala b/app/models/task/TaskCreationService.scala
@@ -1,9 +1,8 @@
 package models.task
 
-import collections.SequenceUtils
-
 import java.io.File
 import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
+import com.scalableminds.util.collections.SequenceUtils
 import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int}
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.SkeletonTracing.{
diff --git a/conf/noDS.routes b/conf/noDS.routes
@@ -1,4 +1,4 @@
--> /api/                         webknossos.versioned.Routes
--> /tracings/                    com.scalableminds.webknossos.tracingstore.Routes
+->   /api/                         webknossos.versioned.Routes
+->   /tracings/                    tracingstore.versioned.Routes
 
-GET   /.well-known/security.txt  controllers.Application.getSecurityTxt
+GET  /.well-known/security.txt     controllers.Application.getSecurityTxt
diff --git a/conf/noTS.routes b/conf/noTS.routes
@@ -1,4 +1,4 @@
--> /api/                         webknossos.versioned.Routes
--> /data/                        com.scalableminds.webknossos.datastore.Routes
+->   /api/                         webknossos.versioned.Routes
+->   /data/                        datastore.versioned.Routes
 
-GET   /.well-known/security.txt  controllers.Application.getSecurityTxt
+GET  /.well-known/security.txt     controllers.Application.getSecurityTxt
diff --git a/conf/routes b/conf/routes
@@ -1,5 +1,5 @@
--> /api/                         webknossos.versioned.Routes
--> /data/                        com.scalableminds.webknossos.datastore.Routes
--> /tracings/                    com.scalableminds.webknossos.tracingstore.Routes
+->   /api/                         webknossos.versioned.Routes
+->   /data/                        datastore.versioned.Routes
+->   /tracings/                    tracingstore.versioned.Routes
 
-GET   /.well-known/security.txt  controllers.Application.getSecurityTxt
+GET  /.well-known/security.txt     controllers.Application.getSecurityTxt
diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes
@@ -1,6 +1,5 @@
-# Routes
-# This file defines all application routes (Higher priority routes first)
-# ~~~~
+# Defines latest version of webknossos backend routes (Higher priority routes first)
+
 GET           /buildinfo                                                                        controllers.Application.buildInfo()
 GET           /features                                                                         controllers.Application.features()
 GET           /health                                                                           controllers.Application.health()
diff --git a/conf/webknossos.versioned.routes b/conf/webknossos.versioned.routes
@@ -1,7 +1,7 @@
 # API versioning is handled here. Higher-Priority routes first
 
 # example: assume, the features route has changed, introducing v2. The older v1 needs to be provided in the legacyApiController
-# Note: keep this in sync with the reported version numbers in the utils.ApiVersioning trait
+# Note: keep this in sync with the reported version numbers in the com.scalableminds.util.mvc.ApiVersioning trait
 
 # version log:updateDatasetV8
  # changed in v9: Datasets are now identified by their id, not their name. The routes now need to pass a dataset id instead of a name and organization id tuple.
diff --git a/unreleased_changes/8705.md b/unreleased_changes/8705.md
@@ -0,0 +1,2 @@
+### Changed
+- The HTTP API of tracing store and data store endpoints now also support versioning for better compatibility across webknossos-libs python clients.
diff --git a/util/src/main/scala/com/scalableminds/util/collections/SequenceUtils.scala b/util/src/main/scala/com/scalableminds/util/collections/SequenceUtils.scala
@@ -1,4 +1,4 @@
-package collections
+package com.scalableminds.util.collections
 
 object SequenceUtils {
   def findUniqueElement[T](list: Seq[T]): Option[T] = {
diff --git a/util/src/main/scala/com/scalableminds/util/mvc/ApiVersioning.scala b/util/src/main/scala/com/scalableminds/util/mvc/ApiVersioning.scala
@@ -1,12 +1,18 @@
-package utils
+package com.scalableminds.util.mvc
 
+import play.api.libs.json.{JsObject, Json}
 import play.api.mvc.RequestHeader
 
 trait ApiVersioning {
 
-  protected def CURRENT_API_VERSION: Int = 9
+  private lazy val CURRENT_API_VERSION: Int = 9
+  private lazy val OLDEST_SUPPORTED_API_VERSION: Int = 5
 
-  protected def OLDEST_SUPPORTED_API_VERSION: Int = 5
+  protected lazy val apiVersioningInfo: JsObject =
+    Json.obj(
+      "currentApiVersion" -> CURRENT_API_VERSION,
+      "oldestSupportedApiVersion" -> OLDEST_SUPPORTED_API_VERSION
+    )
 
   protected def isInvalidApiVersion(request: RequestHeader): Boolean = {
     val requestedVersion = extractRequestedApiVersion(request)
@@ -19,9 +25,9 @@ trait ApiVersioning {
   }
 
   private def extractRequestedApiVersion(request: RequestHeader): Int =
-    "^/api/v(\\d+).*$".r.findFirstMatchIn(request.uri) match {
+    "^/(api|data|tracings)/v(\\d+).*$".r.findFirstMatchIn(request.uri) match {
       case Some(m) =>
-        m.group(1).toInt
+        m.group(2).toInt
       case None => CURRENT_API_VERSION
     }
 }
diff --git a/webknossos-datastore/app/DsRequestHandler.scala b/webknossos-datastore/app/DsRequestHandler.scala
@@ -1,3 +1,4 @@
+import com.scalableminds.util.mvc.{ApiVersioning, ExtendedController}
 import com.scalableminds.webknossos.datastore.DataStoreConfig
 import com.typesafe.scalalogging.LazyLogging
 import play.api.OptionalDevContext
@@ -34,20 +35,25 @@ class DsRequestHandler @Inject()(webCommands: WebCommands,
                                       configuration,
                                       filters)
     with InjectedController
+    with ExtendedController
     with AdditionalHeaders
-    with LazyLogging {
+    with LazyLogging
+    with ApiVersioning {
+
   override def routeRequest(request: RequestHeader): Option[Handler] =
     if (request.method == "OPTIONS") {
       Some(Action { options(request) })
+    } else if (request.path == "/" || request.path == "/index.html") {
+      Some(Action {
+        Ok(
+          views.html
+            .datastoreFrontpage("Datastore", conf.Datastore.name, conf.Datastore.WebKnossos.uri, "/data/health"))
+      })
+    } else if (isInvalidApiVersion(request)) {
+      Some(Action {
+        JsonNotFound(invalidApiVersionMessage(request))
+      })
     } else {
-      if (request.path == "/" || request.path == "/index.html") {
-        Some(Action {
-          Ok(
-            views.html
-              .datastoreFrontpage("Datastore", conf.Datastore.name, conf.Datastore.WebKnossos.uri, "/data/health"))
-        })
-      } else {
-        super.routeRequest(request)
-      }
+      super.routeRequest(request)
     }
 }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/StandaloneDatastore.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/StandaloneDatastore.scala
@@ -1,18 +1,20 @@
 package com.scalableminds.webknossos.datastore.controllers
 
+import com.scalableminds.util.mvc.ApiVersioning
 import play.api.libs.json.Json
 import play.api.mvc.{Action, AnyContent}
 
 import javax.inject.Inject
 
-class StandaloneDatastore @Inject()() extends Controller {
+class StandaloneDatastore @Inject()() extends Controller with ApiVersioning {
 
   def buildInfo: Action[AnyContent] = Action {
     addNoCacheHeaderFallback(
       addRemoteOriginHeaders(
         Ok(
           Json.obj(
-            "webknossosDatastore" -> Json.toJson(webknossosDatastore.BuildInfo.toMap.view.mapValues(_.toString).toMap)
+            "webknossosDatastore" -> Json.toJson(webknossosDatastore.BuildInfo.toMap.view.mapValues(_.toString).toMap),
+            "httpApiVersioning" -> apiVersioningInfo,
           ))
       ))
   }
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/RemoteLayerExplorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/RemoteLayerExplorer.scala
@@ -1,8 +1,8 @@
 package com.scalableminds.webknossos.datastore.explore
 
 import com.scalableminds.util.geometry.BoundingBox
-import collections.SequenceUtils
 import com.scalableminds.util.accesscontext.TokenContext
+import com.scalableminds.util.collections.SequenceUtils
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.scalableminds.webknossos.datastore.dataformats.MagLocator
 import com.scalableminds.webknossos.datastore.datavault.VaultPath
diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala
@@ -1,8 +1,8 @@
 package com.scalableminds.webknossos.datastore.services
 
-import collections.SequenceUtils
 import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.cache.AlfuCache
+import com.scalableminds.util.collections.SequenceUtils
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.util.tools.ExtendedTypes.ExtendedArraySeq
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
diff --git a/webknossos-datastore/conf/datastore.latest.routes b/webknossos-datastore/conf/datastore.latest.routes
@@ -1,5 +1,4 @@
-# Datastore Routes (Higher priority routes first)
-
+# Defines latest version of datastore routes (Higher priority routes first)
 
 # Health endpoint
 GET           /health                                                                                                                                 @com.scalableminds.webknossos.datastore.controllers.Application.health
diff --git a/webknossos-datastore/conf/datastore.versioned.routes b/webknossos-datastore/conf/datastore.versioned.routes
@@ -0,0 +1,9 @@
+# Note: keep this in sync with the reported version numbers in the com.scalableminds.util.mvc.ApiVersioning trait
+
+->  /v9/  datastore.latest.Routes
+->  /v8/  datastore.latest.Routes
+->  /v7/  datastore.latest.Routes
+->  /v6/  datastore.latest.Routes
+->  /v5/  datastore.latest.Routes
+
+->  /     datastore.latest.Routes
diff --git a/webknossos-datastore/conf/routes b/webknossos-datastore/conf/routes
@@ -1,7 +1,3 @@
-# Routes
-# This file defines all application routes (Higher priority routes first)
-# ~~~~
+->    /data/          datastore.versioned.Routes
 
-GET   /api/buildinfo com.scalableminds.webknossos.datastore.controllers.StandaloneDatastore.buildInfo
-
-->    /data/         com.scalableminds.webknossos.datastore.Routes
+GET   /api/buildinfo  com.scalableminds.webknossos.datastore.controllers.StandaloneDatastore.buildInfo
diff --git a/webknossos-tracingstore/app/TsRequestHandler.scala b/webknossos-tracingstore/app/TsRequestHandler.scala
@@ -1,3 +1,4 @@
+import com.scalableminds.util.mvc.{ApiVersioning, ExtendedController}
 import com.scalableminds.webknossos.tracingstore.TracingStoreConfig
 import com.typesafe.scalalogging.LazyLogging
 import play.api.OptionalDevContext
@@ -22,22 +23,26 @@ class TsRequestHandler @Inject()(webCommands: WebCommands,
                                       configuration,
                                       filters)
     with InjectedController
+    with ExtendedController
     with AdditionalHeaders
-    with LazyLogging {
+    with LazyLogging
+    with ApiVersioning {
   override def routeRequest(request: RequestHeader): Option[Handler] =
     if (request.method == "OPTIONS") {
       Some(Action { options(request) })
+    } else if (request.path == "/" || request.path == "/index.html") {
+      Some(Action {
+        Ok(
+          views.html.datastoreFrontpage("Tracingstore",
+                                        conf.Tracingstore.name,
+                                        conf.Tracingstore.WebKnossos.uri,
+                                        "/tracings/health"))
+      })
+    } else if (isInvalidApiVersion(request)) {
+      Some(Action {
+        JsonNotFound(invalidApiVersionMessage(request))
+      })
     } else {
-      if (request.path == "/" || request.path == "/index.html") {
-        Some(Action {
-          Ok(
-            views.html.datastoreFrontpage("Tracingstore",
-                                          conf.Tracingstore.name,
-                                          conf.Tracingstore.WebKnossos.uri,
-                                          "/tracings/health"))
-        })
-      } else {
-        super.routeRequest(request)
-      }
+      super.routeRequest(request)
     }
 }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/TSAnnotationService.scala
@@ -1,8 +1,8 @@
 package com.scalableminds.webknossos.tracingstore.annotation
 
-import collections.SequenceUtils
 import com.scalableminds.util.accesscontext.TokenContext
 import com.scalableminds.util.cache.AlfuCache
+import com.scalableminds.util.collections.SequenceUtils
 import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int}
 import com.scalableminds.util.objectid.ObjectId
 import com.scalableminds.util.time.Instant
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/annotation/UpdateGroupHandling.scala
@@ -1,6 +1,6 @@
 package com.scalableminds.webknossos.tracingstore.annotation
 
-import collections.SequenceUtils
+import com.scalableminds.util.collections.SequenceUtils
 import com.typesafe.scalalogging.LazyLogging
 import com.scalableminds.util.tools.{Box, Full}
 
@@ -19,7 +19,7 @@ trait UpdateGroupHandling extends LazyLogging {
       updateActionGroupsWithVersions: List[(Long, List[UpdateAction])]): Box[List[(Long, List[UpdateAction])]] =
     for {
       groupVersions <- Full(updateActionGroupsWithVersions.map(_._1))
-      _ <- Box.fromBool(groupVersions.sorted((Ordering[Long].reverse)) == groupVersions) ?~! "updateGroupVersions.notSortedDesc"
+      _ <- Box.fromBool(groupVersions.sorted(Ordering[Long].reverse) == groupVersions) ?~! "updateGroupVersions.notSortedDesc"
       splitGroupLists: List[List[(Long, List[UpdateAction])]] = SequenceUtils.splitAndIsolate(
         updateActionGroupsWithVersions.reverse)(actionGroup =>
         actionGroup._2.exists(updateAction => isIsolationSensitiveAction(updateAction)))
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/StandaloneTracingstore.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/StandaloneTracingstore.scala
@@ -1,20 +1,22 @@
 package com.scalableminds.webknossos.tracingstore.controllers
 
+import com.scalableminds.util.mvc.ApiVersioning
 import com.scalableminds.webknossos.datastore.controllers.Controller
 import play.api.libs.json.Json
 import play.api.mvc.{Action, AnyContent}
 
 import javax.inject.Inject
 
-class StandaloneTracingstore @Inject()() extends Controller {
+class StandaloneTracingstore @Inject()() extends Controller with ApiVersioning {
 
   def buildInfo: Action[AnyContent] = Action {
     addNoCacheHeaderFallback(
       addRemoteOriginHeaders(
         Ok(
           Json.obj(
             "webknossosTracingstore" -> Json.toJson(
-              webknossosTracingstore.BuildInfo.toMap.view.mapValues(_.toString).toMap)
+              webknossosTracingstore.BuildInfo.toMap.view.mapValues(_.toString).toMap),
+            "httpApiVersioning" -> apiVersioningInfo,
           )
         )))
   }
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/TSAnnotationController.scala
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/controllers/VolumeTracingController.scala
diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingMergeService.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingMergeService.scala
diff --git a/webknossos-tracingstore/conf/routes b/webknossos-tracingstore/conf/routes
diff --git a/webknossos-tracingstore/conf/tracingstore.latest.routes b/webknossos-tracingstore/conf/tracingstore.latest.routes
diff --git a/webknossos-tracingstore/conf/tracingstore.versioned.routes b/webknossos-tracingstore/conf/tracingstore.versioned.routes
