Exploring remote datasets creates virtual dataset

app/controllers/AnnotationIOController.scala

@@ -337,8 +337,7 @@ class AnnotationIOController @Inject()(
       else volumeTracing.boundingBox
 
     for {
-      tracingCanHaveSegmentIndex <- canHaveSegmentIndex(organizationId,
-                                                        dataset.name,
+      tracingCanHaveSegmentIndex <- canHaveSegmentIndex(dataset._id.toString,
                                                         fallbackLayerOpt.map(_.name),
                                                         remoteDataStoreClient)
       elementClassProto <- fallbackLayerOpt
@@ -358,13 +357,12 @@ class AnnotationIOController @Inject()(
   }
 
   private def canHaveSegmentIndex(
-      organizationId: String,
-      datasetName: String,
+      datasetId: String,
       fallbackLayerName: Option[String],
       remoteDataStoreClient: WKRemoteDataStoreClient)(implicit ec: ExecutionContext): Fox[Boolean] =
     fallbackLayerName match {
       case Some(layerName) =>
-        remoteDataStoreClient.hasSegmentIndexFile(organizationId, datasetName, layerName)
+        remoteDataStoreClient.hasSegmentIndexFile(datasetId, layerName)
       case None =>
         Fox.successful(true)
     }

app/controllers/DatasetController.scala

@@ -145,10 +145,10 @@ class DatasetController @Inject()(userService: UserService,
         _ <- Fox.fromBool(dataSource.dataLayers.nonEmpty) ?~> "dataset.explore.zeroLayers"
         folderIdOpt <- Fox.runOptional(request.body.folderPath)(folderPath =>
           folderService.getOrCreateFromPathLiteral(folderPath, request.identity._organization)) ?~> "dataset.explore.autoAdd.getFolder.failed"
-        _ <- wkExploreRemoteLayerService.addRemoteDatasource(dataSource,
-                                                             request.body.datasetName,
-                                                             request.identity,
-                                                             folderIdOpt) ?~> "dataset.explore.autoAdd.failed"
+        _ <- wkExploreRemoteLayerService.addRemoteDatasourceToDatabase(dataSource,
+                                                                       request.body.datasetName,
+                                                                       request.identity,
+                                                                       folderIdOpt) ?~> "dataset.explore.autoAdd.failed"
       } yield Ok
     }
 

app/controllers/WKRemoteDataStoreController.scala

@@ -9,7 +9,7 @@ import com.scalableminds.webknossos.datastore.helpers.{LayerMagLinkInfo, MagLink
 import com.scalableminds.webknossos.datastore.models.UnfinishedUpload
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
 import com.scalableminds.webknossos.datastore.models.datasource.inbox.{InboxDataSourceLike => InboxDataSource}
-import com.scalableminds.webknossos.datastore.services.{DataSourcePathInfo, DataStoreStatus}
+import com.scalableminds.webknossos.datastore.services.{DataSourcePathInfo, DataSourceRegistrationInfo, DataStoreStatus}
 import com.scalableminds.webknossos.datastore.services.uploading.{
   LinkedLayerIdentifier,
   ReserveAdditionalInformation,
@@ -270,6 +270,34 @@ class WKRemoteDataStoreController @Inject()(
 
     }
 
+  // Register a datasource from the datastore as a dataset in the database.
+  // This is called when adding remote virtual datasets (that should only exist in the database)
+  // by the data store after exploration.
+  def registerDataSource(name: String,
+                         key: String,
+                         organizationId: String,
+                         directoryName: String,
+                         token: String): Action[DataSourceRegistrationInfo] =
+    Action.async(validateJson[DataSourceRegistrationInfo]) { implicit request =>
+      dataStoreService.validateAccess(name, key) { dataStore =>
+        for {
+          user <- bearerTokenService.userForToken(token)
+          organization <- organizationDAO.findOne(organizationId)(GlobalAccessContext) ?~> Messages(
+            "organization.notFound",
+            organizationId) ~> NOT_FOUND
+          _ <- Fox.fromBool(organization._id == user._organization) ?~> "notAllowed" ~> FORBIDDEN
+          dataset <- datasetService.createVirtualDataset(
+            directoryName,
+            organizationId,
+            dataStore,
+            request.body.dataSource,
+            request.body.folderId,
+            user
+          )
+        } yield Ok(dataset._id.toString)
+      }
+    }
+
   def jobExportProperties(name: String, key: String, jobId: ObjectId): Action[AnyContent] = Action.async {
     implicit request =>
       dataStoreService.validateAccess(name, key) { _ =>

app/models/annotation/AnnotationService.scala

@@ -121,6 +121,7 @@ class AnnotationService @Inject()(
   private def createVolumeTracing(
       dataSource: DataSource,
       datasetOrganizationId: String,
+      datasetId: ObjectId,
       datasetDataStore: DataStore,
       fallbackLayer: Option[SegmentationLayer],
       boundingBox: Option[BoundingBox] = None,
@@ -138,7 +139,7 @@ class AnnotationService @Inject()(
       remoteDatastoreClient = new WKRemoteDataStoreClient(datasetDataStore, rpc)
       fallbackLayerHasSegmentIndex <- fallbackLayer match {
         case Some(layer) =>
-          remoteDatastoreClient.hasSegmentIndexFile(datasetOrganizationId, dataSource.id.directoryName, layer.name)
+          remoteDatastoreClient.hasSegmentIndexFile(datasetId.toString, layer.name)
         case None => Fox.successful(false)
       }
       elementClassProto <- ElementClass
@@ -237,6 +238,7 @@ class AnnotationService @Inject()(
             volumeTracing <- createVolumeTracing(
               dataSource,
               dataset._organization,
+              dataset._id,
               dataStore,
               fallbackLayer,
               magRestrictions = params.magRestrictions.getOrElse(MagRestrictions.empty),
@@ -429,6 +431,7 @@ class AnnotationService @Inject()(
       volumeTracing <- createVolumeTracing(
         dataSource,
         dataset._organization,
+        datasetId,
         dataStore,
         fallbackLayer = fallbackLayer,
         boundingBox = boundingBox.flatMap { box =>

app/models/dataset/Dataset.scala

@@ -762,8 +762,8 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte
       layer.magsOpt match {
         case Some(mags) =>
           mags.map(mag => {
-            q"""INSERT INTO webknossos.dataset_mags(_dataset, dataLayerName, mag, axisOrder, channelIndex, credentialId)
-                VALUES($datasetId, ${layer.name}, ${mag.mag}, ${mag.axisOrder
+            q"""INSERT INTO webknossos.dataset_mags(_dataset, dataLayerName, mag, path, axisOrder, channelIndex, credentialId)
+                VALUES($datasetId, ${layer.name}, ${mag.mag}, ${mag.path}, ${mag.axisOrder
               .map(Json.toJson(_))}, ${mag.channelIndex}, ${mag.credentialId})
            """.asUpdate
           })

app/models/dataset/DatasetService.scala

@@ -1,6 +1,6 @@
 package models.dataset
 
-import com.scalableminds.util.accesscontext.{DBAccessContext, GlobalAccessContext}
+import com.scalableminds.util.accesscontext.{AuthorizedAccessContext, DBAccessContext, GlobalAccessContext}
 import com.scalableminds.util.objectid.ObjectId
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
@@ -9,6 +9,8 @@ import com.scalableminds.webknossos.datastore.dataformats.layers.{
   N5SegmentationLayer,
   PrecomputedDataLayer,
   PrecomputedSegmentationLayer,
+  WKWDataLayer,
+  WKWSegmentationLayer,
   Zarr3DataLayer,
   Zarr3SegmentationLayer,
   ZarrDataLayer,
@@ -23,6 +25,7 @@ import com.scalableminds.webknossos.datastore.models.datasource.{
   AbstractDataLayer,
   AbstractSegmentationLayer,
   DataFormat,
+  DataSource,
   DataSourceId,
   GenericDataSource,
   DataLayerLike => DataLayer
@@ -50,7 +53,6 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
                                datasetLastUsedTimesDAO: DatasetLastUsedTimesDAO,
                                datasetDataLayerDAO: DatasetLayerDAO,
                                datasetMagsDAO: DatasetMagsDAO,
-                               datasetLayerAttachmentsDAO: DatasetLayerAttachmentsDAO,
                                teamDAO: TeamDAO,
                                folderDAO: FolderDAO,
                                dataStoreService: DataStoreService,
@@ -63,7 +65,9 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
     with LazyLogging {
   private val unreportedStatus = datasetDAO.unreportedStatus
   private val notYetUploadedStatus = "Not yet fully uploaded."
-  private val inactiveStatusList = List(unreportedStatus, notYetUploadedStatus, datasetDAO.deletedByUserStatus)
+  private val virtualRemoteDatasetStatus = "Virtual remote dataset" // Virtual datasets should not be deleted when not reported
+  private val inactiveStatusList =
+    List(unreportedStatus, notYetUploadedStatus, datasetDAO.deletedByUserStatus, virtualRemoteDatasetStatus)
 
   def assertValidDatasetName(name: String): Fox[Unit] =
     for {
@@ -97,6 +101,32 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
     } yield newDataset
   }
 
+  def createVirtualDataset(datasetName: String,
+                           organizationId: String,
+                           dataStore: DataStore,
+                           dataSource: DataSource,
+                           folderId: Option[String],
+                           user: User): Fox[Dataset] =
+    for {
+      _ <- assertValidDatasetName(datasetName)
+      isDatasetNameAlreadyTaken <- datasetDAO.doesDatasetDirectoryExistInOrganization(datasetName, organizationId)(
+        GlobalAccessContext)
+      _ <- Fox.fromBool(!isDatasetNameAlreadyTaken) ?~> "dataset.name.alreadyTaken"
+      organization <- organizationDAO.findOne(organizationId)(GlobalAccessContext) ?~> "organization.notFound"
+      folderId <- ObjectId.fromString(folderId.getOrElse(organization._rootFolder.toString)) ?~> "dataset.upload.folderId.invalid"
+      _ <- folderDAO.assertUpdateAccess(folderId)(AuthorizedAccessContext(user)) ?~> "folder.noWriteAccess"
+      newDatasetId = ObjectId.generate
+      abstractDataSource = dataSource.copy(dataLayers = dataSource.dataLayers.map(_.asAbstractLayer))
+      dataset <- createDataset(dataStore,
+                               newDatasetId,
+                               datasetName,
+                               abstractDataSource,
+                               status = Some(virtualRemoteDatasetStatus))
+      datasetId = dataset._id
+      _ <- datasetDAO.updateFolder(datasetId, folderId)(GlobalAccessContext)
+      _ <- addUploader(dataset, user._id)(GlobalAccessContext)
+    } yield dataset
+
   def getAllUnfinishedDatasetUploadsOfUser(userId: ObjectId, organizationId: String)(
       implicit ctx: DBAccessContext): Fox[List[DatasetCompactInfo]] =
     datasetDAO.findAllCompactWithSearch(
@@ -114,7 +144,8 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
       datasetId: ObjectId,
       datasetName: String,
       dataSource: InboxDataSource,
-      publication: Option[ObjectId] = None
+      publication: Option[ObjectId] = None,
+      status: Option[String] = None
   ): Fox[Dataset] = {
     implicit val ctx: DBAccessContext = GlobalAccessContext
     val metadata =
@@ -147,7 +178,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
         name = datasetName,
         voxelSize = dataSource.voxelSizeOpt,
         sharingToken = None,
-        status = dataSource.statusOpt.getOrElse(""),
+        status = status.orElse(dataSource.statusOpt).getOrElse(""),
         logoUrl = None,
         metadata = metadata
       )
@@ -331,8 +362,18 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
                 case Some(df) =>
                   df match {
                     case DataFormat.wkw =>
-                      throw new NotImplementedError(
-                        "WKW data format not supported in this context, only datasets with MagLocators are supported")
+                      WKWDataLayer(
+                        name,
+                        category,
+                        boundingBox,
+                        mags,
+                        elementClass,
+                        defaultViewConfiguration,
+                        adminViewConfiguration,
+                        coordinateTransformations,
+                        additionalAxes,
+                        attachmentsOpt
+                      )
                     case DataFormat.neuroglancerPrecomputed =>
                       PrecomputedDataLayer(
                         name,
@@ -413,8 +454,19 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
                 case Some(df) =>
                   df match {
                     case DataFormat.wkw =>
-                      throw new NotImplementedError(
-                        "WKW data format not supported in this context, only datasets with MagLocators are supported")
+                      WKWSegmentationLayer(
+                        name,
+                        boundingBox,
+                        mags,
+                        elementClass,
+                        mappings,
+                        largestSegmentId,
+                        defaultViewConfiguration,
+                        adminViewConfiguration,
+                        coordinateTransformations,
+                        additionalAxes,
+                        attachmentsOpt
+                      )
                     case DataFormat.neuroglancerPrecomputed =>
                       PrecomputedSegmentationLayer(
                         name,

app/models/dataset/WKRemoteDataStoreClient.scala

@@ -9,21 +9,19 @@ import com.scalableminds.webknossos.datastore.explore.{
   ExploreRemoteLayerParameters
 }
 import com.scalableminds.webknossos.datastore.models.{AdditionalCoordinate, RawCuboidRequest}
-import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, GenericDataSource}
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.datastore.services.DirectoryStorageReport
 import com.typesafe.scalalogging.LazyLogging
 import controllers.RpcTokenHolder
 import play.api.libs.json.JsObject
 import play.utils.UriEncoding
-import com.scalableminds.util.objectid.ObjectId
 
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration.DurationInt
 
 class WKRemoteDataStoreClient(dataStore: DataStore, rpc: RPC) extends LazyLogging {
 
-  private lazy val hasSegmentIndexFileCache: AlfuCache[(String, String, String), Boolean] =
+  private lazy val hasSegmentIndexFileCache: AlfuCache[(String, String), Boolean] =
     AlfuCache(timeToLive = 1 minute)
 
   def getDataLayerThumbnail(dataset: Dataset,
@@ -36,8 +34,7 @@ class WKRemoteDataStoreClient(dataStore: DataStore, rpc: RPC) extends LazyLoggin
     val targetMagBoundingBox = mag1BoundingBox / mag
     logger.debug(
       s"Thumbnail called for: ${dataset._id}, organization: ${dataset._organization}, directoryName: ${dataset.directoryName}, Layer: $dataLayerName")
-    rpc(
-      s"${dataStore.url}/data/datasets/${urlEncode(dataset._organization)}/${urlEncode(dataset.directoryName)}/layers/$dataLayerName/thumbnail.jpg")
+    rpc(s"${dataStore.url}/data/datasets/${dataset._id}/layers/$dataLayerName/thumbnail.jpg")
       .addQueryString("token" -> RpcTokenHolder.webknossosToken)
       .addQueryString("mag" -> mag.toMagLiteral())
       .addQueryString("x" -> mag1BoundingBox.topLeft.x.toString)
@@ -60,16 +57,15 @@ class WKRemoteDataStoreClient(dataStore: DataStore, rpc: RPC) extends LazyLoggin
                    additionalCoordinates: Option[Seq[AdditionalCoordinate]]): Fox[Array[Byte]] = {
     val targetMagBoundingBox = mag1BoundingBox / mag
     logger.debug(s"Fetching raw data. Mag $mag, mag1 bbox: $mag1BoundingBox, target-mag bbox: $targetMagBoundingBox")
-    rpc(
-      s"${dataStore.url}/data/datasets/${urlEncode(dataset._organization)}/${urlEncode(dataset.directoryName)}/layers/$layerName/readData")
+    rpc(s"${dataStore.url}/data/datasets/${dataset._id}/layers/$layerName/readData")
       .addQueryString("token" -> RpcTokenHolder.webknossosToken)
       .postJsonWithBytesResponse(
         RawCuboidRequest(mag1BoundingBox.topLeft, targetMagBoundingBox.size, mag, additionalCoordinates))
   }
 
   def findPositionWithData(dataset: Dataset, dataLayerName: String): Fox[JsObject] =
     rpc(
-      s"${dataStore.url}/data/datasets/${dataset._organization}/${dataset.directoryName}/layers/$dataLayerName/findData")
+      s"${dataStore.url}/data/datasets/${dataset._id}/layers/$dataLayerName/findData")
       .addQueryString("token" -> RpcTokenHolder.webknossosToken)
       .getWithJsonResponse[JsObject]
 
@@ -82,25 +78,13 @@ class WKRemoteDataStoreClient(dataStore: DataStore, rpc: RPC) extends LazyLoggin
       .silent
       .getWithJsonResponse[List[DirectoryStorageReport]]
 
-  def addDataSource(organizationId: String,
-                    datasetName: String,
-                    dataSource: GenericDataSource[DataLayer],
-                    folderId: Option[ObjectId],
-                    userToken: String): Fox[Unit] =
-    for {
-      _ <- rpc(s"${dataStore.url}/data/datasets/$organizationId/$datasetName")
-        .addQueryString("token" -> userToken)
-        .addQueryStringOptional("folderId", folderId.map(_.toString))
-        .postJson(dataSource)
-    } yield ()
-
-  def hasSegmentIndexFile(organizationId: String, datasetName: String, layerName: String)(
+  def hasSegmentIndexFile(datasetId: String, layerName: String)(
       implicit ec: ExecutionContext): Fox[Boolean] = {
-    val cacheKey = (organizationId, datasetName, layerName)
+    val cacheKey = (datasetId, layerName)
     hasSegmentIndexFileCache.getOrLoad(
       cacheKey,
       k =>
-        rpc(s"${dataStore.url}/data/datasets/${k._1}/${k._2}/layers/${k._3}/hasSegmentIndex")
+        rpc(s"${dataStore.url}/data/datasets/${k._1}/layers/${k._2}/hasSegmentIndex")
           .addQueryString("token" -> RpcTokenHolder.webknossosToken)
           .silent
           .getWithJsonResponse[Boolean]
@@ -117,7 +101,7 @@ class WKRemoteDataStoreClient(dataStore: DataStore, rpc: RPC) extends LazyLoggin
 
   def updateDatasetInDSCache(datasetId: String): Fox[Unit] =
     for {
-      _ <- rpc(s"${dataStore.url}/data/wkDatasets/$datasetId")
+      _ <- rpc(s"${dataStore.url}/data/datasets/$datasetId")
         .addQueryString("token" -> RpcTokenHolder.webknossosToken)
         .delete()
     } yield ()

app/models/dataset/explore/WKExploreRemoteLayerService.scala

@@ -105,17 +105,21 @@ class WKExploreRemoteLayerService @Inject()(credentialService: CredentialService
       credentialId <- Fox.runOptional(credentialOpt)(c => credentialService.insertOne(c)) ?~> "dataVault.credential.insert.failed"
     } yield credentialId
 
-  def addRemoteDatasource(dataSource: GenericDataSource[DataLayer],
-                          datasetName: String,
-                          user: User,
-                          folderId: Option[ObjectId])(implicit ctx: DBAccessContext): Fox[Unit] =
+  def addRemoteDatasourceToDatabase(dataSource: GenericDataSource[DataLayer],
+                                    datasetName: String,
+                                    user: User,
+                                    folderId: Option[ObjectId])(implicit ctx: DBAccessContext): Fox[Unit] =
     for {
-      organization <- organizationDAO.findOne(user._organization)
       dataStore <- dataStoreDAO.findOneWithUploadsAllowed
+      organizationId = user._organization
       _ <- datasetService.assertValidDatasetName(datasetName)
-      client = new WKRemoteDataStoreClient(dataStore, rpc)
-      userToken <- bearerTokenService.createAndInitDataStoreTokenForUser(user)
-      _ <- client.addDataSource(organization._id, datasetName, dataSource, folderId, userToken)
+      datasetId <- datasetService.createVirtualDataset(
+        dataSource.id.directoryName,
+        organizationId,
+        dataStore,
+        dataSource,
+        folderId.map(_.toString),
+        user
+      )
     } yield ()
-
 }