Skip to content

Commit b9d05f3

Browse files
frcrothfrcroth
committed
In-Memory Cache
1 parent 63ddf7b commit b9d05f3

File tree

8 files changed

+34
-55
lines changed

8 files changed

+34
-55
lines changed

‎app/models/dataset/Dataset.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -858,7 +858,7 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte
858858
} yield
859859
MagLocator(
860860
mag,
861-
row.path, // Does this make sense -> mag path may be something different than path in DB :/
861+
row.path,
862862
None,
863863
axisOrderParsed,
864864
row.channelindex,

‎app/models/dataset/DatasetService.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -486,7 +486,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
486486
dataset <- datasetDAO.findOne(datasetId) ?~> "dataset.notFound"
487487
dataSource <- fullDataSourceFor(dataset)
488488
dataStoreClient <- clientFor(dataset)
489-
_ <- dataStoreClient.updateDatasetInDSCache(dataset._organization, dataset._id.toString, dataSource)
489+
_ <- dataStoreClient.updateDatasetInDSCache(dataset._id.toString, dataSource)
490490
} yield ()
491491

492492
private def logoUrlFor(dataset: Dataset, organization: Option[Organization]): Fox[String] =

‎app/models/dataset/WKRemoteDataStoreClient.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,9 +116,9 @@ class WKRemoteDataStoreClient(dataStore: DataStore, rpc: RPC) extends LazyLoggin
116116
.postJsonWithJsonResponse[ExploreRemoteDatasetRequest, ExploreRemoteDatasetResponse](
117117
ExploreRemoteDatasetRequest(layerParameters, organizationId))
118118

119-
def updateDatasetInDSCache(organizationId: String, datasetId: String, dataSource: InboxDataSourceLike): Fox[Unit] =
119+
def updateDatasetInDSCache(datasetId: String, dataSource: InboxDataSourceLike): Fox[Unit] =
120120
for {
121-
_ <- rpc(s"${dataStore.url}/data/datasets/$organizationId/byId/$datasetId")
121+
_ <- rpc(s"${dataStore.url}/data/wkDatasets/$datasetId")
122122
.addQueryString("token" -> RpcTokenHolder.webknossosToken)
123123
.putJson(dataSource)
124124
} yield ()

‎frontend/javascripts/viewer/model/bucket_data_handling/wkstore_adapter.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,8 @@ export async function requestFromStore(
227227
if (maybeAnnotationId != null) {
228228
params.append("annotationId", maybeAnnotationId);
229229
}
230+
// REMOVE ME
231+
dataUrl = "http://localhost:9000/data/wkDatasets/683d9643f600008d0e26a0a3/layers/layer424242"
230232
const { buffer: responseBuffer, headers } =
231233
await Request.sendJSONReceiveArraybufferWithHeaders(`${dataUrl}/data?${params}`, {
232234
data: bucketInfo,

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/BinaryDataController.scala

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -309,16 +309,13 @@ class BinaryDataController @Inject()(
309309
}
310310
}
311311

312-
// TODO: ObjectId params
313-
def requestViaWebknossosById(organizationId: String,
314-
datasetId: String,
315-
dataLayerName: String): Action[List[WebknossosDataRequest]] =
312+
def requestViaWebknossosById(datasetId: String, dataLayerName: String): Action[List[WebknossosDataRequest]] =
316313
Action.async(validateJson[List[WebknossosDataRequest]]) { implicit request =>
317314
accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataset(datasetId)) {
318315
// TODO Log time
319316
for {
320317
datasetId <- ObjectId.fromString(datasetId)
321-
dataSource <- datasetCache.getById(organizationId, datasetId)
318+
dataSource <- datasetCache.getById(datasetId)
322319
dataLayer <- dataSource.getDataLayer(dataLayerName).toFox ?~> "Data layer not found" ~> NOT_FOUND
323320
(data, indices) <- requestData(dataSource.id, dataLayer, request.body)
324321
} yield Ok(data).withHeaders(createMissingBucketsHeaders(indices): _*)

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -775,12 +775,11 @@ class DataSourceController @Inject()(
775775
}
776776
}
777777

778-
def updateCache(organizationId: String, datasetId: String): Action[DataSource] =
778+
def updateCache(datasetId: String): Action[DataSource] =
779779
Action.async(validateJson[DataSource]) { implicit request =>
780780
accessTokenService.validateAccessFromTokenContext(UserAccessRequest.writeDataset(datasetId)) {
781-
for {
782-
_ <- datasetCache.updateById(organizationId, datasetId, request.body)
783-
} yield Ok
781+
datasetCache.updateById(datasetId, request.body)
782+
Future.successful(Ok)
784783
}
785784
}
786785

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DatasetCache.scala

Lines changed: 19 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,32 @@
11
package com.scalableminds.webknossos.datastore.services
22

3-
import com.scalableminds.util.io.PathUtils
3+
import com.google.inject.name.Named
44
import com.scalableminds.util.objectid.ObjectId
5-
import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper}
6-
import com.scalableminds.webknossos.datastore.DataStoreConfig
7-
import com.scalableminds.webknossos.datastore.models.datasource.{DataSource, GenericDataSource}
8-
import net.liftweb.common.Full
5+
import com.scalableminds.util.tools.{Fox, FoxImplicits}
6+
import com.scalableminds.webknossos.datastore.models.datasource.DataSource
7+
import com.scalableminds.webknossos.datastore.storage.TemporaryStore
8+
import org.apache.pekko.actor.ActorSystem
99

10-
import java.nio.file.{Path, Paths}
1110
import javax.inject.Inject
1211
import scala.concurrent.ExecutionContext
1312

14-
class DatasetCache @Inject()(remoteWebknossosClient: DSRemoteWebknossosClient, config: DataStoreConfig)(
15-
implicit ec: ExecutionContext)
16-
extends PathUtils
17-
with FoxImplicits {
13+
class DatasetCache @Inject()(remoteWebknossosClient: DSRemoteWebknossosClient,
14+
@Named("webknossos-datastore") val actorSystem: ActorSystem)(implicit ec: ExecutionContext)
15+
extends FoxImplicits {
1816

19-
val datasetCacheDir = ".datasetCache"
20-
val dataBaseDir: Path = Paths.get(config.Datastore.baseDirectory)
17+
val cache = new TemporaryStore[ObjectId, DataSource](actorSystem)
2118

22-
def getById(organizationId: String, id: ObjectId): Fox[DataSource] = {
23-
val cachePath = dataBaseDir.resolve(organizationId).resolve(datasetCacheDir).resolve(id.toString)
24-
val cacheFile = cachePath.resolve(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON)
25-
if (cacheFile.toFile.exists()) {
26-
JsonHelper.parseFromFileAs[DataSource](cacheFile, cachePath) match {
27-
case Full(dataSource) =>
28-
if (dataSource.dataLayers.nonEmpty) Fox.successful(dataSource)
29-
else {
30-
// TODO: Handle unhappy cases
31-
???
32-
}
33-
case e => ???
34-
}
35-
} else {
36-
// Request dataset from remote webknossos
37-
for {
38-
dataSource <- remoteWebknossosClient.getDataset(id.toString)
39-
_ = PathUtils.ensureDirectory(cacheFile.getParent)
40-
_ <- JsonHelper.writeToFile(cacheFile, dataSource).toFox
41-
} yield dataSource
19+
def getById(id: ObjectId): Fox[DataSource] =
20+
cache.get(id) match {
21+
case Some(dataSource) => Fox.successful(dataSource)
22+
case None =>
23+
for {
24+
dataSource <- remoteWebknossosClient.getDataset(id.toString)
25+
_ = cache.insert(id, dataSource)
26+
} yield dataSource
4227
}
43-
}
4428

45-
def updateById(organizationId: String, datasetId: String, dataSource: DataSource): Fox[Unit] = {
46-
val cachePath = dataBaseDir.resolve(organizationId).resolve(datasetCacheDir).resolve(datasetId)
47-
val cacheFile = cachePath.resolve(GenericDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON)
48-
PathUtils.ensureDirectory(cacheFile.getParent)
49-
JsonHelper.writeToFile(cacheFile, dataSource).toFox
50-
}
29+
def updateById(datasetId: String, dataSource: DataSource): Unit =
30+
cache.insert(ObjectId(datasetId), dataSource)
5131

5232
}

‎webknossos-datastore/conf/com.scalableminds.webknossos.datastore.routes

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ GET /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerN
1111
GET /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/thumbnail.jpg @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.thumbnailJpeg(organizationId: String, datasetDirectoryName: String, dataLayerName: String, x: Int, y: Int, z: Int, width: Int, height: Int, mag: String, mappingName: Option[String], intensityMin: Option[Double], intensityMax: Option[Double], color: Option[String], invertColor: Option[Boolean])
1212
GET /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/findData @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.findData(organizationId: String, datasetDirectoryName: String, dataLayerName: String)
1313
GET /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/histogram @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.histogram(organizationId: String, datasetDirectoryName: String, dataLayerName: String)
14-
# TODO: Route name is not nice (also disallow "byId" dataset name)
15-
POST /datasets/:organizationId/byId/:datasetId/layers/:dataLayerName/data @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaWebknossosById(organizationId: String, datasetId: String, dataLayerName: String)
14+
15+
POST /wkDatasets/:datasetId/layers/:dataLayerName/data @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaWebknossosById(datasetId: String, dataLayerName: String)
1616

1717
# Knossos compatible routes
1818
GET /datasets/:organizationId/:datasetDirectoryName/layers/:dataLayerName/mag:mag/x:x/y:y/z:z/bucket.raw @com.scalableminds.webknossos.datastore.controllers.BinaryDataController.requestViaKnossos(organizationId: String, datasetDirectoryName: String, dataLayerName: String, mag: Int, x: Int, y: Int, z: Int, cubeSize: Int)
@@ -118,7 +118,8 @@ POST /datasets/:organizationId/:datasetName
118118
DELETE /datasets/:organizationId/:datasetDirectoryName/deleteOnDisk @com.scalableminds.webknossos.datastore.controllers.DataSourceController.deleteOnDisk(organizationId: String, datasetDirectoryName: String)
119119
POST /datasets/compose @com.scalableminds.webknossos.datastore.controllers.DataSourceController.compose()
120120
POST /datasets/exploreRemote @com.scalableminds.webknossos.datastore.controllers.DataSourceController.exploreRemoteDataset()
121-
PUT /datasets/:organizationId/byId/:datasetId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.updateCache(organizationId: String, datasetId: String)
121+
122+
PUT /wkDatasets/:datasetId @com.scalableminds.webknossos.datastore.controllers.DataSourceController.updateCache(datasetId: String)
122123

123124
# Actions
124125
POST /triggers/checkInboxBlocking @com.scalableminds.webknossos.datastore.controllers.DataSourceController.triggerInboxCheckBlocking()

0 commit comments

Comments
 (0)