Skip to content

Commit 11bcc9c

Browse files
frcrothfrcroth
committed
Load data via dataset ID
1 parent a535d3b commit 11bcc9c

File tree

11 files changed

+344
-6
lines changed

11 files changed

+344
-6
lines changed

‎app/controllers/UserTokenController.scala

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,9 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO,
9696
answer <- accessRequest.resourceType match {
9797
case AccessResourceType.datasource =>
9898
handleDataSourceAccess(accessRequest.resourceId, accessRequest.mode, userBox)(sharingTokenAccessCtx)
99+
case AccessResourceType.dataset =>
100+
handleDataSetAccess(accessRequest.resourceId.directoryName, accessRequest.mode, userBox)(
101+
sharingTokenAccessCtx)
99102
case AccessResourceType.tracing =>
100103
handleTracingAccess(accessRequest.resourceId.directoryName, accessRequest.mode, userBox, token)
101104
case AccessResourceType.annotation =>
@@ -158,6 +161,25 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO,
158161
}
159162
}
160163

164+
def handleDataSetAccess(id: String, mode: AccessMode.Value, userBox: Box[User])(
165+
implicit ctx: DBAccessContext): Fox[UserAccessAnswer] = {
166+
167+
def tryRead: Fox[UserAccessAnswer] =
168+
for {
169+
datasetId <- ObjectId.fromString(id)
170+
datasetBox <- datasetDAO.findOne(datasetId).shiftBox
171+
} yield
172+
datasetBox match {
173+
case Full(_) => UserAccessAnswer(granted = true)
174+
case _ => UserAccessAnswer(granted = false, Some("No read access on dataset"))
175+
}
176+
177+
mode match {
178+
case AccessMode.read => tryRead
179+
case _ => Fox.successful(UserAccessAnswer(granted = false, Some("invalid access token")))
180+
}
181+
}
182+
161183
private def handleTracingAccess(tracingId: String,
162184
mode: AccessMode,
163185
userBox: Box[User],

‎app/controllers/WKRemoteDataStoreController.scala

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,17 @@ class WKRemoteDataStoreController @Inject()(
259259
}
260260
}
261261

262+
def getDataset(name: String, key: String, datasetId: ObjectId): Action[AnyContent] =
263+
Action.async { implicit request =>
264+
dataStoreService.validateAccess(name, key) { _ =>
265+
for {
266+
dataset <- datasetDAO.findOne(datasetId)(GlobalAccessContext)
267+
dataSource <- datasetService.fullDataSourceFor(dataset)
268+
} yield Ok(Json.toJson(dataSource))
269+
}
270+
271+
}
272+
262273
def jobExportProperties(name: String, key: String, jobId: ObjectId): Action[AnyContent] = Action.async {
263274
implicit request =>
264275
dataStoreService.validateAccess(name, key) { _ =>

‎app/models/dataset/Dataset.scala

Lines changed: 208 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,20 @@ import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int}
55
import com.scalableminds.util.objectid.ObjectId
66
import com.scalableminds.util.time.Instant
77
import com.scalableminds.util.tools.{Fox, JsonHelper}
8+
import com.scalableminds.webknossos.datastore.dataformats.MagLocator
9+
import com.scalableminds.webknossos.datastore.dataformats.layers.{
10+
N5DataLayer,
11+
N5SegmentationLayer,
12+
PrecomputedDataLayer,
13+
PrecomputedSegmentationLayer,
14+
Zarr3DataLayer,
15+
Zarr3SegmentationLayer,
16+
ZarrDataLayer,
17+
ZarrSegmentationLayer
18+
}
19+
import com.scalableminds.webknossos.datastore.datareaders.AxisOrder
820
import com.scalableminds.webknossos.datastore.helpers.DataSourceMagInfo
9-
import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize}
21+
import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize, datasource}
1022
import com.scalableminds.webknossos.datastore.models.datasource.DatasetViewConfiguration.DatasetViewConfiguration
1123
import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfiguration.LayerViewConfiguration
1224
import com.scalableminds.webknossos.datastore.models.datasource.inbox.{InboxDataSourceLike => InboxDataSource}
@@ -17,8 +29,10 @@ import com.scalableminds.webknossos.datastore.models.datasource.{
1729
Category,
1830
CoordinateTransformation,
1931
CoordinateTransformationType,
32+
DataFormat,
2033
DataSourceId,
2134
ElementClass,
35+
SegmentationLayerLike,
2236
ThinPlateSplineCorrespondences,
2337
DataLayerLike => DataLayer
2438
}
@@ -839,6 +853,30 @@ class DatasetMagsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionConte
839853
magInfos <- rowsToMagInfos(rows)
840854
} yield magInfos
841855

856+
def parseMagLocator(row: DatasetMagsRow): Fox[MagLocator] =
857+
for {
858+
mag <- parseMag(row.mag)
859+
axisOrderParsed = row.axisorder match {
860+
case Some(axisOrder) => JsonHelper.parseAs[AxisOrder](axisOrder).toOption
861+
case None => None
862+
}
863+
} yield
864+
MagLocator(
865+
mag,
866+
row.path, // Does this make sense -> mag path may be something different than path in DB :/
867+
None,
868+
axisOrderParsed,
869+
row.channelindex,
870+
row.credentialid
871+
)
872+
873+
def findAllByDatasetId(datasetId: ObjectId): Fox[Seq[(String, MagLocator)]] =
874+
for {
875+
// We assume non-WKW Datasets here (WKW Resolutions are not handled)
876+
rows <- run(q"""SELECT * FROM webknossos.dataset_mags WHERE _dataset = $datasetId""".as[DatasetMagsRow])
877+
mags <- Fox.combined(rows.map(parseMagLocator))
878+
} yield rows.map(r => r.datalayername).zip(mags)
879+
842880
}
843881

844882
class DatasetLayerDAO @Inject()(
@@ -865,6 +903,7 @@ class DatasetLayerDAO @Inject()(
865903
coordinateTransformationsOpt = if (coordinateTransformations.isEmpty) None else Some(coordinateTransformations)
866904
additionalAxes <- datasetLayerAdditionalAxesDAO.findAllForDatasetAndDataLayerName(datasetId, row.name)
867905
additionalAxesOpt = if (additionalAxes.isEmpty) None else Some(additionalAxes)
906+
dataFormat = row.dataformat.flatMap(df => DataFormat.fromString(df))
868907
} yield {
869908
category match {
870909
case Category.segmentation =>
@@ -881,7 +920,9 @@ class DatasetLayerDAO @Inject()(
881920
defaultViewConfigurationOpt,
882921
adminViewConfigurationOpt,
883922
coordinateTransformationsOpt,
884-
additionalAxesOpt
923+
additionalAxesOpt,
924+
numChannels = row.numchannels,
925+
dataFormat = dataFormat
885926
))
886927
case Category.color =>
887928
Fox.successful(
@@ -894,7 +935,9 @@ class DatasetLayerDAO @Inject()(
894935
defaultViewConfigurationOpt,
895936
adminViewConfigurationOpt,
896937
coordinateTransformationsOpt,
897-
additionalAxesOpt
938+
additionalAxesOpt,
939+
numChannels = row.numchannels,
940+
dataFormat = dataFormat
898941
))
899942
case _ => Fox.failure(s"Could not match dataset layer with category $category")
900943
}
@@ -905,13 +948,174 @@ class DatasetLayerDAO @Inject()(
905948
def findAllForDataset(datasetId: ObjectId): Fox[List[DataLayer]] =
906949
for {
907950
rows <- run(q"""SELECT _dataset, name, category, elementClass, boundingBox, largestSegmentId, mappings,
908-
defaultViewConfiguration, adminViewConfiguration
951+
defaultViewConfiguration, adminViewConfiguration, numChannels, dataFormat
909952
FROM webknossos.dataset_layers
910953
WHERE _dataset = $datasetId
911954
ORDER BY name""".as[DatasetLayersRow])
912955
rowsParsed <- Fox.combined(rows.toList.map(parseRow(_, datasetId)))
913956
} yield rowsParsed
914957

958+
def findAllForDatasetWithMags(datasetId: ObjectId): Fox[List[DataLayer]] =
959+
for {
960+
layers <- findAllForDataset(datasetId)
961+
layerNamesAndMags <- datasetMagsDAO.findAllByDatasetId(datasetId)
962+
layersWithMags = layers.map { layer =>
963+
val mags = layerNamesAndMags.filter(_._1 == layer.name).map(_._2).toList
964+
layer match {
965+
// Maybe move this somewhere else?
966+
case AbstractDataLayer(name,
967+
category,
968+
boundingBox,
969+
resolutions,
970+
elementClass,
971+
defaultViewConfiguration,
972+
adminViewConfiguration,
973+
coordinateTransformations,
974+
additionalAxes,
975+
_,
976+
numChannels,
977+
dataFormat,
978+
_) =>
979+
dataFormat match {
980+
case Some(df) =>
981+
df match {
982+
case DataFormat.wkw =>
983+
throw new NotImplementedError(
984+
"WKW data format not supported in this context, only datasets with MagLocators are supported")
985+
case DataFormat.neuroglancerPrecomputed =>
986+
PrecomputedDataLayer(name,
987+
boundingBox,
988+
category,
989+
elementClass,
990+
mags,
991+
defaultViewConfiguration,
992+
adminViewConfiguration,
993+
coordinateTransformations,
994+
numChannels,
995+
additionalAxes)
996+
case DataFormat.n5 =>
997+
N5DataLayer(name,
998+
category,
999+
boundingBox,
1000+
elementClass,
1001+
mags,
1002+
defaultViewConfiguration,
1003+
adminViewConfiguration,
1004+
coordinateTransformations,
1005+
numChannels,
1006+
additionalAxes)
1007+
case DataFormat.zarr =>
1008+
ZarrDataLayer(name,
1009+
category,
1010+
boundingBox,
1011+
elementClass,
1012+
mags,
1013+
defaultViewConfiguration,
1014+
adminViewConfiguration,
1015+
coordinateTransformations,
1016+
numChannels,
1017+
additionalAxes,
1018+
df)
1019+
case DataFormat.zarr3 =>
1020+
Zarr3DataLayer(name,
1021+
category,
1022+
boundingBox,
1023+
elementClass,
1024+
mags,
1025+
defaultViewConfiguration,
1026+
adminViewConfiguration,
1027+
coordinateTransformations,
1028+
numChannels,
1029+
additionalAxes)
1030+
}
1031+
case None => ???
1032+
}
1033+
case AbstractSegmentationLayer(name,
1034+
category,
1035+
boundingBox,
1036+
resolutions,
1037+
elementClass,
1038+
largestSegmentId,
1039+
mappings,
1040+
defaultViewConfiguration,
1041+
adminViewConfiguration,
1042+
coordinateTransformations,
1043+
additionalAxes,
1044+
_,
1045+
numChannels,
1046+
dataFormat,
1047+
_) =>
1048+
dataFormat match {
1049+
case Some(df) =>
1050+
df match {
1051+
case DataFormat.wkw =>
1052+
throw new NotImplementedError(
1053+
"WKW data format not supported in this context, only datasets with MagLocators are supported")
1054+
case DataFormat.neuroglancerPrecomputed =>
1055+
PrecomputedSegmentationLayer(
1056+
name,
1057+
boundingBox,
1058+
elementClass,
1059+
mags,
1060+
largestSegmentId,
1061+
mappings,
1062+
defaultViewConfiguration,
1063+
adminViewConfiguration,
1064+
coordinateTransformations,
1065+
numChannels,
1066+
additionalAxes,
1067+
)
1068+
case DataFormat.n5 =>
1069+
N5SegmentationLayer(
1070+
name,
1071+
boundingBox,
1072+
elementClass,
1073+
mags,
1074+
largestSegmentId,
1075+
mappings,
1076+
defaultViewConfiguration,
1077+
adminViewConfiguration,
1078+
coordinateTransformations,
1079+
numChannels,
1080+
additionalAxes
1081+
)
1082+
case DataFormat.zarr =>
1083+
ZarrSegmentationLayer(
1084+
name,
1085+
boundingBox,
1086+
elementClass,
1087+
mags,
1088+
largestSegmentId,
1089+
mappings,
1090+
defaultViewConfiguration,
1091+
adminViewConfiguration,
1092+
coordinateTransformations,
1093+
numChannels,
1094+
additionalAxes,
1095+
df
1096+
)
1097+
case DataFormat.zarr3 =>
1098+
Zarr3SegmentationLayer(
1099+
name,
1100+
boundingBox,
1101+
elementClass,
1102+
mags,
1103+
largestSegmentId,
1104+
mappings,
1105+
defaultViewConfiguration,
1106+
adminViewConfiguration,
1107+
coordinateTransformations,
1108+
numChannels,
1109+
additionalAxes
1110+
)
1111+
}
1112+
case None => ???
1113+
}
1114+
case _ => throw new NotImplementedError("DataLayer type mismatch (unreachable)")
1115+
}
1116+
}
1117+
} yield layersWithMags
1118+
9151119
private def insertLayerQuery(datasetId: ObjectId, layer: DataLayer): SqlAction[Int, NoStream, Effect] =
9161120
layer match {
9171121
case s: AbstractSegmentationLayer =>

‎app/models/dataset/DatasetService.scala

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
260260
}
261261
}
262262

263+
// TODO: This needs to be extended to include new properties, also mags
263264
def dataSourceFor(dataset: Dataset): Fox[InboxDataSource] =
264265
(for {
265266
dataLayers <- datasetDataLayerDAO.findAllForDataset(dataset._id)
@@ -273,6 +274,23 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
273274
Fox.successful(UnusableDataSource[DataLayer](dataSourceId, dataset.status, dataset.voxelSize))
274275
}).flatten
275276

277+
278+
// Returns a JSON that includes all properties of the data source and of data layers to read the dataset
279+
def fullDataSourceFor(dataset: Dataset) : Fox[InboxDataSource] =
280+
(for {
281+
dataLayers <- datasetDataLayerDAO.findAllForDatasetWithMags(dataset._id)
282+
dataSourceId = DataSourceId(dataset.directoryName, dataset._organization)
283+
} yield {
284+
if (dataset.isUsable)
285+
for {
286+
scale <- dataset.voxelSize.toFox ?~> "dataset.source.usableButNoScale"
287+
} yield GenericDataSource[DataLayer](dataSourceId, dataLayers, scale)
288+
else
289+
Fox.successful(UnusableDataSource[DataLayer](dataSourceId, dataset.status, dataset.voxelSize))
290+
}
291+
292+
).flatten
293+
276294
private def logoUrlFor(dataset: Dataset, organization: Option[Organization]): Fox[String] =
277295
dataset.logoUrl match {
278296
case Some(url) => Fox.successful(url)

‎conf/webknossos.latest.routes

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ PUT /datastores/:name/datasource
110110
PUT /datastores/:name/datasources controllers.WKRemoteDataStoreController.updateAll(name: String, key: String)
111111
PUT /datastores/:name/datasources/paths controllers.WKRemoteDataStoreController.updatePaths(name: String, key: String)
112112
GET /datastores/:name/datasources/:organizationId/:directoryName/paths controllers.WKRemoteDataStoreController.getPaths(name: String, key: String, organizationId: String, directoryName: String)
113+
GET /datastores/:name/datasets/:datasetId controllers.WKRemoteDataStoreController.getDataset(name: String, key: String, datasetId: ObjectId)
113114
PATCH /datastores/:name/status controllers.WKRemoteDataStoreController.statusUpdate(name: String, key: String)
114115
POST /datastores/:name/reserveUpload controllers.WKRemoteDataStoreController.reserveDatasetUpload(name: String, key: String, token: String)
115116
GET /datastores/:name/getUnfinishedUploadsForUser controllers.WKRemoteDataStoreController.getUnfinishedUploadsForUser(name: String, key: String, token: String, organizationName: String)

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,6 @@ class DataStoreModule extends AbstractModule {
3434
bind(classOf[MeshFileService]).asEagerSingleton()
3535
bind(classOf[NeuroglancerPrecomputedMeshFileService]).asEagerSingleton()
3636
bind(classOf[RemoteSourceDescriptorService]).asEagerSingleton()
37+
bind(classOf[DatasetCache]).asEagerSingleton()
3738
}
3839
}

0 commit comments

Comments
 (0)