Skip to content

Commit 63ddf7b

Browse files
frcrothfrcroth
committed
Include attachments in WK generated datasource
1 parent 9d3ad67 commit 63ddf7b

File tree

5 files changed

+120
-73
lines changed

5 files changed

+120
-73
lines changed

‎app/controllers/UserTokenController.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,9 +183,9 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO,
183183
} yield UserAccessAnswer(isAllowed)
184184

185185
mode match {
186-
case AccessMode.read => tryRead
186+
case AccessMode.read => tryRead
187187
case AccessMode.write => tryWrite
188-
case _ => Fox.successful(UserAccessAnswer(granted = false, Some("invalid access token")))
188+
case _ => Fox.successful(UserAccessAnswer(granted = false, Some("invalid access token")))
189189
}
190190
}
191191

‎app/models/dataset/Dataset.scala

Lines changed: 44 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,6 @@ import com.scalableminds.util.objectid.ObjectId
66
import com.scalableminds.util.time.Instant
77
import com.scalableminds.util.tools.{Fox, JsonHelper}
88
import com.scalableminds.webknossos.datastore.dataformats.MagLocator
9-
import com.scalableminds.webknossos.datastore.dataformats.layers.{
10-
N5DataLayer,
11-
N5SegmentationLayer,
12-
PrecomputedDataLayer,
13-
PrecomputedSegmentationLayer,
14-
Zarr3DataLayer,
15-
Zarr3SegmentationLayer,
16-
ZarrDataLayer,
17-
ZarrSegmentationLayer
18-
}
199
import com.scalableminds.webknossos.datastore.datareaders.AxisOrder
2010
import com.scalableminds.webknossos.datastore.helpers.DataSourceMagInfo
2111
import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize, datasource}
@@ -30,11 +20,13 @@ import com.scalableminds.webknossos.datastore.models.datasource.{
3020
CoordinateTransformation,
3121
CoordinateTransformationType,
3222
DataFormat,
23+
DatasetLayerAttachments => AttachmentWrapper,
3324
DataSourceId,
3425
ElementClass,
35-
SegmentationLayerLike,
3626
LayerAttachment,
27+
LayerAttachmentDataFormat,
3728
LayerAttachmentType,
29+
SegmentationLayerLike,
3830
ThinPlateSplineCorrespondences,
3931
DataLayerLike => DataLayer
4032
}
@@ -54,6 +46,7 @@ import slick.lifted.Rep
5446
import slick.sql.SqlAction
5547
import utils.sql.{SQLDAO, SimpleSQLDAO, SqlClient, SqlToken}
5648

49+
import java.net.URI
5750
import scala.concurrent.ExecutionContext
5851

5952
case class Dataset(_id: ObjectId,
@@ -905,6 +898,9 @@ class DatasetLayerDAO @Inject()(sqlClient: SqlClient,
905898
coordinateTransformationsOpt = if (coordinateTransformations.isEmpty) None else Some(coordinateTransformations)
906899
additionalAxes <- datasetLayerAdditionalAxesDAO.findAllForDatasetAndDataLayerName(datasetId, row.name)
907900
additionalAxesOpt = if (additionalAxes.isEmpty) None else Some(additionalAxes)
901+
attachments <- datasetLayerAttachmentsDAO.findAllForDatasetAndDataLayerName(datasetId, row.name)
902+
attachmentsOpt = if (attachments.isEmpty) None else Some(attachments)
903+
908904
dataFormat = row.dataformat.flatMap(df => DataFormat.fromString(df))
909905
} yield {
910906
category match {
@@ -924,7 +920,8 @@ class DatasetLayerDAO @Inject()(sqlClient: SqlClient,
924920
coordinateTransformationsOpt,
925921
additionalAxesOpt,
926922
numChannels = row.numchannels,
927-
dataFormat = dataFormat
923+
dataFormat = dataFormat,
924+
attachments = attachmentsOpt
928925
))
929926
case Category.color =>
930927
Fox.successful(
@@ -939,7 +936,8 @@ class DatasetLayerDAO @Inject()(sqlClient: SqlClient,
939936
coordinateTransformationsOpt,
940937
additionalAxesOpt,
941938
numChannels = row.numchannels,
942-
dataFormat = dataFormat
939+
dataFormat = dataFormat,
940+
attachments = attachmentsOpt
943941
))
944942
case _ => Fox.failure(s"Could not match dataset layer with category $category")
945943
}
@@ -1058,6 +1056,39 @@ class DatasetLastUsedTimesDAO @Inject()(sqlClient: SqlClient)(implicit ec: Execu
10581056

10591057
class DatasetLayerAttachmentsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
10601058
extends SimpleSQLDAO(sqlClient) {
1059+
1060+
def parseRow(row: DatasetLayerAttachmentsRow): Fox[LayerAttachment] =
1061+
for {
1062+
dataFormat <- LayerAttachmentDataFormat.fromString(row.dataformat).toFox ?~> "Could not parse data format"
1063+
uri = new URI(row.path)
1064+
} yield LayerAttachment(row.name, uri, dataFormat)
1065+
1066+
def parseAttachments(rows: List[DatasetLayerAttachmentsRow]): Fox[AttachmentWrapper] =
1067+
for {
1068+
meshFiles <- Fox.serialCombined(rows.filter(_.`type` == LayerAttachmentType.mesh.toString))(parseRow)
1069+
agglomerateFiles <- Fox.serialCombined(rows.filter(_.`type` == LayerAttachmentType.agglomerate.toString))(
1070+
parseRow)
1071+
connectomeFiles <- Fox.serialCombined(rows.filter(_.`type` == LayerAttachmentType.connectome.toString))(parseRow)
1072+
segmentIndexFiles <- Fox.serialCombined(rows.filter(_.`type` == LayerAttachmentType.segmentIndex.toString))(
1073+
parseRow)
1074+
cumsumFiles <- Fox.serialCombined(rows.filter(_.`type` == LayerAttachmentType.cumsum.toString))(parseRow)
1075+
} yield
1076+
AttachmentWrapper(
1077+
agglomerates = agglomerateFiles,
1078+
connectomes = connectomeFiles,
1079+
segmentIndex = segmentIndexFiles.headOption,
1080+
meshes = meshFiles,
1081+
cumsum = cumsumFiles.headOption
1082+
)
1083+
1084+
def findAllForDatasetAndDataLayerName(datasetId: ObjectId, layerName: String): Fox[AttachmentWrapper] =
1085+
for {
1086+
rows <- run(q"""SELECT _dataset, layerName, name, path, type, dataFormat
1087+
FROM webknossos.dataset_layer_attachments
1088+
WHERE _dataset = $datasetId AND layerName = $layerName""".as[DatasetLayerAttachmentsRow])
1089+
attachments <- parseAttachments(rows.toList) ?~> "Could not parse attachments"
1090+
} yield attachments
1091+
10611092
def updateAttachments(datasetId: ObjectId, dataLayersOpt: Option[List[DataLayer]]): Fox[Unit] = {
10621093
def insertQuery(attachment: LayerAttachment, layerName: String, fileType: String) =
10631094
q"""INSERT INTO webknossos.dataset_layer_attachments(_dataset, layerName, name, path, type, dataFormat)

‎app/models/dataset/DatasetService.scala

Lines changed: 61 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
319319
adminViewConfiguration,
320320
coordinateTransformations,
321321
additionalAxes,
322+
attachmentsOpt,
322323
_,
323324
numChannels,
324325
dataFormat,
@@ -330,50 +331,62 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
330331
throw new NotImplementedError(
331332
"WKW data format not supported in this context, only datasets with MagLocators are supported")
332333
case DataFormat.neuroglancerPrecomputed =>
333-
PrecomputedDataLayer(name,
334-
boundingBox,
335-
category,
336-
elementClass,
337-
mags,
338-
defaultViewConfiguration,
339-
adminViewConfiguration,
340-
coordinateTransformations,
341-
numChannels,
342-
additionalAxes)
334+
PrecomputedDataLayer(
335+
name,
336+
boundingBox,
337+
category,
338+
elementClass,
339+
mags,
340+
defaultViewConfiguration,
341+
adminViewConfiguration,
342+
coordinateTransformations,
343+
numChannels,
344+
additionalAxes,
345+
attachmentsOpt
346+
)
343347
case DataFormat.n5 =>
344-
N5DataLayer(name,
345-
category,
346-
boundingBox,
347-
elementClass,
348-
mags,
349-
defaultViewConfiguration,
350-
adminViewConfiguration,
351-
coordinateTransformations,
352-
numChannels,
353-
additionalAxes)
348+
N5DataLayer(
349+
name,
350+
category,
351+
boundingBox,
352+
elementClass,
353+
mags,
354+
defaultViewConfiguration,
355+
adminViewConfiguration,
356+
coordinateTransformations,
357+
numChannels,
358+
additionalAxes,
359+
attachmentsOpt
360+
)
354361
case DataFormat.zarr =>
355-
ZarrDataLayer(name,
356-
category,
357-
boundingBox,
358-
elementClass,
359-
mags,
360-
defaultViewConfiguration,
361-
adminViewConfiguration,
362-
coordinateTransformations,
363-
numChannels,
364-
additionalAxes,
365-
df)
362+
ZarrDataLayer(
363+
name,
364+
category,
365+
boundingBox,
366+
elementClass,
367+
mags,
368+
defaultViewConfiguration,
369+
adminViewConfiguration,
370+
coordinateTransformations,
371+
numChannels,
372+
additionalAxes,
373+
attachmentsOpt,
374+
df
375+
)
366376
case DataFormat.zarr3 =>
367-
Zarr3DataLayer(name,
368-
category,
369-
boundingBox,
370-
elementClass,
371-
mags,
372-
defaultViewConfiguration,
373-
adminViewConfiguration,
374-
coordinateTransformations,
375-
numChannels,
376-
additionalAxes)
377+
Zarr3DataLayer(
378+
name,
379+
category,
380+
boundingBox,
381+
elementClass,
382+
mags,
383+
defaultViewConfiguration,
384+
adminViewConfiguration,
385+
coordinateTransformations,
386+
numChannels,
387+
additionalAxes,
388+
attachmentsOpt,
389+
)
377390
}
378391
case None => ???
379392
}
@@ -388,6 +401,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
388401
adminViewConfiguration,
389402
coordinateTransformations,
390403
additionalAxes,
404+
attachmentsOpt,
391405
_,
392406
numChannels,
393407
dataFormat,
@@ -411,6 +425,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
411425
coordinateTransformations,
412426
numChannels,
413427
additionalAxes,
428+
attachmentsOpt
414429
)
415430
case DataFormat.n5 =>
416431
N5SegmentationLayer(
@@ -424,7 +439,8 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
424439
adminViewConfiguration,
425440
coordinateTransformations,
426441
numChannels,
427-
additionalAxes
442+
additionalAxes,
443+
attachmentsOpt
428444
)
429445
case DataFormat.zarr =>
430446
ZarrSegmentationLayer(
@@ -439,6 +455,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
439455
coordinateTransformations,
440456
numChannels,
441457
additionalAxes,
458+
attachmentsOpt,
442459
df
443460
)
444461
case DataFormat.zarr3 =>
@@ -453,7 +470,8 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
453470
adminViewConfiguration,
454471
coordinateTransformations,
455472
numChannels,
456-
additionalAxes
473+
additionalAxes,
474+
attachmentsOpt
457475
)
458476
}
459477
case None => ???

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DatasetLayerAttachments.scala

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,16 @@ case class DatasetLayerAttachments(
1515
segmentIndex: Option[LayerAttachment],
1616
connectomes: Seq[LayerAttachment],
1717
cumsum: Option[LayerAttachment]
18-
)
18+
) {
19+
def isEmpty: Boolean =
20+
meshes.isEmpty && agglomerates.isEmpty && segmentIndex.isEmpty && connectomes.isEmpty && cumsum.isEmpty
21+
}
1922

2023
object DatasetLayerAttachments {
2124
implicit val jsonFormat: Format[DatasetLayerAttachments] = Json.format[DatasetLayerAttachments]
2225
}
2326

24-
object LayerAttachmentDataformat extends ExtendedEnumeration {
27+
object LayerAttachmentDataFormat extends ExtendedEnumeration {
2528
type LayerAttachmentDataformat = Value
2629
val hdf5, json, zarr3 = Value
2730
}
@@ -31,14 +34,14 @@ object LayerAttachmentType extends ExtendedEnumeration {
3134
val mesh, agglomerate, segmentIndex, connectome, cumsum = Value
3235
}
3336

34-
case class LayerAttachment(name: String, path: URI, dataFormat: LayerAttachmentDataformat.LayerAttachmentDataformat)
37+
case class LayerAttachment(name: String, path: URI, dataFormat: LayerAttachmentDataFormat.LayerAttachmentDataformat)
3538

3639
object LayerAttachment {
3740
implicit val jsonFormat: Format[LayerAttachment] = Json.format[LayerAttachment]
3841

3942
def scanForFiles(layerDirectory: Path,
4043
directoryName: String,
41-
dataFormat: LayerAttachmentDataformat.LayerAttachmentDataformat): Seq[LayerAttachment] = {
44+
dataFormat: LayerAttachmentDataFormat.LayerAttachmentDataformat): Seq[LayerAttachment] = {
4245
val dir = layerDirectory.resolve(directoryName)
4346
val scanExtension = dataFormat.toString
4447
if (Files.exists(dir)) {
@@ -58,39 +61,39 @@ object LayerAttachment {
5861

5962
object MeshFileInfo {
6063
val directoryName = "meshes"
61-
private val scanDataFormat = LayerAttachmentDataformat.hdf5
64+
private val scanDataFormat = LayerAttachmentDataFormat.hdf5
6265

6366
def scanForMeshFiles(layerDirectory: Path): Seq[LayerAttachment] =
6467
LayerAttachment.scanForFiles(layerDirectory, directoryName, scanDataFormat)
6568
}
6669

6770
object AgglomerateFileInfo {
6871
val directoryName = "agglomerates"
69-
private val scanDataFormat = LayerAttachmentDataformat.hdf5
72+
private val scanDataFormat = LayerAttachmentDataFormat.hdf5
7073

7174
def scanForAgglomerateFiles(layerDirectory: Path): Seq[LayerAttachment] =
7275
LayerAttachment.scanForFiles(layerDirectory, directoryName, scanDataFormat)
7376
}
7477

7578
object SegmentIndexFileInfo {
7679
val directoryName = "segmentIndex"
77-
private val scanDataFormat = LayerAttachmentDataformat.hdf5
80+
private val scanDataFormat = LayerAttachmentDataFormat.hdf5
7881

7982
def scanForSegmentIndexFile(layerDirectory: Path): Option[LayerAttachment] =
8083
LayerAttachment.scanForFiles(layerDirectory, directoryName, scanDataFormat).headOption
8184
}
8285

8386
object ConnectomeFileInfo {
8487
val directoryName = "connectomes"
85-
private val scanDataFormat = LayerAttachmentDataformat.hdf5
88+
private val scanDataFormat = LayerAttachmentDataFormat.hdf5
8689

8790
def scanForConnectomeFiles(layerDirectory: Path): Seq[LayerAttachment] =
8891
LayerAttachment.scanForFiles(layerDirectory, directoryName, scanDataFormat)
8992
}
9093

9194
object CumsumFileInfo {
9295
val directoryName = "agglomerates"
93-
private val scanDataFormat = LayerAttachmentDataformat.json
96+
private val scanDataFormat = LayerAttachmentDataFormat.json
9497

9598
def scanForCumsumFile(layerDirectory: Path): Option[LayerAttachment] =
9699
LayerAttachment.scanForFiles(layerDirectory, directoryName, scanDataFormat).headOption

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSRemoteWebknossosClient.scala

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,7 @@ import com.scalableminds.webknossos.datastore.controllers.JobExportProperties
1212
import com.scalableminds.webknossos.datastore.helpers.{IntervalScheduler, LayerMagLinkInfo}
1313
import com.scalableminds.webknossos.datastore.models.UnfinishedUpload
1414
import com.scalableminds.webknossos.datastore.models.annotation.AnnotationSource
15-
import com.scalableminds.webknossos.datastore.models.datasource.{
16-
DataLayer,
17-
DataLayerLike,
18-
DataSourceId,
19-
GenericDataSource
20-
}
15+
import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, DataSourceId, GenericDataSource}
2116
import com.scalableminds.webknossos.datastore.models.datasource.inbox.InboxDataSourceLike
2217
import com.scalableminds.webknossos.datastore.rpc.RPC
2318
import com.scalableminds.webknossos.datastore.services.uploading.{

0 commit comments

Comments
 (0)