From 979769c63753f2bc63299b89f4b73c06ae2eb1bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Thu, 27 Feb 2025 12:30:28 +0000 Subject: [PATCH 01/12] module works --- bin/recyze.py | 341 ++++++++++++++++++ modules/local/roadie/recyze/main.nf | 50 +++ modules/local/roadie/recyze/meta.yml | 58 +++ .../local/roadie/recyze/tests/main.nf.test | 65 ++++ workflows/mcmicro.nf | 4 + 5 files changed, 518 insertions(+) create mode 100755 bin/recyze.py create mode 100644 modules/local/roadie/recyze/main.nf create mode 100644 modules/local/roadie/recyze/meta.yml create mode 100644 modules/local/roadie/recyze/tests/main.nf.test diff --git a/bin/recyze.py b/bin/recyze.py new file mode 100755 index 0000000..0666cbd --- /dev/null +++ b/bin/recyze.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +import math +import sys +import tifffile +import zarr +import numpy as np +from ome_types import from_tiff, to_xml +from pathlib import Path +import argparse +import os +import uuid + +class PyramidWriter: + + def __init__( + self, _in_path, _out_path, _channels, _nuclear_channels, _membrane_channels, _max_projection, _x, _y, _x2, _y2, _w, _h, scale=2, tile_size=1024, peak_size=1024, + verbose=False + ): + if tile_size % 16 != 0: + raise ValueError("tile_size must be a multiple of 16") + self.in_path = Path(_in_path) + self.in_tiff = tifffile.TiffFile(self.in_path, is_ome=False) + self.in_data = zarr.open(self.in_tiff.series[0].aszarr()) + self.out_path = Path(_out_path) + self.metadata = from_tiff(self.in_path) + self.max_projection = _max_projection + + self.is_zarr_hierarchy_group = isinstance(self.in_data, zarr.hierarchy.Group) + self.base_data = self.in_data[0] if self.is_zarr_hierarchy_group else self.in_data + + self.tile_size = tile_size + self.peak_size = peak_size + self.scale = scale + print(f" ndim: {self.base_data.ndim}") + if self.base_data.ndim == 3: # Multi-channel image + self.single_channel = False + if _channels: + if max(_channels) >= self.base_data.shape[0]: + print("Channel out of range", file=sys.stderr) + sys.exit(1) + else: + self.channels = _channels + else: + self.channels = np.arange(self.base_data.shape[0], dtype=int).tolist() + if _nuclear_channels: + if max(_nuclear_channels) >= self.base_data.shape[0]: + print("Nuclear channel out of range", file=sys.stderr) + sys.exit(1) + else: + self.nuclear_channels = _nuclear_channels + else: + self.nuclear_channels = [] + if _membrane_channels: + if max(_membrane_channels) >= self.base_data.shape[0]: + print("Membrane channel out of range", file=sys.stderr) + sys.exit(1) + else: + self.membrane_channels = _membrane_channels + else: + self.membrane_channels = [] + else: # Single Channel image + self.single_channel = True + if _channels and max(_channels) > 0: + print("Channel out of range", file=sys.stderr) + sys.exit(1) + if _nuclear_channels and max(_nuclear_channels) > 0: + print("Nuclear channel out of range", file=sys.stderr) + sys.exit(1) + if _membrane_channels and max(_membrane_channels) > 0: + print("Membrane channel out of range", file=sys.stderr) + sys.exit(1) + if _max_projection: + print("Max projection not possible on single channel image.", file=sys.stderr) + sys.exit(1) + self.channels = [0] + + if self.max_projection and len(self.membrane_channels) > 0 and len(self.nuclear_channels) > 0: + self.channels = [0, 1] + elif self.max_projection and len(self.nuclear_channels) > 0: + self.channels = [0] + + xy = _x is not None and _y is not None + xy2 = _x2 is not None and _y2 is not None + wh = _w is not None and _h is not None + if all(v is None for v in (_x, _y, _x2, _y2, _w, _h)): + _w = self.base_data.shape[-1] + _h = self.base_data.shape[-2] + _x = _y = 0 + elif not xy or not (wh ^ xy2): + print("Please specify x/y and either x2/y2 or w/h", file=sys.stderr) + sys.exit(1) + elif xy2: + _w = _x2 - _x + _h = _y2 - _y + + self.num_levels = math.ceil(math.log((max([_h, _w]) / self.peak_size), self.scale)) + 1 + + rounded_x = np.floor(_x / (self.scale ** (self.num_levels - 1))).astype(int) * (2 ** (self.num_levels - 1)) + self.x = max([rounded_x, 0]) + + rounded_y = np.floor(_y / (self.scale ** (self.num_levels - 1))).astype(int) * (2 ** (self.num_levels - 1)) + self.y = max([rounded_y, 0]) + + rounded_width = np.ceil((_w + self.x) / (self.scale ** (self.num_levels - 1))).astype(int) * \ + (2 ** (self.num_levels - 1)) - self.x + self.width = min([rounded_width, self.base_data.shape[-1]]) + + rounded_height = np.ceil((_h + self.y) / (self.scale ** (self.num_levels - 1))).astype( + int) * (2 ** (self.num_levels - 1)) - self.y + self.height = min([rounded_height, self.base_data.shape[-2]]) + + print('Params:', 'x', self.x, 'y', self.y, 'height', self.height, 'width', self.width, 'levels', + self.num_levels, + 'channels', self.channels, 'nuclear_channels', self.nuclear_channels, 'membrane_channels', self.membrane_channels, + 'max_projection', self.max_projection + ) + + self.verbose = verbose + + @property + def base_shape(self): + "Shape of the base level." + return [self.height, self.width] + + @property + def num_channels(self): + return len(self.channels) + + @property + def level_shapes(self): + "Shape of all levels." + factors = self.scale ** np.arange(self.num_levels) + shapes = np.ceil(np.array(self.base_shape) / factors[:, None]) + return [tuple(map(int, s)) for s in shapes] + + @property + def level_full_shapes(self): + "Shape of all levels, including channel dimension." + return [(self.num_channels, *shape) for shape in self.level_shapes] + + @property + def tile_shapes(self): + "Tile shape of all levels." + level_shapes = np.array(self.level_shapes) + # The last level where we want to use the standard square tile size. + tip_level = np.argmax(np.all(level_shapes < self.tile_size, axis=1)) + tile_shapes = [ + (self.tile_size, self.tile_size) if i <= tip_level else None + for i in range(len(level_shapes)) + ] + # Remove NONE from list + + return tile_shapes + + def max_projection_channel(self, channels): + "Compute the maximum projection of the specified channels." + if not channels: + channels = self.channels + maxprojection = np.max( + self.base_data + .get_orthogonal_selection(( + channels, + slice(self.y,self.y + self.height), + slice(self.x,self.x + self.width ))),axis=0) + return maxprojection + + def base_tiles(self): + h, w = self.base_shape + th, tw = self.tile_shapes[0] + + if self.max_projection: + # If max projection is enabled, generate the maximum projection image + if self.nuclear_channels: + nuclear_img = self.max_projection_channel(self.nuclear_channels) + imgs = [nuclear_img] + if self.membrane_channels: + membrane_img = self.max_projection_channel(self.membrane_channels) + imgs.append(membrane_img) + else: + imgs = [self.max_projection_channel(self.channels)] + else: + imgs = [self.base_data[ci, self.y:self.y + self.height, self.x:self.x + self.width] for ci in self.channels] + + for img in imgs: + for y in range(0, h, th): + for x in range(0, w, tw): + yield img[y:y + th, x:x + tw].copy() + img = None + + def cropped_subres_image(self, base_img, level): + scale = 2 ** level + subres_x1 = int(self.x / scale) + subres_y1 = int(self.y / scale) + subres_width = int(self.width / scale) + subres_height = int(self.height / scale) + subres_x2 = min([subres_x1 + subres_width, base_img.shape[-1]]) + subres_y2 = min([subres_y1 + subres_height, base_img.shape[-2]]) + return base_img[subres_y1:subres_y2, subres_x1:subres_x2] + + def subres_tiles(self, level): + print(level, 'level') + assert level >= 1 + num_channels, h, w = self.level_full_shapes[level] + tshape = self.tile_shapes[level] or (h, w) + + for c in self.channels: + if self.single_channel: + base_img = self.in_data[level] + else: + base_img = self.in_data[level][c] + img = self.cropped_subres_image(base_img, level) + if self.verbose: + sys.stdout.write( + f"\r processing channel {c + 1}/{num_channels}" + ) + sys.stdout.flush() + th = tshape[0] + tw = tshape[1] + for y in range(0, img.shape[0], th): + for x in range(0, img.shape[1], tw): + a = img[y:y + th, x:x + tw] + a = a.astype(img.dtype) + yield a + + def run(self): + dtype = self.base_data.dtype + with tifffile.TiffWriter(self.out_path, ome=True, bigtiff=True) as tiff: + tiff.write( + data=self.base_tiles(), + software=self.in_tiff.pages[0].software, + shape=self.level_full_shapes[0], + subifds=int(self.num_levels - 1), + dtype=self.in_tiff.pages[0].dtype, + resolution=( + self.in_tiff.pages[0].tags["XResolution"].value, + self.in_tiff.pages[0].tags["YResolution"].value, + self.in_tiff.pages[0].tags["ResolutionUnit"].value), + tile=self.tile_shapes[0], + photometric=self.in_tiff.pages[0].photometric, + compression="adobe_deflate", + predictor=True, + ) + if self.verbose: + print("Generating pyramid") + for level, (shape, tile_shape) in enumerate( + zip(self.level_full_shapes[1:], self.tile_shapes[1:]), 1 + ): + if self.verbose: + print(f" Level {level} ({shape[2]} x {shape[1]})") + tiff.write( + data=self.subres_tiles(level), + shape=shape, + subfiletype=1, + dtype=dtype, + tile=tile_shape, + compression="adobe_deflate", + predictor=True, + ) + if self.verbose: + print() + self.metadata.images[0].pixels.channels = [self.metadata.images[0].pixels.channels[i] for i in + self.channels] + self.metadata.uuid = uuid.uuid4().urn + self.metadata.images[0].pixels.size_c = self.num_channels + self.metadata.images[0].pixels.size_x = self.width + self.metadata.images[0].pixels.size_y = self.height + if self.metadata.images[0].pixels.planes: + temp_planes = [] + for i, channel_id in enumerate(self.channels): + temp_plane = self.metadata.images[0].pixels.planes[channel_id] + temp_plane.the_c = i + temp_planes.append(temp_plane) + self.metadata.images[0].pixels.planes = temp_planes + if self.metadata.images[0].pixels.tiff_data_blocks and len( + self.metadata.images[0].pixels.tiff_data_blocks) > 0: + self.metadata.images[0].pixels.tiff_data_blocks[0].plane_count = self.num_channels + + # Write + tifffile.tiffcomment(self.out_path, to_xml(self.metadata).encode()) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--in', type=str, required=True, help="Input Image Path") + parser.add_argument('--out', type=str, required=False, help="Output Image Path") + parser.add_argument('--x', type=int, required=False, default=None, help="Crop X1") + parser.add_argument('--x2', type=int, required=False, default=None, help="Crop X2") + parser.add_argument('--y', type=int, required=False, default=None, help="Crop Y1") + parser.add_argument('--y2', type=int, required=False, default=None, help="Crop Y2") + parser.add_argument('--w', type=int, required=False, default=None, help="Crop Width") + parser.add_argument('--h', type=int, required=False, default=None, help="Crop Height") + parser.add_argument( + '--channels', type=int, nargs="+", required=False, default=None, metavar="C", + help="Channels to keep (Default: all)", + ) + parser.add_argument( + '--nuclear_channels', type=int, nargs="+", required=False, default=None, metavar="N", + help="Specifying nuclear channels to keep", + ) + parser.add_argument( + '--membrane_channels', type=int, nargs="+", required=False, default=None, metavar="M", + help="Specifying membrane channels to keep", + ) + parser.add_argument( + '--max_projection', action='store_true', help="Use max projection", + ) + parser.add_argument( + '--num-threads', type=int, required=False, default=0, metavar="N", + help="Worker thread count (Default: auto-scale based on number of available CPUs)", + ) + args = parser.parse_args() + + # Automatically infer the output filename, if not specified + in_path = vars(args)['in'] + out_path = args.out + if out_path is None: + # Tokenize the input filename and insert "_crop" + # at the appropriate location + tokens = os.path.basename(in_path).split(os.extsep) + if len(tokens) < 2: + out_path = in_path + "_crop" + elif tokens[-2] == "ome": + stem = os.extsep.join(tokens[0:-2]) + "_crop" + out_path = os.extsep.join([stem] + tokens[-2:]) + else: + stem = os.extsep.join(tokens[0:-1]) + "_crop" + out_path = os.extsep.join([stem, tokens[-1]]) + + num_threads = args.num_threads + if num_threads == 0: + if hasattr(os, "sched_getaffinity"): + num_threads = len(os.sched_getaffinity(0)) + else: + num_threads = os.cpu_count() + tifffile.TIFF.MAXWORKERS = num_threads + tifffile.TIFF.MAXIOWORKERS = num_threads * 5 + print(f"Nuclear channels: {args.nuclear_channels}") + print(f"Nuclear channels: {args.membrane_channels}") + + writer = PyramidWriter(in_path, out_path, args.channels, args.nuclear_channels, args.membrane_channels, args.max_projection, args.x, args.y, args.x2, args.y2, args.w, args.h) + writer.run() diff --git a/modules/local/roadie/recyze/main.nf b/modules/local/roadie/recyze/main.nf new file mode 100644 index 0000000..b76f8b6 --- /dev/null +++ b/modules/local/roadie/recyze/main.nf @@ -0,0 +1,50 @@ +process ROADIE_RECYZE { + tag "$meta.id" + label 'process_single' + + container "ghcr.io/labsyspharm/mcmicro:roadie-2023-10-25" + + input: + tuple val(meta), path(image) + + output: + tuple val(meta), path("*.tif"), emit: extracted_channels + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + + """ + recyze.py \\ + --in ${image} \\ + --out ${prefix}.tif \\ + --channels 0 \\ + --num-threads $task.cpus \\ + $args \\ + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + recyze: \$(recyze.py --version) + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + // TODO nf-core: A stub section should mimic the execution of the original module as best as possible + // Have a look at the following examples: + // Simple example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bcftools/annotate/main.nf#L47-L63 + // Complex example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bedtools/split/main.nf#L38-L54 + """ + touch ${prefix}.bam + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + roadie: \$(samtools --version |& sed '1!d ; s/samtools //') + END_VERSIONS + """ +} diff --git a/modules/local/roadie/recyze/meta.yml b/modules/local/roadie/recyze/meta.yml new file mode 100644 index 0000000..a3ad2f4 --- /dev/null +++ b/modules/local/roadie/recyze/meta.yml @@ -0,0 +1,58 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "roadie_recyze" +## TODO nf-core: Add a description of the module and list keywords +description: write your description here +keywords: + - sort + - example + - genomics +tools: + - "roadie": + ## TODO nf-core: Add a description and other details for the software below + description: "Channel extraction, crop creation and preprocessing module" + homepage: "https://mcmicro.org" + documentation: "https://mcmicro.org" + tool_dev_url: "https://github.com/labsyspharm/mcmicro" + doi: "10.1038/s41592-021-01308-y" + licence: ["MIT"] + identifier: + +## TODO nf-core: Add a description of all of the variables used as input +input: + # Only when we have meta + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1' ]` + + - image: + type: file + description: Registered multiplexed immunofluorescence image + pattern: "*.{tif,tiff}" + +## TODO nf-core: Add a description of all of the variables used as output +output: + - bam: + #Only when we have meta + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1' ]` + - "*.tif": + type: file + description: Extracted channels for segmentation as (pyramidal) TIFF + pattern: "*.{tif}" + + - versions: + - "versions.yml": + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@kbestak" +maintainers: + - "@kbestak" diff --git a/modules/local/roadie/recyze/tests/main.nf.test b/modules/local/roadie/recyze/tests/main.nf.test new file mode 100644 index 0000000..69d0ec2 --- /dev/null +++ b/modules/local/roadie/recyze/tests/main.nf.test @@ -0,0 +1,65 @@ +// test with nf-core modules test roadie/recyze +nextflow_process { + + name "Test Process ROADIE_RECYZE" + script "../main.nf" + process "ROADIE_RECYZE" + + tag "modules" + tag "modules_" + tag "roadie" + tag "roadie/recyze" + + test("exemplar - tif") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + //TODO nf-core: Add all required assertions to verify the test output. + // See https://nf-co.re/docs/contributing/tutorials/nf-test_assertions for more information and examples. + ) + } + + } + + // TODO nf-core: Change the test name preferably indicating the test-data and file-format used but keep the " - stub" suffix. + test("sarscov2 - bam - stub") { + + options "-stub" + + when { + process { + """ + // TODO nf-core: define inputs of the process here. Example: + + input[0] = [ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + //TODO nf-core: Add all required assertions to verify the test output. + ) + } + + } + +} diff --git a/workflows/mcmicro.nf b/workflows/mcmicro.nf index 5bbe145..e8db13c 100644 --- a/workflows/mcmicro.nf +++ b/workflows/mcmicro.nf @@ -20,6 +20,7 @@ include { COREOGRAPH } from '../modules/nf-core/coreograph/main' include { DEEPCELL_MESMER } from '../modules/nf-core/deepcell/mesmer/main' include { SCIMAP_MCMICRO } from '../modules/nf-core/scimap/mcmicro/main' include { MCQUANT } from '../modules/nf-core/mcquant/main' +include { ROADIE_RECYZE } from'../modules/local/roadie/recyze/main' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -109,6 +110,9 @@ workflow MCMICRO { ch_segmentation_input = post_registration } + ROADIE_RECYZE(ch_segmentation_input) + ROADIE_RECYZE.out.extracted_channels.view() + // Run Segmentation ch_masks = Channel.empty() From fa40c132d90e29a88ad078147d44e24f91a2895c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Fri, 28 Feb 2025 18:25:02 +0000 Subject: [PATCH 02/12] recyze working - added param - validation --- assets/markers-test_roadie.csv | 5 ++++ assets/schema_marker.json | 3 ++ conf/modules.config | 8 +++++ nextflow.config | 1 + nextflow_schema.json | 4 +++ .../utils_nfcore_mcmicro_pipeline/main.nf | 22 +++++++++++--- workflows/mcmicro.nf | 29 +++++++++++++++---- 7 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 assets/markers-test_roadie.csv diff --git a/assets/markers-test_roadie.csv b/assets/markers-test_roadie.csv new file mode 100644 index 0000000..7cc3371 --- /dev/null +++ b/assets/markers-test_roadie.csv @@ -0,0 +1,5 @@ +channel_number,cycle_number,marker_name,segmentation_channel +1,1,DNA 1,TRUE +2,1,Na/K ATPase,TRUE +3,1,CD3, +4,1,CD45RO, diff --git a/assets/schema_marker.json b/assets/schema_marker.json index 835ac33..e49f630 100644 --- a/assets/schema_marker.json +++ b/assets/schema_marker.json @@ -45,6 +45,9 @@ }, "remove": { "type": "boolean" + }, + "segmentation_channel": { + "type": "boolean" } }, "required": ["channel_number", "cycle_number", "marker_name"] diff --git a/conf/modules.config b/conf/modules.config index 54e99d6..c42fd4f 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -44,6 +44,14 @@ process { ] } + withName: ROADIE_RECYZE { + publishDir = [ + path: { "${params.outdir}/roadie" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + withName: "DEEPCELL_MESMER" { ext.prefix = { "mask_${meta.id}" } ext.args = {"--image-mpp=${params.pixel_size ?: 0.65} --nuclear-channel 0 --compartment nuclear"} diff --git a/nextflow.config b/nextflow.config index 97c4a9a..160fc5c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -16,6 +16,7 @@ params { tma_dearray = false pixel_size = null segmentation = 'mesmer' + segmentation_recyze = false cellpose_model = [] // Illumination correction diff --git a/nextflow_schema.json b/nextflow_schema.json index 20df01e..55a3d87 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -75,6 +75,10 @@ "pattern": "^((cellpose|mesmer)?,?)*(? segmentation_channels ?: null } + // Check if the 'segmentation_channel' column exists + if (segmentation_channel_list.isEmpty()) { + error "The 'segmentation_channel' column is missing from the markersheet or all values are null. This column is required when params.segmentation_recyze is given." + } + // Check if at least one value in the 'segmentation_channel' column is TRUE + def hasSegmentationChannel = segmentation_channel_list.any { it?.toString().toLowerCase() == 'true' } + if (!hasSegmentationChannel) { + error "The 'segmentation_channel' column must have at least one TRUE value when params.segmentation_recyze is given." + } + } // uniqueness of (channel, cycle) tuple in marker sheet def test_tuples = [channel_number_list, cycle_number_list].transpose() def dups = test_tuples.countBy{ it }.findAll{ _, count -> count > 1 }*.key @@ -221,9 +235,9 @@ def validateInputMarkersheet( markersheet_data ) { } // validate backsub columns if present - def exposure_list = markersheet_data.findResults{ _1, _2, _3, _4, _5, _6, exposure, _8, _9 -> exposure ?: null } - def background_list = markersheet_data.findResults{ _1, _2, _3, _4, _5, _6, _7, background, _9 -> background ?: null } - def remove_list = markersheet_data.findResults{ _1, _2, _3, _4, _5, _6, _7, _8, remove -> remove ?: null } + def exposure_list = markersheet_data.findResults{ _1, _2, _3, _4, _5, _6, exposure, _8, _9, _10 -> exposure ?: null } + def background_list = markersheet_data.findResults{ _1, _2, _3, _4, _5, _6, _7, background, _9, _10 -> background ?: null } + def remove_list = markersheet_data.findResults{ _1, _2, _3, _4, _5, _6, _7, _8, remove, _10 -> remove ?: null } if (!background_list && (exposure_list || remove_list)) { error("No values in background column, but values in either exposure or remove columns. Must have background column values to perform background subtraction.") @@ -248,7 +262,7 @@ def validateInputMarkersheet( markersheet_data ) { def validateInputSamplesheetMarkersheet ( samples, markers ) { def sample_cycles = samples.collect{ meta, image_tiles, dfp, ffp -> meta.cycle_number } - def marker_cycles = markers.collect{ channel_number, cycle_number, marker_name, _1, _2, _3, _4, _5, _6 -> cycle_number } + def marker_cycles = markers.collect{ channel_number, cycle_number, marker_name, _1, _2, _3, _4, _5, _6, _7 -> cycle_number } if (marker_cycles.unique(false) != sample_cycles.unique(false) ) { error("cycle_number values must match between sample and marker sheets") diff --git a/workflows/mcmicro.nf b/workflows/mcmicro.nf index e8db13c..58ada1e 100644 --- a/workflows/mcmicro.nf +++ b/workflows/mcmicro.nf @@ -110,14 +110,31 @@ workflow MCMICRO { ch_segmentation_input = post_registration } - ROADIE_RECYZE(ch_segmentation_input) - ROADIE_RECYZE.out.extracted_channels.view() + ch_roadie_channels = ch_markersheet + .flatMap { row -> + row.collect { channel_number, _2, _3, _4, _5, _6, _7, _8, _9, segmentation_channel -> + segmentation_channel ? (channel_number - 1) : null + } + }.view() + + if (params.segmentation_recyze) { + ch_roadie_channels = ch_markersheet + .flatMap { row -> + row.collect { channel_number, _2, _3, _4, _5, _6, _7, _8, _9, segmentation_channel -> + segmentation_channel ? (channel_number - 1) : null + }.findAll { it != null } + }.collect() + ROADIE_RECYZE(ch_segmentation_input, ch_roadie_channels) + ch_segmentation_input_extracted = ROADIE_RECYZE.out.extracted_channels + } else { + ch_segmentation_input_extracted = ch_segmentation_input + } // Run Segmentation ch_masks = Channel.empty() - ch_segmentation_input + ch_segmentation_input_extracted .multiMap{ meta, image -> img: [meta + [segmenter: 'mesmer'], image] membrane_img: [[:], []] @@ -126,7 +143,7 @@ workflow MCMICRO { ch_masks = ch_masks.mix(DEEPCELL_MESMER.out.mask) ch_versions = ch_versions.mix(DEEPCELL_MESMER.out.versions) - ch_segmentation_input + ch_segmentation_input_extracted .multiMap{ meta, image -> image: [meta + [segmenter: 'cellpose'], image] model: params.cellpose_model @@ -141,12 +158,12 @@ workflow MCMICRO { ch_mcquant_markers = ch_markersheet .flatMap{ ['marker_name'] + - it.collect{ _1, _2, marker_name, _4, _5, _6, _7, _8, _9 -> '"' + marker_name + '"' } + it.collect{ _1, _2, marker_name, _4, _5, _6, _7, _8, _9, _10 -> '"' + marker_name + '"' } } .dump(tag: "MARKERS") .collectFile(name: 'markers.csv', sort: false, newLine: true) - ch_segmentation_input + ch_segmentation_input // using post-registration/post-TMA input .cross(ch_masks) { it[0]['id'] } .map{ t_ashlar, t_mask -> [t_mask[0], t_ashlar[1], t_mask[1]] } .combine(ch_mcquant_markers) From 53080ed54ca21dccf56c4ef79e40254ff736ff25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Fri, 28 Feb 2025 18:25:29 +0000 Subject: [PATCH 03/12] removed when due to deprecation --- modules/local/roadie/recyze/main.nf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/local/roadie/recyze/main.nf b/modules/local/roadie/recyze/main.nf index b76f8b6..2af8ab9 100644 --- a/modules/local/roadie/recyze/main.nf +++ b/modules/local/roadie/recyze/main.nf @@ -6,23 +6,22 @@ process ROADIE_RECYZE { input: tuple val(meta), path(image) + val(channels) output: tuple val(meta), path("*.tif"), emit: extracted_channels path "versions.yml" , emit: versions - when: - task.ext.when == null || task.ext.when - script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + def channels_str = channels.join(' ') """ recyze.py \\ --in ${image} \\ --out ${prefix}.tif \\ - --channels 0 \\ + --channels ${channels_str} \\ --num-threads $task.cpus \\ $args \\ From 8b96bfbdc3be016838a4afbfa7523cb3cfc48491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Fri, 28 Feb 2025 18:31:05 +0000 Subject: [PATCH 04/12] added usage documentation --- docs/usage.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index cef5863..eec77b1 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -80,11 +80,12 @@ channel_number,cycle_number,marker_name ### optional markersheet columns -| Column | Description | -| ----------------------- | ---------------------------------------------- | -| `filter` | Microscope filter common name. | -| `excitation_wavelength` | Excitation wavelength for this channel, in nm. | -| `emission_wavelength` | Emission wavelength for this channel, in nm. | +| Column | Description | +| ----------------------- | ---------------------------------------------------------------------- | +| `filter` | Microscope filter common name. | +| `excitation_wavelength` | Excitation wavelength for this channel, in nm. | +| `emission_wavelength` | Emission wavelength for this channel, in nm. | +| `segmentation_channel` | Boolean specifying whether the marker should be used for segmentation. | ## Running the pipeline @@ -157,6 +158,10 @@ This is an optional step that occurs immediately following registration. It is t This is an optional step that occurs immediately following background subtration if that optional step was run or after registration if is was not. It is triggered by the `--tma_dearray` flag. When this flag is selected, the coreograph module is run on the output from either the background subtraction step or the registration step if background subtration was not performed. Coreograph separates the input image into a set of images for each of the cores. It uses UNet, a deep learning model, to identify complete/incomplete tissue cores on a tissue microarray. It has been trained on 9 TMA slides of different sizes and tissue types. More information about it can be found on the [coreograph nf-core module website](https://nf-co.re/modules/coreograph/) +#### Channel extraction for segmentation + +Channel extraction from preprocessed images that can be passed to segmentation tools can be done using `recyze` by adding the `--segmentation_recyze` parameter and specifying channels to be extracted in the markersheet. By default, the whole stack is passed to segmentation tools. + #### Segmentation This is a required step that follows the TMA Core Separation step. The workflow will run the deepcell_mesmer module by default, but other options are available by using the `--segmentation` flag. The flag should be followed by a single segmentation module name or a comma separated list of names to run multiple segmentation modules in parallel. The available options currently supported are `mesmer` and `cellpose`. More information about each of these modules can be found on their respective nf-core module websites: [deepcell_mesmer](https://nf-co.re/modules/deepcell_mesmer/) [cellpose](https://nf-co.re/modules/cellpose/) From da873e77968084fe7b6dca70025ef07cc66ea477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Fri, 28 Feb 2025 18:35:28 +0000 Subject: [PATCH 05/12] passing linting --- .github/workflows/linting_comment.yml | 2 +- .../local/roadie/recyze/tests/main.nf.test | 30 ++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 0bed96d..95b6b6a 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 + uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8 with: workflow: linting.yml workflow_conclusion: completed diff --git a/modules/local/roadie/recyze/tests/main.nf.test b/modules/local/roadie/recyze/tests/main.nf.test index 69d0ec2..5a029eb 100644 --- a/modules/local/roadie/recyze/tests/main.nf.test +++ b/modules/local/roadie/recyze/tests/main.nf.test @@ -10,14 +10,23 @@ nextflow_process { tag "roadie" tag "roadie/recyze" - test("exemplar - tif") { + test("exemplar - tif - ASHLAR preprocessing") { when { process { + // TODO when test data available """ + input[0] = ASHLAR( + [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) + ] + ) + + // Then, use ASHLAR output for ROADIE_RECYZE input[0] = [ - [ id:'test' ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + input[0][0], // meta map from ASHLAR output + input[0][1] // file from ASHLAR output ] """ } @@ -35,18 +44,24 @@ nextflow_process { } // TODO nf-core: Change the test name preferably indicating the test-data and file-format used but keep the " - stub" suffix. - test("sarscov2 - bam - stub") { + test("exemplar - tif - stub") { options "-stub" when { process { """ - // TODO nf-core: define inputs of the process here. Example: + input[0] = ASHLAR( + [ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) + ] + ) + // Then, use ASHLAR output for ROADIE_RECYZE input[0] = [ - [ id:'test', single_end:false ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true), + input[0][0], // meta map from ASHLAR output + input[0][1] // file from ASHLAR output ] """ } @@ -56,7 +71,6 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot(process.out).match() } - //TODO nf-core: Add all required assertions to verify the test output. ) } From 74f2ccca96945683457b9dd3040f2f3245620d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Fri, 7 Mar 2025 10:28:40 +0000 Subject: [PATCH 06/12] updated tests and added recyze test --- tests/main.nf.test | 191 ++++++++++++++++++++++++++++----------------- 1 file changed, 118 insertions(+), 73 deletions(-) diff --git a/tests/main.nf.test b/tests/main.nf.test index 2328980..361e6b4 100644 --- a/tests/main.nf.test +++ b/tests/main.nf.test @@ -24,10 +24,10 @@ nextflow_workflow { ) input[1] = Channel.of( [ - [1,1,'DNA_6',[],[],[],[],[],[]], - [2,1,'ELANE',[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]], - [4,1,'CD45',[],[],[],[],[],[]], + [1,1,'DNA_6',[],[],[],[],[],[],[]], + [2,1,'ELANE',[],[],[],[],[],[],[]], + [3,1,'CD57',[],[],[],[],[],[]],[], + [4,1,'CD45',[],[],[],[],[],[],[]], ], ) """ @@ -68,10 +68,10 @@ nextflow_workflow { ) input[1] = Channel.of( [ - [1,1,'DNA_6',[],[],[],[],[],[]], - [2,1,'ELANE',[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]], - [4,1,'CD45',[],[],[],[],[],[]], + [1,1,'DNA_6',[],[],[],[],[],[],[]], + [2,1,'ELANE',[],[],[],[],[],[],[]], + [3,1,'CD57',[],[],[],[],[],[]],[], + [4,1,'CD45',[],[],[],[],[],[],[]], ], ) """ @@ -113,10 +113,10 @@ nextflow_workflow { ) input[1] = Channel.of( [ - [1,1,'DNA_6',[],[],[],[],[],[]], - [2,1,'ELANE',[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]], - [4,1,'CD45',[],[],[],[],[],[]], + [1,1,'DNA_6',[],[],[],[],[],[],[]], + [2,1,'ELANE',[],[],[],[],[],[],[]], + [3,1,'CD57',[],[],[],[],[],[]],[], + [4,1,'CD45',[],[],[],[],[],[],[]], ], ) """ @@ -167,18 +167,18 @@ nextflow_workflow { ) input[1] = Channel.of( [ - [1,1,'DNA_6',[],[],[],[],[],[]], - [2,1,'ELANE',[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]], - [4,1,'CD45',[],[],[],[],[],[]], - [5,1,'DNA_7',[],[],[],[],[],[]], - [6,1,'ELANE7',[],[],[],[],[],[]], - [7,1,'CD577',[],[],[],[],[],[]], - [8,1,'CD457',[],[],[],[],[],[]], - [9,1,'DNA_8',[],[],[],[],[],[]], - [10,1,'ELANE8',[],[],[],[],[],[]], - [11,1,'CD578',[],[],[],[],[],[]], - [12,1,'CD458',[],[],[],[],[],[]], + [1,1,'DNA_6',[],[],[],[],[],[],[]], + [2,1,'ELANE',[],[],[],[],[],[],[]], + [3,1,'CD57',[],[],[],[],[],[]],[], + [4,1,'CD45',[],[],[],[],[],[],[]], + [5,1,'DNA_7',[],[],[],[],[],[],[]], + [6,1,'ELANE7',[],[],[],[],[],[],[]], + [7,1,'CD577',[],[],[],[],[],[],[]], + [8,1,'CD457',[],[],[],[],[],[],[]], + [9,1,'DNA_8',[],[],[],[],[],[],[]], + [10,1,'ELANE8',[],[],[],[],[],[],[]], + [11,1,'CD578',[],[],[],[],[],[],[]], + [12,1,'CD458',[],[],[],[],[],[],[]], ], ) """ @@ -235,14 +235,14 @@ nextflow_workflow { ) input[1] = Channel.of( [ - [1,1,'DNA_6',[],[],[],[],[],[]], - [2,1,'ELANE',[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]], - [4,1,'CD45',[],[],[],[],[],[]], - [5,1,'DNA_7',[],[],[],[],[],[]], - [6,1,'ELANE7',[],[],[],[],[],[]], - [7,1,'CD577',[],[],[],[],[],[]], - [8,1,'CD457',[],[],[],[],[],[]], + [1,1,'DNA_6',[],[],[],[],[],[],[]], + [2,1,'ELANE',[],[],[],[],[],[],[]], + [3,1,'CD57',[],[],[],[],[],[]],[], + [4,1,'CD45',[],[],[],[],[],[],[]], + [5,1,'DNA_7',[],[],[],[],[],[],[]], + [6,1,'ELANE7',[],[],[],[],[],[],[]], + [7,1,'CD577',[],[],[],[],[],[],[]], + [8,1,'CD457',[],[],[],[],[],[],[]], ], ) """ @@ -302,14 +302,14 @@ nextflow_workflow { ) input[1] = Channel.of( [ - [1,1,'DNA_6',[],[],[],[],[],[]], - [2,1,'ELANE',[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]], - [4,1,'CD45',[],[],[],[],[],[]], - [5,1,'DNA_7',[],[],[],[],[],[]], - [6,1,'ELANE7',[],[],[],[],[],[]], - [7,1,'CD577',[],[],[],[],[],[]], - [8,1,'CD457',[],[],[],[],[],[]], + [1,1,'DNA_6',[],[],[],[],[],[],[]], + [2,1,'ELANE',[],[],[],[],[],[],[]], + [3,1,'CD57',[],[],[],[],[],[]],[], + [4,1,'CD45',[],[],[],[],[],[],[]], + [5,1,'DNA_7',[],[],[],[],[],[],[]], + [6,1,'ELANE7',[],[],[],[],[],[],[]], + [7,1,'CD577',[],[],[],[],[],[],[]], + [8,1,'CD457',[],[],[],[],[],[],[]], ], ) """ @@ -368,18 +368,18 @@ nextflow_workflow { ) input[1] = Channel.of( [ - [1,1,'DNA_6',[],[],[],[],[],[]], - [2,1,'ELANE',[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]], - [4,1,'CD45',[],[],[],[],[],[]], - [5,1,'DNA_7',[],[],[],[],[],[]], - [6,1,'ELANE7',[],[],[],[],[],[]], - [7,1,'CD577',[],[],[],[],[],[]], - [8,1,'CD457',[],[],[],[],[],[]], - [9,1,'DNA_8',[],[],[],[],[],[]], - [10,1,'ELANE8',[],[],[],[],[],[]], - [11,1,'CD578',[],[],[],[],[],[]], - [12,1,'CD458',[],[],[],[],[],[]], + [1,1,'DNA_6',[],[],[],[],[],[],[]], + [2,1,'ELANE',[],[],[],[],[],[],[]], + [3,1,'CD57',[],[],[],[],[],[]],[], + [4,1,'CD45',[],[],[],[],[],[],[]], + [5,1,'DNA_7',[],[],[],[],[],[],[]], + [6,1,'ELANE7',[],[],[],[],[],[],[]], + [7,1,'CD577',[],[],[],[],[],[],[]], + [8,1,'CD457',[],[],[],[],[],[],[]], + [9,1,'DNA_8',[],[],[],[],[],[],[]], + [10,1,'ELANE8',[],[],[],[],[],[],[]], + [11,1,'CD578',[],[],[],[],[],[],[]], + [12,1,'CD458',[],[],[],[],[],[],[]], ], ) """ @@ -443,14 +443,14 @@ nextflow_workflow { ) input[1] = Channel.of( [ - [1,1,'DNA_6',[],[],[],[],[],[]], - [2,1,'ELANE',[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]], - [4,1,'CD45',[],[],[],[],[],[]], - [5,1,'DNA_7',[],[],[],[],[],[]], - [6,1,'ELANE7',[],[],[],[],[],[]], - [7,1,'CD577',[],[],[],[],[],[]], - [8,1,'CD457',[],[],[],[],[],[]], + [1,1,'DNA_6',[],[],[],[],[],[],[]], + [2,1,'ELANE',[],[],[],[],[],[],[]], + [3,1,'CD57',[],[],[],[],[],[]],[], + [4,1,'CD45',[],[],[],[],[],[],[]], + [5,1,'DNA_7',[],[],[],[],[],[],[]], + [6,1,'ELANE7',[],[],[],[],[],[],[]], + [7,1,'CD577',[],[],[],[],[],[],[]], + [8,1,'CD457',[],[],[],[],[],[],[]], ], ) """ @@ -498,10 +498,10 @@ nextflow_workflow { ) input[1] = Channel.of( [ - [1,1,'DNA_6',[],[],[],[],[],[]], - [2,1,'ELANE',[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]], - [4,1,'CD45',[],[],[],[],[],[]], + [1,1,'DNA_6',[],[],[],[],[],[],[]], + [2,1,'ELANE',[],[],[],[],[],[],[]], + [3,1,'CD57',[],[],[],[],[],[]],[], + [4,1,'CD45',[],[],[],[],[],[],[]], ], ) """ @@ -544,7 +544,7 @@ nextflow_workflow { ) input[1] = Channel.of( [ - [1,1,'DNA_6',[],[],[],[],[],[]], + [1,1,'DNA_6',[],[],[],[],[],[],[]], ], ) """ @@ -590,14 +590,14 @@ nextflow_workflow { ) input[1] = Channel.of( [ - [1,1,'DNA 1',[],[],[],100,[],[]], - [2,1,'Na/K ATPase',[],[],[],100,[],[]], - [3,1,'CD3',[],[],[],100,[],[]], - [4,1,'CD45RO',[],[],[],100,[],[]], - [5,2,'DNA 2',[],[],[],100,[],[]], - [6,2,'Antigen Ki67',[],[],[],100,[],[]], - [7,2,'Pan-cytokeratin',[],[],[],100,[],[]], - [8,2,'Aortic smooth muscle actin',[],[],[],100,[],[]] + [1,1,'DNA 1',[],[],[],100,[],[],[]], + [2,1,'Na/K ATPase',[],[],[],100,[],[],[]], + [3,1,'CD3',[],[],[],100,[],[],[]], + [4,1,'CD45RO',[],[],[],100,[],[],[]], + [5,2,'DNA 2',[],[],[],100,[],[],[]], + [6,2,'Antigen Ki67',[],[],[],100,[],[],[]], + [7,2,'Pan-cytokeratin',[],[],[],100,[],[],[]], + [8,2,'Aortic smooth muscle actin',[],[],[],100,[],[],[]] ], ) """ @@ -617,6 +617,51 @@ nextflow_workflow { { assert workflow.success } ) } + }, + + test("cycle: no illumination correction, cellpose, recyze") { + + when { + params { + segmentation = "cellpose" + segmentation_recyze = true + + } + workflow { + """ + input[0] = Channel.of( + [ + [id:"TEST1", cycle_number:1, channel_count:4], + "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif", + [], + [], + ], + ) + input[1] = Channel.of( + [ + [1,1,'DNA_6',[],[],[],[],[],[],true], + [2,1,'ELANE',[],[],[],[],[],[],[]], + [3,1,'CD57',[],[],[],[],[],[]],[], + [4,1,'CD45',[],[],[],[],[],[],[]], + ], + ) + """ + } + } + + then { + assertAll ( + { + assert snapshot ( + path("$outputDir/registration/ashlar/TEST1.ome.tif"), + path("$outputDir/roadie/TEST1.tif"), + path("$outputDir/segmentation/cellpose/TEST1.ome_cp_masks.tif"), + CsvUtils.roundAndHashCsv("$outputDir/quantification/mcquant/cellpose/TEST1_TEST1.csv"), + ).match() + }, + { assert workflow.success } + ) + } }, From f79902a1bf3635b54b342e9040bc9f5ea30318a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Fri, 7 Mar 2025 10:39:02 +0000 Subject: [PATCH 07/12] fix typo --- tests/main.nf.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/main.nf.test b/tests/main.nf.test index 361e6b4..6603bdb 100644 --- a/tests/main.nf.test +++ b/tests/main.nf.test @@ -115,7 +115,7 @@ nextflow_workflow { [ [1,1,'DNA_6',[],[],[],[],[],[],[]], [2,1,'ELANE',[],[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]],[], + [3,1,'CD57',[],[],[],[],[],[],[]], [4,1,'CD45',[],[],[],[],[],[],[]], ], ) @@ -169,7 +169,7 @@ nextflow_workflow { [ [1,1,'DNA_6',[],[],[],[],[],[],[]], [2,1,'ELANE',[],[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]],[], + [3,1,'CD57',[],[],[],[],[],[],[]], [4,1,'CD45',[],[],[],[],[],[],[]], [5,1,'DNA_7',[],[],[],[],[],[],[]], [6,1,'ELANE7',[],[],[],[],[],[],[]], From 330a873a09b8a4d168df6690733f10857c7543e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Fri, 7 Mar 2025 12:41:15 +0000 Subject: [PATCH 08/12] nf-test for recyze, remove uuid --- bin/recyze.py | 3 +- modules/local/roadie/recyze/main.nf | 13 +-- .../local/roadie/recyze/tests/main.nf.test | 102 ++++++++++++------ .../roadie/recyze/tests/main.nf.test.snap | 101 +++++++++++++++++ 4 files changed, 178 insertions(+), 41 deletions(-) create mode 100644 modules/local/roadie/recyze/tests/main.nf.test.snap diff --git a/bin/recyze.py b/bin/recyze.py index 0666cbd..aed893a 100755 --- a/bin/recyze.py +++ b/bin/recyze.py @@ -260,7 +260,7 @@ def run(self): print() self.metadata.images[0].pixels.channels = [self.metadata.images[0].pixels.channels[i] for i in self.channels] - self.metadata.uuid = uuid.uuid4().urn + #self.metadata.uuid = uuid.uuid4().urn self.metadata.images[0].pixels.size_c = self.num_channels self.metadata.images[0].pixels.size_x = self.width self.metadata.images[0].pixels.size_y = self.height @@ -308,6 +308,7 @@ def run(self): '--num-threads', type=int, required=False, default=0, metavar="N", help="Worker thread count (Default: auto-scale based on number of available CPUs)", ) + parser.add_argument('--version', action='version', version='2.0.0dev') args = parser.parse_args() # Automatically infer the output filename, if not specified diff --git a/modules/local/roadie/recyze/main.nf b/modules/local/roadie/recyze/main.nf index 2af8ab9..dc31f74 100644 --- a/modules/local/roadie/recyze/main.nf +++ b/modules/local/roadie/recyze/main.nf @@ -13,8 +13,8 @@ process ROADIE_RECYZE { path "versions.yml" , emit: versions script: - def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" def channels_str = channels.join(' ') """ @@ -32,18 +32,13 @@ process ROADIE_RECYZE { """ stub: - def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - // TODO nf-core: A stub section should mimic the execution of the original module as best as possible - // Have a look at the following examples: - // Simple example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bcftools/annotate/main.nf#L47-L63 - // Complex example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bedtools/split/main.nf#L38-L54 """ - touch ${prefix}.bam + touch ${prefix}.tif cat <<-END_VERSIONS > versions.yml "${task.process}": - roadie: \$(samtools --version |& sed '1!d ; s/samtools //') + roadie: \$(recyze.py --version) END_VERSIONS """ } diff --git a/modules/local/roadie/recyze/tests/main.nf.test b/modules/local/roadie/recyze/tests/main.nf.test index 5a029eb..88bb680 100644 --- a/modules/local/roadie/recyze/tests/main.nf.test +++ b/modules/local/roadie/recyze/tests/main.nf.test @@ -10,24 +10,63 @@ nextflow_process { tag "roadie" tag "roadie/recyze" - test("exemplar - tif - ASHLAR preprocessing") { + test("exemplar - tif - 1 channel - ASHLAR setup") { + setup { + run("ASHLAR") { + script "../../../../nf-core/ashlar" + process { + """ + input[0] = [ + [ id:'test' ], + "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif" + ] + input[1] = [] + input[2] = [] + """ + } + } + } when { process { - // TODO when test data available """ - input[0] = ASHLAR( - [ - [ id:'test' ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) - ] - ) + input[0] = ASHLAR.out.tif + input[1] = [0] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( process.out ).match() } + ) + } + + } - // Then, use ASHLAR output for ROADIE_RECYZE - input[0] = [ - input[0][0], // meta map from ASHLAR output - input[0][1] // file from ASHLAR output - ] + test("exemplar - tif - 2 channels - ASHLAR setup") { + + setup { + run("ASHLAR") { + script "../../../../nf-core/ashlar" + process { + """ + input[0] = [ + [ id:'test' ], + "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif" + ] + input[1] = [] + input[2] = [] + """ + } + } + } + when { + process { + """ + input[0] = ASHLAR.out.tif + input[1] = [0,1] """ } } @@ -35,9 +74,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } - //TODO nf-core: Add all required assertions to verify the test output. - // See https://nf-co.re/docs/contributing/tutorials/nf-test_assertions for more information and examples. + { assert snapshot( process.out ).match() } ) } @@ -48,21 +85,26 @@ nextflow_process { options "-stub" + setup { + run("ASHLAR") { + script "../../../../nf-core/ashlar" + process { + """ + input[0] = [ + [ id:'test' ], + "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/imaging/ome-tiff/cycif-tonsil-cycle1.ome.tif" + ] + input[1] = [] + input[2] = [] + """ + } + } + } when { process { """ - input[0] = ASHLAR( - [ - [ id:'test' ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) - ] - ) - - // Then, use ASHLAR output for ROADIE_RECYZE - input[0] = [ - input[0][0], // meta map from ASHLAR output - input[0][1] // file from ASHLAR output - ] + input[0] = ASHLAR.out.tif + input[1] = 1 """ } } @@ -70,10 +112,8 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( process.out ).match() } ) } - } - } diff --git a/modules/local/roadie/recyze/tests/main.nf.test.snap b/modules/local/roadie/recyze/tests/main.nf.test.snap new file mode 100644 index 0000000..2fff8bd --- /dev/null +++ b/modules/local/roadie/recyze/tests/main.nf.test.snap @@ -0,0 +1,101 @@ +{ + "exemplar - tif - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.tif:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,dbd77571d8e21146df95814f31385544" + ], + "extracted_channels": [ + [ + { + "id": "test" + }, + "test.tif:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,dbd77571d8e21146df95814f31385544" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.5" + }, + "timestamp": "2025-03-07T12:19:02.506780911" + }, + "exemplar - tif - 2 channels - ASHLAR setup": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.tif:md5,a30d0f098ed8ccfa63cdaa95d89f2560" + ] + ], + "1": [ + "versions.yml:md5,7a394d888025b1e38544241e1df21069" + ], + "extracted_channels": [ + [ + { + "id": "test" + }, + "test.tif:md5,a30d0f098ed8ccfa63cdaa95d89f2560" + ] + ], + "versions": [ + "versions.yml:md5,7a394d888025b1e38544241e1df21069" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.5" + }, + "timestamp": "2025-03-07T12:36:55.25820006" + }, + "exemplar - tif - 1 channel - ASHLAR setup": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.tif:md5,cf1636501b5acd999a501e5e38f1abb0" + ] + ], + "1": [ + "versions.yml:md5,7a394d888025b1e38544241e1df21069" + ], + "extracted_channels": [ + [ + { + "id": "test" + }, + "test.tif:md5,cf1636501b5acd999a501e5e38f1abb0" + ] + ], + "versions": [ + "versions.yml:md5,7a394d888025b1e38544241e1df21069" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.5" + }, + "timestamp": "2025-03-07T12:36:22.386840514" + } +} \ No newline at end of file From fece2333075c610f160c0e90eac75e38945a1338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Fri, 7 Mar 2025 12:51:41 +0000 Subject: [PATCH 09/12] fixed brackets --- .github/workflows/linting_comment.yml | 2 +- tests/main.nf.test | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 95b6b6a..0bed96d 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8 + uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 with: workflow: linting.yml workflow_conclusion: completed diff --git a/tests/main.nf.test b/tests/main.nf.test index 6603bdb..a487378 100644 --- a/tests/main.nf.test +++ b/tests/main.nf.test @@ -26,7 +26,7 @@ nextflow_workflow { [ [1,1,'DNA_6',[],[],[],[],[],[],[]], [2,1,'ELANE',[],[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]],[], + [3,1,'CD57',[],[],[],[],[],[],[]], [4,1,'CD45',[],[],[],[],[],[],[]], ], ) @@ -70,7 +70,7 @@ nextflow_workflow { [ [1,1,'DNA_6',[],[],[],[],[],[],[]], [2,1,'ELANE',[],[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]],[], + [3,1,'CD57',[],[],[],[],[],[],[]], [4,1,'CD45',[],[],[],[],[],[],[]], ], ) @@ -237,7 +237,7 @@ nextflow_workflow { [ [1,1,'DNA_6',[],[],[],[],[],[],[]], [2,1,'ELANE',[],[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]],[], + [3,1,'CD57',[],[],[],[],[],[],[]], [4,1,'CD45',[],[],[],[],[],[],[]], [5,1,'DNA_7',[],[],[],[],[],[],[]], [6,1,'ELANE7',[],[],[],[],[],[],[]], @@ -304,7 +304,7 @@ nextflow_workflow { [ [1,1,'DNA_6',[],[],[],[],[],[],[]], [2,1,'ELANE',[],[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]],[], + [3,1,'CD57',[],[],[],[],[],[],[]], [4,1,'CD45',[],[],[],[],[],[],[]], [5,1,'DNA_7',[],[],[],[],[],[],[]], [6,1,'ELANE7',[],[],[],[],[],[],[]], @@ -370,7 +370,7 @@ nextflow_workflow { [ [1,1,'DNA_6',[],[],[],[],[],[],[]], [2,1,'ELANE',[],[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]],[], + [3,1,'CD57',[],[],[],[],[],[],[]], [4,1,'CD45',[],[],[],[],[],[],[]], [5,1,'DNA_7',[],[],[],[],[],[],[]], [6,1,'ELANE7',[],[],[],[],[],[],[]], @@ -445,7 +445,7 @@ nextflow_workflow { [ [1,1,'DNA_6',[],[],[],[],[],[],[]], [2,1,'ELANE',[],[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]],[], + [3,1,'CD57',[],[],[],[],[],[],[]], [4,1,'CD45',[],[],[],[],[],[],[]], [5,1,'DNA_7',[],[],[],[],[],[],[]], [6,1,'ELANE7',[],[],[],[],[],[],[]], @@ -500,7 +500,7 @@ nextflow_workflow { [ [1,1,'DNA_6',[],[],[],[],[],[],[]], [2,1,'ELANE',[],[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]],[], + [3,1,'CD57',[],[],[],[],[],[],[]], [4,1,'CD45',[],[],[],[],[],[],[]], ], ) @@ -641,7 +641,7 @@ nextflow_workflow { [ [1,1,'DNA_6',[],[],[],[],[],[],true], [2,1,'ELANE',[],[],[],[],[],[],[]], - [3,1,'CD57',[],[],[],[],[],[]],[], + [3,1,'CD57',[],[],[],[],[],[],[]], [4,1,'CD45',[],[],[],[],[],[],[]], ], ) From ffb8cbdd404d98e047713a63d7fd5b6ec5017560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Sun, 9 Mar 2025 15:10:35 +0000 Subject: [PATCH 10/12] updated tests --- tests/main.nf.test | 4 ++-- tests/main.nf.test.snap | 3 ++- workflows/mcmicro.nf | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/main.nf.test b/tests/main.nf.test index a487378..6f02947 100644 --- a/tests/main.nf.test +++ b/tests/main.nf.test @@ -514,7 +514,7 @@ nextflow_workflow { assert snapshot ( path("$outputDir/registration/ashlar/TEST1.ome.tif"), path("$outputDir/segmentation/cellpose/TEST1.ome_cp_masks.tif"), - CsvUtils.roundAndHashCsv("$outputDir/quantification/mcquant/cellpose/TEST1_TEST1.csv"), + CsvUtils.roundAndHashCsv("$outputDir/quantification/mcquant/cellpose/TEST1_TEST1_cp_masks.csv"), ).match() }, { assert workflow.success } @@ -656,7 +656,7 @@ nextflow_workflow { path("$outputDir/registration/ashlar/TEST1.ome.tif"), path("$outputDir/roadie/TEST1.tif"), path("$outputDir/segmentation/cellpose/TEST1.ome_cp_masks.tif"), - CsvUtils.roundAndHashCsv("$outputDir/quantification/mcquant/cellpose/TEST1_TEST1.csv"), + CsvUtils.roundAndHashCsv("$outputDir/quantification/mcquant/cellpose/TEST1_TEST1_cp_masks.csv"), ).match() }, { assert workflow.success } diff --git a/tests/main.nf.test.snap b/tests/main.nf.test.snap index e67b011..f06608c 100644 --- a/tests/main.nf.test.snap +++ b/tests/main.nf.test.snap @@ -215,6 +215,7 @@ "content": [ "TEST1.ome.tif:md5,988bbb2c74d47dc22a9f3ac348f53ef5", "TEST1.ome_cp_masks.tif:md5,47bd29c261db5fe062ee35c1b5daccc1", + "TEST1.tif:md5,a30d0f098ed8ccfa63cdaa95d89f2560", "TEST1_TEST1.csv:rounded:md5,42ecd15013bea6dbea4e0343c7b0bef7" ], "meta": { @@ -676,4 +677,4 @@ }, "timestamp": "2024-08-19T11:45:01.117904632" } -} \ No newline at end of file +} diff --git a/workflows/mcmicro.nf b/workflows/mcmicro.nf index 58ada1e..527a288 100644 --- a/workflows/mcmicro.nf +++ b/workflows/mcmicro.nf @@ -78,7 +78,7 @@ workflow MCMICRO { if (params.backsub) { ch_backsub_markers = ch_markersheet .map { ['channel_number,cycle_number,marker_name,exposure,background,remove', - it.collect{ channel_number, cycle_number, marker_name, _1, _2, _3, exposure, background, remove -> + it.collect{ channel_number, cycle_number, marker_name, _1, _2, _3, exposure, background, remove, _10 -> channel_number + "," + cycle_number + "," + marker_name + "," + exposure + "," + background + "," + remove}] } .flatten() .map { it.replace('[]', '') } @@ -115,7 +115,7 @@ workflow MCMICRO { row.collect { channel_number, _2, _3, _4, _5, _6, _7, _8, _9, segmentation_channel -> segmentation_channel ? (channel_number - 1) : null } - }.view() + } if (params.segmentation_recyze) { ch_roadie_channels = ch_markersheet From e4ee02706dab51324270f4e53b2e942f6f75813e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Sun, 9 Mar 2025 15:17:06 +0000 Subject: [PATCH 11/12] updated workflow test --- tests/workflows/mcmicro.nf.test | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/workflows/mcmicro.nf.test b/tests/workflows/mcmicro.nf.test index dabfdee..086a694 100644 --- a/tests/workflows/mcmicro.nf.test +++ b/tests/workflows/mcmicro.nf.test @@ -22,10 +22,10 @@ nextflow_workflow { ) input[1] = Channel.of( [ - [1,1,'DNA 1',[],[],[],[],[],[]], - [2,1,'Na/K ATPase',[],[],[],[],[],[]], - [3,1,'CD3',[],[],[],[],[],[]], - [4,1,'CD45RO',[],[],[],[],[],[]], + [1,1,'DNA 1',[],[],[],[],[],[],[]], + [2,1,'Na/K ATPase',[],[],[],[],[],[],[]], + [3,1,'CD3',[],[],[],[],[],[],[]], + [4,1,'CD45RO',[],[],[],[],[],[],[]], ], ) """ From df625b836c43c57c94d8822bc35efa933559c195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Sun, 9 Mar 2025 22:06:55 +0000 Subject: [PATCH 12/12] working tests --- tests/main.nf.test | 4 ++-- tests/main.nf.test.snap | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/main.nf.test b/tests/main.nf.test index 6f02947..40bf5c0 100644 --- a/tests/main.nf.test +++ b/tests/main.nf.test @@ -514,7 +514,7 @@ nextflow_workflow { assert snapshot ( path("$outputDir/registration/ashlar/TEST1.ome.tif"), path("$outputDir/segmentation/cellpose/TEST1.ome_cp_masks.tif"), - CsvUtils.roundAndHashCsv("$outputDir/quantification/mcquant/cellpose/TEST1_TEST1_cp_masks.csv"), + CsvUtils.roundAndHashCsv("$outputDir/quantification/mcquant/cellpose/TEST1_TEST1.csv"), ).match() }, { assert workflow.success } @@ -655,7 +655,7 @@ nextflow_workflow { assert snapshot ( path("$outputDir/registration/ashlar/TEST1.ome.tif"), path("$outputDir/roadie/TEST1.tif"), - path("$outputDir/segmentation/cellpose/TEST1.ome_cp_masks.tif"), + path("$outputDir/segmentation/cellpose/TEST1_cp_masks.tif"), CsvUtils.roundAndHashCsv("$outputDir/quantification/mcquant/cellpose/TEST1_TEST1_cp_masks.csv"), ).match() }, diff --git a/tests/main.nf.test.snap b/tests/main.nf.test.snap index f06608c..78ce1b2 100644 --- a/tests/main.nf.test.snap +++ b/tests/main.nf.test.snap @@ -215,7 +215,6 @@ "content": [ "TEST1.ome.tif:md5,988bbb2c74d47dc22a9f3ac348f53ef5", "TEST1.ome_cp_masks.tif:md5,47bd29c261db5fe062ee35c1b5daccc1", - "TEST1.tif:md5,a30d0f098ed8ccfa63cdaa95d89f2560", "TEST1_TEST1.csv:rounded:md5,42ecd15013bea6dbea4e0343c7b0bef7" ], "meta": { @@ -386,6 +385,19 @@ }, "timestamp": "2024-08-14T13:31:24.583251357" }, + "cycle: no illumination correction, cellpose, recyze": { + "content": [ + "TEST1.ome.tif:md5,988bbb2c74d47dc22a9f3ac348f53ef5", + "TEST1.tif:md5,cf1636501b5acd999a501e5e38f1abb0", + "TEST1_cp_masks.tif:md5,1c073bdb535a0aaa5abc139081e38e91", + "TEST1_TEST1_cp_masks.csv:rounded:md5,2fefbee619edbe133569c1d6e91c9752" + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.5" + }, + "timestamp": "2025-03-09T22:04:09.816002225" + }, "cycle: multiple file ashlar input with multiple samples and basicpy correction": { "content": [ { @@ -677,4 +689,4 @@ }, "timestamp": "2024-08-19T11:45:01.117904632" } -} +} \ No newline at end of file