diff --git a/BRB/PushButton.py b/BRB/PushButton.py index b3744d4..9ca83c9 100755 --- a/BRB/PushButton.py +++ b/BRB/PushButton.py @@ -8,7 +8,7 @@ import stat from pathlib import Path -def createPath(config, group, project, organism, libraryType, tuples): +def createPath(config, group, project, organism, libraryType, tuples, stats): """Ensures that the output path exists, creates it otherwise, and return where it is""" if tuples[0][3]: baseDir = "{}/{}/Analysis_{}".format(config.get('Paths', 'baseData'), @@ -20,10 +20,14 @@ def createPath(config, group, project, organism, libraryType, tuples): BRB.misc.getLatestSeqdir(config.get('Paths','groupData'), group), config.get('Options', 'runID'), BRB.misc.pacifier(project)) - os.makedirs(baseDir, mode=0o700, exist_ok=True) + + if not stats: + os.makedirs(baseDir, mode=0o700, exist_ok=True) oDir = os.path.join(baseDir, "{}_{}".format(BRB.misc.pacifier(libraryType), organism.split(' ')[0].lower())) - os.makedirs(oDir, mode=0o700, exist_ok=True) + if not stats: + os.makedirs(oDir, mode=0o700, exist_ok=True) + return oDir @@ -564,7 +568,7 @@ def scATAC(config, group, project, organism, libraryType, tuples): return outputDir, 0, True -def GetResults(config, project, libraries): +def GetResults(config, project, libraries, stats): """ Project is something like '352_Grzes_PearceEd' and libraries is a dictionary with libraries as keys: {'18L005489': ['FAT_first_A', @@ -591,7 +595,7 @@ def GetResults(config, project, libraries): ) log.info(f"Processing {dataPath}") except: - print("external data") + print(f"GetResults with ignore=True, {project} is external data.") ignore = True validLibraryTypes = {v: i for i, v in enumerate(config.get('Options', 'validLibraryTypes').split(','))} pipelines = config.get('Options', 'pipelines').split(',') @@ -638,7 +642,12 @@ def GetResults(config, project, libraries): reruncount = 0 # RELACS needs the unpacified project name to copy the original sample sheet to the dest dir # hence the pacifier is applied on the project in each pipeline separately - outputDir, rv, sambaUpdate = globals()[pipeline](config, group, project, organism, libraryType, tuples) + if stats: + outputDir, rv, sambaUpdate = ( + createPath(config, group, BRB.misc.pacifier(project), organism, libraryType, tuples, stats), + 0, False) + else: + outputDir, rv, sambaUpdate = globals()[pipeline](config, group, project, organism, libraryType, tuples) if reruncount == 0 and rv != 0: # Allow for one re-run reruncount += 1 diff --git a/BRB/run.py b/BRB/run.py index 82ec7d0..391687b 100755 --- a/BRB/run.py +++ b/BRB/run.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import glob import sys import os import BRB.getConfig @@ -13,6 +14,36 @@ from pathlib import Path from rich import print + +def process_data(config, ParkourDict, stats): + bdir = "{}/{}".format(config.get('Paths', 'baseData'), config.get('Options', 'runID')) + msg = [] + for k, v in ParkourDict.items(): + if not os.path.exists("{}/Project_{}".format(bdir, BRB.misc.pacifier(k))): + log.info("{}/Project_{} doesn't exist, probably lives on another lane.".format(bdir, BRB.misc.pacifier(k))) + continue + try: + msg = msg + BRB.PushButton.GetResults(config, k, v, stats) + except Exception as e: + BRB.email.errorEmail(config, sys.exc_info(), "Received an error running PushButton.GetResults() with {} and {}".format(k, v)) + log.critical("Received an error running PushButton.GetResults() with {} and {}".format(k, v)) + print("Received an error running PushButton.GetResults() with {} and {}".format(k, v), file=sys.stderr) + raise + + # Email finished message + log.info('Create e-mail') + log.info(msg) + BRB.email.finishedEmail(config, msg) + + return + + +def validate_fcid_with_stats(ctx, param, value): + if ctx.params.get('stats') and not value: + raise click.UsageError('--fcid is required when --stats standalone run is active.') + return value + + @click.command( context_settings=dict( help_option_names=["-h", "--help"] @@ -24,49 +55,84 @@ type=click.Path(exists=True), required=False, default=os.path.expanduser('~/configs/BigRedButton.ini'), - help='specify a custom ini file.', + help='Specify a custom ini file.', show_default=True ) -def run_brb(configfile): +@click.option( + "-s", + "--stats", + required=False, + is_flag=True, + help='Standalone run, use only on finished flowcells. Requires --fcid to indicate target.' +) +@click.option('--fcid', callback=validate_fcid_with_stats, help='Flowcell ID to push stats.') +def run_brb(configfile, stats, fcid): + + + # move stats flag here + # have an barebones config object, only with runID, parkourURL.. minimal + # no logs, print pushed dict to stdout, and status 200 + # and pipelines shouldn't crash with pakour failure while True: - #Read the config file + # Read the config file config = BRB.getConfig.getConfig(configfile) + + if not stats: + # Get the next flow cell to process, or sleep + config, ParkourDict = BRB.findFinishedFlowCells.newFlowCell(config) + if (config.get('Options','runID') == '') or ParkourDict is None: + sleep(60*60) + continue - #Get the next flow cell to process, or sleep - config, ParkourDict = BRB.findFinishedFlowCells.newFlowCell(config) - if(config.get('Options','runID') == '') or ParkourDict is None: - sleep(60*60) - continue + # Open log file + logFile = Path( + config['Paths']['logPath'], + config.get('Options','runID') + '.log' + ) + print(f"Logging into: {logFile}") + setLog(logFile) - # Open log file - logFile = Path( - config['Paths']['logPath'], - config.get('Options','runID') + '.log' - ) - print(f"Logging into: {logFile}") - setLog(logFile) + else: + # Push stats on-demand + d = [d for d in glob.glob("{}/*/fastq.made".format(config.get('Paths', 'baseData'))) if fcid in d] + dual_lane = len(d) == 2 + if len(d) == 0: + print(f"ERROR: No fastq.made files found for {fcid}") + return # Exit BRB if no files found. + elif len(d) > 2: + print(f"ERROR: How many lanes does {fcid} have?!") + return # Exit BRB this error shouldn't happen at all. + + config.set('Options','runID',d[0].split("/")[-2]) + ParkourDict = BRB.findFinishedFlowCells.queryParkour(config) + + if dual_lane: + config1 = BRB.getConfig.getConfig(configfile) + config1.set('Options','runID',d[1].split("/")[-2]) + ParkourDict1 = BRB.findFinishedFlowCells.queryParkour(config) - #Process each group's data, ignore cases where the project isn't in the lanes being processed - bdir = "{}/{}".format(config.get('Paths', 'baseData'), config.get('Options', 'runID')) - msg = [] - for k, v in ParkourDict.items(): - if not os.path.exists("{}/Project_{}".format(bdir, BRB.misc.pacifier(k))): - log.info("{}/Project_{} doesn't exist, probably lives on another lane.".format(bdir, BRB.misc.pacifier(k))) - continue - try: - msg = msg + BRB.PushButton.GetResults(config, k, v) - except Exception as e: - BRB.email.errorEmail(config, sys.exc_info(), "Received an error running PushButton.GetResults() with {} and {}".format(k, v)) - log.critical("Received an error running PushButton.GetResults() with {} and {}".format(k, v)) - print("Received an error running PushButton.GetResults() with {} and {}".format(k, v), file=sys.stderr) - raise + # Open log file + if not dual_lane: + logFile = Path(config['Paths']['logPath'], config.get('Options','runID') + '.log') + else: + logFile = Path(config['Paths']['logPath'], config.get('Options','runID')[:-1] + 'both' + '.log') + print(f"Logging into: {logFile}") + setLog(logFile) + log.info(f"Pushing stats for flowcell: {fcid}") + + # Process each group's data, ignore cases where the project isn't in the lanes being processed + process_data(config, ParkourDict, stats) + + if stats and dual_lane: + process_data(config1, ParkourDict1, stats) + + + if not stats: + # Mark the flow cell as having been processed + BRB.findFinishedFlowCells.markFinished(config) + log.info('=== finished flowcell ===') - #Email finished message - log.info('Create e-mail') - log.info(msg) - BRB.email.finishedEmail(config, msg) - #Mark the flow cell as having been processed - BRB.findFinishedFlowCells.markFinished(config) - log.info('=== finished flowcell ===') + if stats: + return # don't do anything else.