Merge branch 'devel'

Author: rhandberg

.travis.yml

@@ -76,13 +76,12 @@ before_deploy:
   - cat VERSION
 deploy:
   provider: releases
-  api_key: $GITHUB_TOKEN
-  skip_cleanup: true
+  token: $GITHUB_TOKEN
   draft: true
   name: "Version $TRAVIS_TAG"
-  body: "Version $TRAVIS_TAG"
+  release_notes: "Version $TRAVIS_TAG"
   on:
     branch:
       - master
-      - /^v\d+\.\d+.+$/
+      - /^v\d+\.\d+.*$/
     tags: true

VERSION

@@ -1 +1 @@
-master-v1.2.4
+master-v1.2.5

corrections/BaseCorrector.py

@@ -85,8 +85,13 @@ def __init__(self, input_folder, plot=False):
 		logging.getLogger('corrections').addHandler(handler)
 
 		# Save inputs:
-		self.input_folder = input_folder
 		self.plot = plot
+		if os.path.isdir(input_folder):
+			self.input_folder = input_folder
+			todo_file = os.path.join(input_folder, 'todo.sqlite')
+		else:
+			self.input_folder = os.path.dirname(input_folder)
+			todo_file = input_folder
 
 		self.CorrMethod = {
 			'BaseCorrector': 'base',
@@ -110,7 +115,6 @@ def __init__(self, input_folder, plot=False):
 			os.makedirs(self.data_folder, exist_ok=True)
 
 		# The path to the TODO list:
-		todo_file = os.path.join(input_folder, 'todo.sqlite')
 		logger.debug("TODO file: %s", todo_file)
 		if not os.path.isfile(todo_file):
 			raise FileNotFoundError("TODO file not found")
@@ -204,11 +208,11 @@ def correct(self, task, output_folder=None):
 
 		except (KeyboardInterrupt, SystemExit):
 			status = STATUS.ABORT
-			logger.warning("Correction was aborted.")
+			logger.warning("Correction was aborted (priority=%d)", task['priority'])
 
 		except: # noqa: E722
 			status = STATUS.ERROR
-			logger.exception("Correction failed.")
+			logger.exception("Correction failed (priority=%d)", task['priority'])
 
 		# Check that the status has been changed:
 		if status == STATUS.UNKNOWN:

corrections/taskmanager.py

@@ -12,14 +12,16 @@
 import sqlite3
 import logging
 import json
+from numpy import atleast_1d
 from . import STATUS
 
 class TaskManager(object):
 	"""
 	A TaskManager which keeps track of which targets to process.
 	"""
 
-	def __init__(self, todo_file, cleanup=False, overwrite=False, summary=None, summary_interval=100):
+	def __init__(self, todo_file, cleanup=False, overwrite=False, cleanup_constraints=None,
+		summary=None, summary_interval=100):
 		"""
 		Initialize the TaskManager which keeps track of which targets to process.
 
@@ -28,6 +30,12 @@ def __init__(self, todo_file, cleanup=False, overwrite=False, summary=None, summ
 			cleanup (boolean, optional): Perform cleanup/optimization of TODO-file before
 				during initialization. Default=False.
 			overwrite (boolean, optional): Overwrite any previously calculated results. Default=False.
+			cleanup_constraints (dict, optional): Dict of constraint for cleanup of the status of
+				previous correction runs. If not specified, all bad results are cleaned up.
+			summary (string, optional): Path to JSON file which will be periodically updated with
+				a status summary of the corrections.
+			summary_interval (integer, optional): Interval at which summary file is updated.
+				Default=100.
 
 		Raises:
 			FileNotFoundError: If TODO-file could not be found.
@@ -90,10 +98,21 @@ def __init__(self, todo_file, cleanup=False, overwrite=False, summary=None, summ
 		);""")
 		self.conn.commit()
 
-		# Reset calculations with status STARTED or ABORT:
+		# Reset calculations with status STARTED, ABORT or ERROR:
 		clear_status = str(STATUS.STARTED.value) + ',' + str(STATUS.ABORT.value) + ',' + str(STATUS.ERROR.value) + ',' + str(STATUS.SKIPPED.value)
-		self.cursor.execute("DELETE FROM diagnostics_corr WHERE priority IN (SELECT todolist.priority FROM todolist WHERE corr_status IN (" + clear_status + "));")
-		self.cursor.execute("UPDATE todolist SET corr_status=NULL WHERE corr_status IN (" + clear_status + ");")
+		constraints = ['corr_status IN (' + clear_status + ')']
+
+		# Add additional constraints from the user input and build SQL query:
+		if cleanup_constraints:
+			cc = cleanup_constraints.copy()
+			if cc.get('datasource'):
+				constraints.append("datasource='ffi'" if cc.pop('datasource') == 'ffi' else "datasource!='ffi'")
+			for key, val in cc.items():
+				constraints.append(key + ' IN (%s)' % ','.join([str(v) for v in atleast_1d(val)]))
+
+		constraints = ' AND '.join(constraints)
+		self.cursor.execute("DELETE FROM diagnostics_corr WHERE priority IN (SELECT todolist.priority FROM todolist WHERE " + constraints + ");")
+		self.cursor.execute("UPDATE todolist SET corr_status=NULL WHERE " + constraints + ";")
 		self.conn.commit()
 
 		# Set all targets that did not return good photometry or were not approved by the Data Validation to SKIPPED:
@@ -175,7 +194,7 @@ def get_number_tasks(self, starid=None, camera=None, ccd=None, datasource=None,
 		if ccd is not None:
 			constraints.append('todolist.ccd=%d' % ccd)
 		if datasource is not None:
-			constraints.append('todolist.datasource="%s"' % datasource)
+			constraints.append("todolist.datasource='ffi'" if datasource == 'ffi' else "todolist.datasource!='ffi'")
 
 		if constraints:
 			constraints = ' AND ' + " AND ".join(constraints)
@@ -208,7 +227,7 @@ def get_task(self, starid=None, camera=None, ccd=None, datasource=None, priority
 		if ccd is not None:
 			constraints.append('todolist.ccd=%d' % ccd)
 		if datasource is not None:
-			constraints.append('todolist.datasource="%s"' % datasource)
+			constraints.append("todolist.datasource='ffi'" if datasource == 'ffi' else "todolist.datasource!='ffi'")
 
 		if constraints:
 			constraints = ' AND ' + " AND ".join(constraints)

run_tesscorr.py

@@ -72,32 +72,35 @@ def main():
 
 	output_folder = args.output_folder
 	if output_folder is None:
-		output_folder = os.environ.get('TESSCORR_OUTPUT', os.path.join(input_folder, 'lightcurves'))
+		output_folder = os.environ.get('TESSCORR_OUTPUT', os.path.join(os.path.dirname(input_folder), 'lightcurves'))
 
 	logger.info("Loading input data from '%s'", input_folder)
 	logger.info("Putting output data in '%s'", output_folder)
 
 	# Make sure the output directory exists:
 	os.makedirs(output_folder, exist_ok=True)
 
+	# Constraints on which targets to process:
+	constraints = {
+		'camera': args.camera,
+		'ccd': args.ccd,
+		'datasource': args.datasource,
+		'starid': args.starid,
+		'priority': args.priority
+	}
+
 	# Get the class for the selected method:
 	CorrClass = corrections.corrclass(args.method)
 
 	# Initialize the corrector class:
 	with CorrClass(input_folder, plot=args.plot) as corr:
 		# Start the TaskManager:
-		with corrections.TaskManager(input_folder, overwrite=args.overwrite) as tm:
+		with corrections.TaskManager(input_folder, overwrite=args.overwrite, cleanup_constraints=constraints) as tm:
 			while True:
 				if args.random:
 					task = tm.get_random_task()
 				else:
-					task = tm.get_task(
-						starid=args.starid,
-						camera=args.camera,
-						ccd=args.ccd,
-						datasource=args.datasource,
-						priority=args.priority
-					)
+					task = tm.get_task(**constraints)
 
 				if task is None: break
 

run_tesscorr_mpi.py

@@ -59,7 +59,7 @@ def main():
 		input_folder = os.environ.get('TESSCORR_INPUT')
 	if not input_folder:
 		parser.error("Please specify an INPUT_FOLDER.")
-	output_folder = os.environ.get('TESSCORR_OUTPUT', os.path.join(input_folder, 'lightcurves'))
+	output_folder = os.environ.get('TESSCORR_OUTPUT', os.path.join(os.path.dirname(input_folder), 'lightcurves'))
 
 	# Define MPI message tags
 	tags = enum.IntEnum('tags', ('READY', 'DONE', 'EXIT', 'START'))
@@ -72,12 +72,23 @@ def main():
 
 	if rank == 0:
 		try:
-			with corrections.TaskManager(input_folder, cleanup=True, overwrite=args.overwrite, summary=os.path.join(output_folder, 'summary_corr.json')) as tm:
+			# Constraints on which targets to process:
+			constraints = {
+				'camera': args.camera,
+				'ccd': args.ccd,
+				'datasource': args.datasource
+			}
+
+			# Start TaskManager, which keeps track of the task that needs to be performed:
+			with corrections.TaskManager(input_folder, cleanup=True, overwrite=args.overwrite,
+				cleanup_constraints=constraints,
+				summary=os.path.join(output_folder, 'summary_corr.json')) as tm:
+
 				# Set level of TaskManager logger:
 				tm.logger.setLevel(logging_level)
 
 				# Get list of tasks:
-				numtasks = tm.get_number_tasks(camera=args.camera, ccd=args.ccd, datasource=args.datasource)
+				numtasks = tm.get_number_tasks(**constraints)
 				tm.logger.info("%d tasks to be run", numtasks)
 
 				# Start the master loop that will assign tasks
@@ -98,7 +109,7 @@ def main():
 
 					if tag in (tags.DONE, tags.READY):
 						# Worker is ready, so send it a task
-						task = tm.get_task(camera=args.camera, ccd=args.ccd, datasource=args.datasource)
+						task = tm.get_task(**constraints)
 						if task:
 							task_index = task['priority']
 							tm.start_task(task_index)
@@ -161,7 +172,7 @@ def main():
 							error_msg = traceback.format_exc().strip()
 							result.update({
 								'status_corr': corrections.STATUS.ERROR,
-								'details': {'errors': error_msg},
+								'details': {'errors': [error_msg]},
 							})
 
 						result.update({'worker_wait_time': toc-tic})

tests/test_taskmanager.py

@@ -11,10 +11,10 @@
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 from corrections import TaskManager, STATUS
 
+#--------------------------------------------------------------------------------------------------
 def test_taskmanager():
 	"""Test of TaskManager"""
 
-	# Load the first image in the input directory:
 	INPUT_DIR = os.path.join(os.path.dirname(__file__), 'input')
 
 	# Find the shape of the original image:
@@ -60,5 +60,53 @@ def test_taskmanager():
 
 		assert(task1_status == STATUS.STARTED.value)
 
+#--------------------------------------------------------------------------------------------------
+def test_taskmanager_constraints():
+
+	INPUT_DIR = os.path.join(os.path.dirname(__file__), 'input')
+
+	constraints = {'datasource': 'tpf', 'priority': 17}
+	with TaskManager(INPUT_DIR, overwrite=True, cleanup_constraints=constraints) as tm:
+		task = tm.get_task(**constraints)
+		print(task)
+		assert task is None, "Task1 should be None"
+
+	constraints = {'datasource': 'ffi', 'priority': 17}
+	with TaskManager(INPUT_DIR, overwrite=True, cleanup_constraints=constraints) as tm:
+		task = tm.get_task(**constraints)
+		print(task)
+		assert task['priority'] == 17, "Task2 should be #17"
+
+#--------------------------------------------------------------------------------------------------
+def test_taskmanager_cleanup():
+
+	INPUT_DIR = os.path.join(os.path.dirname(__file__), 'input')
+
+	# Reset the TODO-file completely, and mark the first task as STARTED:
+	with TaskManager(INPUT_DIR, overwrite=True) as tm:
+		task1 = tm.get_task()
+		print(task1)
+		pri = task1['priority']
+		tm.start_task(pri)
+
+	# Cleanup, but with a constraint not matching the one we changed:
+	with TaskManager(INPUT_DIR, cleanup_constraints={'priority': 18}) as tm:
+		# Check that the status did actually change in the todolist:
+		tm.cursor.execute("SELECT corr_status FROM todolist WHERE priority=?;", [pri])
+		task1_status = tm.cursor.fetchone()['corr_status']
+		print(task1_status)
+		assert task1_status == STATUS.STARTED.value
+
+	# Now clean with a constraint that matches:
+	with TaskManager(INPUT_DIR, cleanup_constraints={'priority': pri}) as tm:
+		# Check that the status did actually change in the todolist:
+		tm.cursor.execute("SELECT corr_status FROM todolist WHERE priority=?;", [pri])
+		task1_status = tm.cursor.fetchone()['corr_status']
+		print(task1_status)
+		assert task1_status is None
+
+#--------------------------------------------------------------------------------------------------
 if __name__ == '__main__':
 	test_taskmanager()
+	test_taskmanager_constraints()
+	test_taskmanager_cleanup()