From 509415746ed94039d11dff6aacbca908e40daab9 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 16 Oct 2017 18:26:50 -0700 Subject: [PATCH 1/2] Adding functionality allowing third-party packages to define functions, aggregates, and collations. --- bin/q | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/bin/q b/bin/q index c3b01828..12d296ca 100755 --- a/bin/q +++ b/bin/q @@ -49,9 +49,16 @@ import hashlib import uuid import cStringIO import math +import pkg_resources +import logging +import inspect DEBUG = False + +logger = logging.getLogger(__name__) + + def get_stdout_encoding(encoding_override=None): if encoding_override is not None and encoding_override != 'none': return encoding_override @@ -118,12 +125,61 @@ class Sqlite3DB(object): str: 'TEXT', int: 'INT', long : 'INT' , float: 'FLOAT', None: 'TEXT'} self.numeric_column_types = set([int, long, float]) self.add_user_functions() + self.add_entrypoints() def add_user_functions(self): self.conn.create_function("regexp", 2, regexp) self.conn.create_function("sha1", 1, sha1) self.conn.create_aggregate("percentile",2,StrictPercentile) + def _get_entrypoint_argument_count(self, entrypoint): + if isinstance(entrypoint, object): + return len(inspect.getargspec(entrypoint.step).args[1:]) + + return len(inspect.getargspec(entrypoint).args) + + def add_entrypoints(self): + for entry_point in pkg_resources.iter_entry_points(group='q_functions'): + try: + loaded = entry_point.load() + self.conn.create_function( + entry_point.name, + self._get_entrypoint_argument_count(loaded), + loaded, + ) + except ImportError: + logger.warning( + "Could not load entry point function: %s", + entry_point, + ) + + for entry_point in pkg_resources.iter_entry_points(group='q_aggregates'): + try: + loaded = entry_point.load() + self.conn.create_aggregate( + entry_point.name, + self._get_entrypoint_argument_count(loaded), + loaded, + ) + except ImportError: + logger.warning( + "Could not load entry point aggregate: %s", + entry_point, + ) + + for entry_point in pkg_resources.iter_entry_points(group='q_collations'): + try: + loaded = entry_point.load() + self.conn.create_collation( + entry_point.name, + loaded, + ) + except ImportError: + logger.warning( + "Could not load entry point collations: %s", + entry_point, + ) + def is_numeric_type(self, column_type): return column_type in self.numeric_column_types From c6c09590f6439eae374d700e6c5b4d50e0174c29 Mon Sep 17 00:00:00 2001 From: Adam Coddington Date: Mon, 16 Oct 2017 18:40:10 -0700 Subject: [PATCH 2/2] Make it possible for errors when loading entrypoints to be surfaced. --- bin/q | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/q b/bin/q index 12d296ca..2140bb1a 100755 --- a/bin/q +++ b/bin/q @@ -1798,6 +1798,9 @@ def run_standalone(): print >> sys.stderr, "Max column length limit must be a positive integer (%s)" % max_column_length_limit sys.exit(31) + if options.verbose: + logging.basicConfig(level=logging.DEBUG) + default_input_params = QInputParams(skip_header=options.skip_header, delimiter=options.delimiter, input_encoding=options.encoding,