Improved logging; added quiet option to read_tables; improved packaging

Author: msbentley

pds4_utils/__init__.py

@@ -9,7 +9,10 @@
 # Set up the root logger
 
 import logging
-import sys
-logging.basicConfig(format='%(levelname)s %(asctime)s (%(name)s): %(message)s',
-                    level=logging.INFO, stream=sys.stdout, datefmt='%Y-%m-%d %H:%M:%S')
+# import sys
+
+# logging.basicConfig(format='%(levelname)s %(asctime)s (%(name)s): %(message)s',
+#                     level=logging.INFO, stream=sys.stdout, datefmt='%Y-%m-%d %H:%M:%S')
+
+logging.getLogger(__name__).addHandler(logging.NullHandler())
 

pds4_utils/pds4_utils.py

@@ -4,34 +4,52 @@
 
 from pds4_tools import pds4_read
 from pds4_tools.reader.table_objects import TableManifest
-import pds4_tools
+# import pds4_tools
 from pathlib import Path
 from lxml import etree
-import numpy as np
 import pandas as pd
 import sys
 import yaml
 import os
+import logging
 
 try:
    import cPickle as pickle
 except:
    import pickle
-   
 
-import logging
-log = logging.getLogger(__name__)
+# set up module level logging
 
 # only show warning or higher messages from PDS4 tools
 pds4_logger = logging.getLogger('PDS4ToolsLogger')
 pds4_logger.setLevel(logging.WARNING)
 
+class DuplicateFilter(logging.Filter):
+
+    def filter(self, record):
+        current_log = (record.module, record.levelno, record.msg)
+        last = getattr(self, "last_log", None)
+        if (last is None) or (last != current_log):
+            self.last_log = current_log
+            return True  # i.e. log the message
+        else:
+            return False  # i.e. do not log the message
+
+log = logging.getLogger(__name__)
+handler = logging.StreamHandler(sys.stdout)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+handler.setFormatter(formatter)
+log.addHandler(handler)
+log.setLevel(logging.WARNING)
+
+
 default_config = os.path.join(
     os.environ.get('APPDATA') or
     os.environ.get('XDG_CONFIG_HOME') or
     os.path.join(os.environ['HOME'], '.config'),
     "pds_dbase.yml")
 
+
 def index_products(directory='.', pattern='*.xml'):
     """
     Accepts a directory containing PDS4 products, indexes the labels and returns a 
@@ -222,7 +240,7 @@ def _constructor(self):
         return pds4_df
 
 
-def read_tables(label, label_directory='.', recursive=False, table_name=None, index_col=None, add_filename=False):
+def read_tables(label, label_directory='.', recursive=False, table_name=None, index_col=None, add_filename=False, quiet=False):
     """
     Accepts a directory and file-pattern or list and attempts to load the specified table
     (or first table, if none is specified) into a merged DataFrane. If the tables 
@@ -254,17 +272,22 @@ def read_tables(label, label_directory='.', recursive=False, table_name=None, in
     # de-dupe list
     file_list = list(set(file_list))
 
+    handler.addFilter(DuplicateFilter())
+    filter_inst = log.handlers[0].filters[-1]
+
     for f in file_list:
         if table is None:
-            table = read_table(f, table_name=table_name, index_col=index_col)
+            table = read_table(f, table_name=table_name, index_col=index_col, quiet=quiet)
             if add_filename:
                 table['filename'] = table.filename
         else:
-            temp_table = read_table(f, table_name=table_name, index_col=index_col)
+            temp_table = read_table(f, table_name=table_name, index_col=index_col, quiet=quiet)
             if add_filename:
                 temp_table['filename'] = temp_table.filename
             table = table.append(temp_table)
 
+    handler.removeFilter(filter_inst)
+
     table.sort_index(inplace=True)
 
     log.info('{:d} files read with {:d} total records'.format(len(file_list), len(table)))

setup.py

@@ -1,10 +1,19 @@
 from setuptools import setup
 
-setup(name='pds4_utils',
-      version='0.1',
-      description='PDS4 utilities',
-      author='Mark S. Bentley',
-      author_email='mark@lunartech.org',
-      packages=['pds4_utils'],
-      zip_safe=False)
+with open("README.md", "r") as fh:
+    long_description = fh.read()
 
+setup(name='pds4_utils',
+    version='0.1',
+    author='Mark S. Bentley',
+    author_email='mark@lunartech.org',
+    description='A collection of PDS4 utilities',
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/msbentley/pds4_utils",
+    download_url = 'https://github.com/msbentley/pds4_utils/archive/0.1.tar.gz',
+    install_requires=['pandas','pyyaml','lxml','pds4-tools'],
+    python_requires='>=3.0',
+    keywords = ['PDS','archive','data'],
+    packages=['pds4_utils'],
+    zip_safe=False)