Improved config logging

This commit adds twarc.config.ConfigProvider which is based on
click_config_file.configobj_provider and stores the file path for the
config file that was used. This is useful for logging.

Also when --verbose is used the log will now contain the keys that are
being used to talk to the API. This isn't something you would normally
want in your logs, but it can be useful for debugging situations like #441
and #469.

Author: edsu

twarc/client2.py

@@ -720,13 +720,15 @@ def connect(self):
             self.client.close()
 
         if self.auth_type == "application" and self.bearer_token:
-            log.info('Creating HTTP session headers for app auth.')
+            log.info('creating HTTP session headers for app auth.')
+            auth = f"Bearer {self.bearer_token}"
+            log.debug('authorization: %s', auth)
             self.client = requests.Session()
-            self.client.headers.update(
-                {"Authorization": f"Bearer {self.bearer_token}"}
-            )
+            self.client.headers.update({"Authorization": auth})
         elif self.auth_type == "application":
-            log.info('Creating app auth client via OAuth2')
+            log.info('creating app auth client via OAuth2')
+            log.debug('client_id: %s', self.consumer_key)
+            log.debug('client_secret: %s', self.client_secret)
             client = BackendApplicationClient(client_id=self.consumer_key)
             self.client = OAuth2Session(client=client)
             self.client.fetch_token(
@@ -736,6 +738,10 @@ def connect(self):
             )
         else:
             log.info('creating user auth client')
+            log.debug('client_id: %s', self.consumer_key)
+            log.debug('client_secret: %s', self.client_secret)
+            log.debug('resource_owner_key: %s', self.access_token)
+            log.debug('resource_owner_secret: %s', self.access_token_secret)
             self.client = OAuth1Session(
                 client_key=self.consumer_key,
                 client_secret=self.consumer_secret,

twarc/command2.py

@@ -19,10 +19,12 @@
 
 from twarc.version import version
 from twarc.handshake import handshake
+from twarc.config import ConfigProvider 
 from twarc.decorators import cli_api_error
 from twarc.expansions import ensure_flattened
 from click_config_file import configuration_option
 
+config_provider = ConfigProvider()
 
 @with_plugins(iter_entry_points('twarc.plugins'))
 @click.group()
@@ -42,23 +44,26 @@
     show_default=True,
 )
 @click.option('--log', default='twarc.log')
+@click.option('--verbose', is_flag=True, default=False)
 @click.option('--metadata/--no-metadata', default=True, show_default=True,
     help="Include/don't include metadata about when and how data was collected.")
-@configuration_option(cmd_name='twarc')
+@configuration_option(cmd_name='twarc', config_file_name='config', provider=config_provider)
 @click.pass_context
 def twarc2(
     ctx, consumer_key, consumer_secret, access_token, access_token_secret, bearer_token,
-    log, metadata, app_auth
+    log, metadata, app_auth, verbose
 ):
     """
     Collect data from the Twitter V2 API.
     """
     logging.basicConfig(
         filename=log,
-        level=logging.INFO,
+        level=logging.DEBUG if verbose else logging.INFO,
         format="%(asctime)s %(levelname)s %(message)s"
     )
 
+    logging.info("using config %s", config_provider.file_path)
+
     if bearer_token or (consumer_key and consumer_secret):
         if app_auth and (bearer_token or (consumer_key and consumer_secret)):
             ctx.obj = twarc.Twarc2(
@@ -103,15 +108,14 @@ def configure(ctx):
     """
     Set up your Twitter app keys.
     """
+
+    config_file = config_provider.file_path
+    logging.info('creating config file: %s', config_file)
+
     keys = handshake()
     if keys is None:
         raise click.ClickException("Unable to authenticate")
 
-    config_dir = pathlib.Path(click.get_app_dir('twarc'))
-    if not config_dir.is_dir():
-        config_dir.mkdir(parents=True)
-    config_file = config_dir / 'config'
-
     config = configobj.ConfigObj(unrepr=True)
     config.filename = config_file
 
@@ -682,3 +686,12 @@ def _error_str(errors):
 def _write(results, outfile, pretty=False):
     indent = 2 if pretty else None
     click.echo(json.dumps(results, indent=indent), file=outfile)
+
+"""
+def _get_config_file():
+    config_dir = pathlib.Path(click.get_app_dir('twarc'))
+    if not config_dir.is_dir():
+        config_dir.mkdir(parents=True)
+    config_file = config_dir / 'config'
+    return config_file
+"""

twarc/config.py

@@ -0,0 +1,16 @@
+import logging
+import configobj
+
+# Adapted from click_config_file.configobj_provider so that we can store the
+# file path that the config was loaded from in order to log it later.
+
+log = logging
+
+class ConfigProvider():
+
+    def __init__(self):
+        self.file_path = None
+
+    def __call__(self, file_path, cmd_name):
+        self.file_path = file_path
+        return configobj.ConfigObj(file_path, unrepr=True)

twarc/version.py

@@ -1 +1 @@
-version = '2.1.3'
+version = '2.1.4'