Merge remote-tracking branch 'origin/master' into edge

Author: jonasbardino

mig/install/generateconfs.py

@@ -46,6 +46,7 @@
 
 # NOTE: moved mig imports into try/except to avoid autopep8 moving to top!
 try:
+    from mig.shared.defaults import MIG_BASE, MIG_ENV
     from mig.shared.install import generate_confs
 except ImportError:
     print("ERROR: the migrid modules must be in PYTHONPATH")
@@ -351,14 +352,25 @@ def usage(options):
     if settings['destination_suffix'] == 'DEFAULT':
         suffix = "-%s" % datetime.datetime.now().isoformat()
         settings['destination_suffix'] = suffix
+    if os.getenv('MIG_ENV', 'default') == 'local':
+        output_path = os.path.join(MIG_BASE, 'envhelp/output')
+    elif settings['destination'] == 'DEFAULT' or \
+            not os.path.isabs(settings['destination']):
+        # Default to generate in subdir of CWD ...
+        output_path = os.getcwd()
+    else:
+        # ... but use verbatim passthrough for absolute destination
+        output_path = settings['destination']
     print('# Creating confs with:')
     # NOTE: force list to avoid problems with in-line edits
     for (key, val) in list(settings.items()):
         print('%s: %s' % (key, val))
         # Remove default values to use generate_confs default values
         if val == 'DEFAULT':
             del settings[key]
-    options = generate_confs(**settings)
+
+    options = generate_confs(output_path, **settings)
+
     # TODO: avoid reconstructing this path (also done inside generate_confs)
     instructions_path = os.path.join(options['destination_dir'],
                                      'instructions.txt')

mig/shared/configuration.py

@@ -2486,11 +2486,17 @@ def reload_config(self, verbose, skip_log=False):
 
         # Init auth logger
 
+        auth_logger_logfile = None
+        if skip_log:
+            auth_logger_logfile = None
+        else:
+            auth_logger_logfile = self.user_auth_log
+
         if self.auth_logger_obj:
             self.auth_logger_obj.reopen()
         else:
             self.auth_logger_obj = Logger(
-                self.loglevel, logfile=self.user_auth_log, app='main-auth')
+                self.loglevel, logfile=auth_logger_logfile, app='main-auth')
         self.auth_logger = self.auth_logger_obj.logger
 
         # cert and key for generating a default proxy for nordugrid/ARC

mig/shared/defaults.py

@@ -77,31 +77,6 @@
 any_protocol = keyword_any
 any_state = keyword_any
 
-mig_user = {
-    'default': 'mig',
-    'local': keyword_auto,
-}[MIG_ENV]
-
-mig_group = {
-    'default': 'mig',
-    'local': keyword_auto,
-}[MIG_ENV]
-
-default_source = {
-    'default': keyword_auto,
-    'local': os.path.join(MIG_BASE, "mig/install"),
-}[MIG_ENV]
-
-default_destination = {
-    'default': keyword_auto,
-    'local': os.path.join(MIG_BASE, "envhelp/output/confs"),
-}[MIG_ENV]
-
-default_enable_events = {
-    'default': True,
-    'local': False
-}[MIG_ENV]
-
 AUTH_NONE, AUTH_GENERIC, AUTH_CERTIFICATE = "None", "Generic", "X.509 Certificate"
 AUTH_OPENID_CONNECT, AUTH_OPENID_V2 = "OpenID Connect", "OpenID 2.0"
 

mig/shared/install.py

@@ -50,8 +50,7 @@
 import sys
 
 from mig.shared.defaults import default_http_port, default_https_port, \
-    MIG_BASE, mig_user, mig_group, default_source, default_destination, \
-    auth_openid_mig_db, auth_openid_ext_db, STRONG_TLS_CIPHERS, \
+    auth_openid_mig_db, auth_openid_ext_db, MIG_BASE, STRONG_TLS_CIPHERS, \
     STRONG_TLS_CURVES, STRONG_SSH_KEXALGOS, STRONG_SSH_LEGACY_KEXALGOS, \
     STRONG_SSH_CIPHERS, STRONG_SSH_LEGACY_CIPHERS, STRONG_SSH_MACS, \
     STRONG_SSH_LEGACY_MACS, CRACK_USERNAME_REGEX, CRACK_WEB_REGEX, \
@@ -257,14 +256,14 @@ def template_remove(template_file, remove_pattern):
 
 
 _GENERATE_CONFS_NOFORWARD_KEYS = [
+    'generateconfs_output_path',
     'generateconfs_command',
     'source',
     'destination',
     'destination_suffix',
     'group',
     'user',
     'timezone',
-    '_getcwd',
     '_getpwnam',
     '_prepare',
     '_writefiles',
@@ -273,12 +272,13 @@ def template_remove(template_file, remove_pattern):
 
 
 def generate_confs(
+    generateconfs_output_path,
     # NOTE: make sure command line args with white-space are properly wrapped
     generateconfs_command=subprocess.list2cmdline(sys.argv),
-    source=default_source,
-    destination=default_destination,
-    user=mig_user,
-    group=mig_group,
+    source=keyword_auto,
+    destination=keyword_auto,
+    user=keyword_auto,
+    group=keyword_auto,
     timezone=keyword_auto,
     destination_suffix="",
     base_fqdn='',
@@ -315,9 +315,9 @@ def generate_confs(
     apache_log='/var/log/apache2',
     apache_worker_procs=256,
     openssh_version='7.4',
-    mig_code='/home/mig/mig',
-    mig_state='/home/mig/state',
-    mig_certs='/home/mig/certs',
+    mig_code=keyword_auto,
+    mig_state=keyword_auto,
+    mig_certs=keyword_auto,
     auto_add_cert_user=False,
     auto_add_oid_user=False,
     auto_add_oidc_user=False,
@@ -498,14 +498,16 @@ def generate_confs(
     ca_fqdn='',
     ca_user='mig-ca',
     ca_smtp='localhost',
-    _getcwd=os.getcwd,
     _getpwnam=pwd.getpwnam,
     _prepare=None,
     _writefiles=None,
     _instructions=None,
 ):
     """Generate Apache and MiG server confs with specified variables"""
 
+    assert os.path.isabs(
+        generateconfs_output_path), "output directory must be an absolute path"
+
     # TODO: override in signature as a non-functional follow-up change
     if _prepare is None:
         _prepare = _generate_confs_prepare
@@ -521,29 +523,44 @@ def generate_confs(
                 _GENERATE_CONFS_NOFORWARD_KEYS}
 
     # expand any directory path specific as "auto" relative to CWD
-    thecwd = _getcwd()
 
     if source == keyword_auto:
         # use the templates from this copy of the code tree
         template_dir = os.path.join(MIG_BASE, "mig/install")
     else:
         # construct a path using the supplied value made absolute
-        template_dir = abspath(source, start=thecwd)
+        template_dir = abspath(source, start=generateconfs_output_path)
 
     if destination == keyword_auto:
         # write output into a confs folder within the CWD
-        destination = os.path.join(thecwd, 'confs')
+        destination = os.path.join(generateconfs_output_path, 'confs')
+    elif os.path.isabs(destination):
+        # take the caller at face-value and do not change the path
+        pass
     else:
         # construct a path from the supplied value made absolute
-        destination = abspath(destination, start=thecwd)
+        destination = abspath(destination, start=generateconfs_output_path)
 
     # finalize destination paths up-front
     destination_link = destination
     destination_dir = "%s%s" % (destination, destination_suffix)
 
+    # expand mig, certs and state paths relative to base if left to "AUTO"
+
+    if mig_code == keyword_auto:
+        mig_code = expanded['mig_code'] = os.path.join(MIG_BASE, 'mig')
+
+    if mig_certs == keyword_auto:
+        mig_certs = expanded['mig_certs'] = os.path.join(MIG_BASE, 'certs')
+
+    if mig_state == keyword_auto:
+        mig_state = expanded['mig_state'] = os.path.join(MIG_BASE, 'state')
+
     # expand any user information marked as "auto" based on the environment
+
     if user == keyword_auto:
         user = pwd.getpwuid(os.getuid())[0]
+
     if group == keyword_auto:
         group = grp.getgrgid(os.getgid())[0]
 

mig/unittest/__init__.py

@@ -28,17 +28,57 @@
 """Unit tests for core helper functions"""
 from __future__ import print_function
 
+from configparser import ConfigParser
+import os
+import stat
 import sys
 import time
 import logging
 
+sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), "../..")))
+from tests.support import MIG_BASE, PY2, is_path_within
+
 from mig.shared.base import client_id_dir, client_dir_id, get_short_id, \
     invisible_path, allow_script, brief_list
 
 
-if __name__ == "__main__":
+_LOCAL_MIG_BASE = '/usr/src/app' if PY2 else MIG_BASE # account for execution in container
+_PYTHON_MAJOR = '2' if PY2 else '3'
+_TEST_CONF_DIR = os.path.join(MIG_BASE, "envhelp/output/testconfs-py%s" % (_PYTHON_MAJOR,))
+_TEST_CONF_FILE = os.path.join(_TEST_CONF_DIR, "MiGserver.conf")
+_TEST_CONF_SYMLINK = os.path.join(MIG_BASE, "envhelp/output/testconfs")
+
+
+def _assert_local_config():
+    try:
+        link_stat = os.lstat(_TEST_CONF_SYMLINK)
+        assert stat.S_ISLNK(link_stat.st_mode)
+        configdir_stat = os.stat(_TEST_CONF_DIR)
+        assert stat.S_ISDIR(configdir_stat.st_mode)
+        config = ConfigParser()
+        config.read([_TEST_CONF_FILE])
+        return config
+    except Exception as exc:
+        raise AssertionError('local configuration invalid or missing: %s' % (str(exc),))
+
+
+def _assert_local_config_global_values(config):
+    config_global_values = dict(config.items('GLOBAL'))
+
+    for path in ('mig_path', 'certs_path', 'state_path'):
+        path_value = config_global_values.get(path)
+        if not is_path_within(path_value, start=_LOCAL_MIG_BASE):
+            raise AssertionError('local config contains bad path: %s=%s' % (path, path_value))
+
+    return config_global_values
+
+
+def main(_exit=sys.exit):
+    config = _assert_local_config()
+    config_global_values = _assert_local_config_global_values(config)
+
     from mig.shared.conf import get_configuration_object
-    configuration = get_configuration_object()
+    configuration = get_configuration_object(_TEST_CONF_FILE, skip_log=True)
     logging.basicConfig(filename=None, level=logging.INFO,
                         format="%(asctime)s %(levelname)s %(message)s")
     configuration.logger = logging
@@ -66,15 +106,16 @@
     if client_id != test_id:
         print("ERROR: Expected match on IDs but found: %s vs %s" %
               (client_id, test_id))
-        sys.exit(1)
+        _exit(1)
     if client_dir != test_dir:
         print("ERROR: Expected match on dirs but found: %s vs %s" %
               (client_dir, test_dir))
-        sys.exit(1)
+        _exit(1)
     if client_short != test_short:
         print("ERROR: Expected match on %s but found: %s vs %s" %
               (short_alias, client_short, test_short))
-        sys.exit(1)
+        _exit(1)
+
 
     orig_id = '/X=ab/Y=cdef ghi/Z=klmn'
     client_dir = client_id_dir(orig_id)
@@ -106,12 +147,12 @@
     for path in legal:
         if invisible_path(path):
             print("ERROR: Expected visible on %s but not the case" % path)
-            sys.exit(1)
+            _exit(1)
     # print("check that these are invisible:")
     for path in illegal:
         if not invisible_path(path):
             print("ERROR: Expected invisible on %s but not the case" % path)
-            sys.exit(1)
+            _exit(1)
 
     print("Check script restrictions:")
     access_any = ['reqoid.py', 'docs.py', 'ls.py']
@@ -121,33 +162,36 @@
         if not allow:
             print("ERROR: Expected anon access to %s but not the case" %
                   script_name)
-            sys.exit(1)
+            _exit(1)
         (allow, msg) = allow_script(configuration, script_name, client_id)
         if not allow:
             print("ERROR: Expected auth access to %s but not the case" %
                   script_name)
-            sys.exit(1)
+            _exit(1)
     for script_name in access_auth:
         (allow, msg) = allow_script(configuration, script_name, '')
         if configuration.site_enable_gdp and allow:
             print("ERROR: Expected anon restrict to %s but not the case" %
                   script_name)
-            sys.exit(1)
+            _exit(1)
         (allow, msg) = allow_script(configuration, script_name, client_id)
         if not allow:
             print("ERROR: Expected auth access to %s but not the case" %
                   script_name)
-            sys.exit(1)
+            _exit(1)
 
     print("Check brief format list limit")
     for (size, outlen) in [(5, 15), (30, 58), (200, 63)]:
-        shortened = "%s" % brief_list(range(size))
+        shortened = "%s" % brief_list(list(range(size)))
         if len(shortened) != outlen:
             print("ERROR: Expected brief range %d list of length %d but not the case: %s" %
                   (size, outlen, len(shortened)))
-            sys.exit(1)
+            _exit(1)
 
     print("Completed shared base functions tests")
 
     print("Done running unit test on shared core functions")
-    sys.exit(0)
+    _exit(0)
+
+if __name__ == "__main__":
+    main()

mig/unittest/testcore.py

@@ -174,6 +174,7 @@ def __init__(self, *args):
     def setUp(self):
         if not self._skip_logging:
             self._reset_logging(stream=self.logger)
+        self.before_each()
 
     def tearDown(self):
         if not self._skip_logging:
@@ -191,6 +192,10 @@ def tearDown(self):
             else:
                 continue
 
+    # hooks
+    def before_each(self):
+        pass
+
     def _reset_logging(self, stream):
         root_logger = logging.getLogger()
         root_handler = root_logger.handlers[0]
@@ -234,6 +239,11 @@ def assertPathExists(self, relative_path):
         else:
             return "file"
 
+    def assertPathWithin(self, path, start=None):
+        if not is_path_within(path, start=start):
+            raise AssertionError(
+                "path %s is not within directory %s" % (path, start))
+
     @staticmethod
     def pretty_display_path(absolute_path):
         assert os.path.isabs(absolute_path)
@@ -242,6 +252,15 @@ def pretty_display_path(absolute_path):
         return relative_path
 
 
+def is_path_within(path, start=None, _msg=None):
+    try:
+        assert os.path.isabs(path), _msg
+        relative = os.path.relpath(path, start=start)
+    except:
+        return False
+    return not relative.startswith('..')
+
+
 def cleanpath(relative_path, test_case):
     assert isinstance(test_case, MigTestCase)
     tmp_path = os.path.join(TEST_OUTPUT_DIR, relative_path)