Merge remote-tracking branch 'origin/master' into edge

Author: jonasbardino

mig/install/generateconfs.py

@@ -355,8 +355,8 @@ def usage(options):
         if val == 'DEFAULT':
             del settings[key]
     conf = generate_confs(**settings)
-    # print "DEBUG: %s" % conf
-    instructions_path = "%(destination)s/instructions.txt" % conf
+    # TODO: avoid reconstructing this path (also done inside generate_confs)
+    instructions_path = "%s/instructions.txt" % conf['destination_path']
     try:
         instructions_fd = open(instructions_path, "r")
         instructions = instructions_fd.read()

mig/shared/defaults.py

@@ -32,6 +32,7 @@
 import os
 import sys
 
+MIG_BASE = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..'))
 MIG_ENV = os.getenv('MIG_ENV', 'default')
 
 # NOTE: python3 switched strings to use unicode by default in contrast to bytes
@@ -76,6 +77,31 @@
 any_protocol = keyword_any
 any_state = keyword_any
 
+mig_user = {
+    'default': 'mig',
+    'local': keyword_auto,
+}[MIG_ENV]
+
+mig_group = {
+    'default': 'mig',
+    'local': keyword_auto,
+}[MIG_ENV]
+
+default_source = {
+    'default': keyword_auto,
+    'local': os.path.join(MIG_BASE, "mig/install"),
+}[MIG_ENV]
+
+default_destination = {
+    'default': keyword_auto,
+    'local': os.path.join(MIG_BASE, "envhelp/output/confs"),
+}[MIG_ENV]
+
+default_enable_events = {
+    'default': True,
+    'local': False
+}[MIG_ENV]
+
 AUTH_NONE, AUTH_GENERIC, AUTH_CERTIFICATE = "None", "Generic", "X.509 Certificate"
 AUTH_OPENID_CONNECT, AUTH_OPENID_V2 = "OpenID Connect", "OpenID 2.0"
 

mig/shared/install.py

@@ -40,6 +40,7 @@
 import base64
 import crypt
 import datetime
+import grp
 import os
 import pwd
 import random
@@ -49,11 +50,13 @@
 import sys
 
 from mig.shared.defaults import default_http_port, default_https_port, \
+    mig_user, mig_group, default_source, default_destination, \
     auth_openid_mig_db, auth_openid_ext_db, STRONG_TLS_CIPHERS, \
     STRONG_TLS_CURVES, STRONG_SSH_KEXALGOS, STRONG_SSH_LEGACY_KEXALGOS, \
     STRONG_SSH_CIPHERS, STRONG_SSH_LEGACY_CIPHERS, STRONG_SSH_MACS, \
     STRONG_SSH_LEGACY_MACS, CRACK_USERNAME_REGEX, CRACK_WEB_REGEX, \
     keyword_any, keyword_auto
+from mig.shared.compat import ensure_native_string
 from mig.shared.fileio import read_file, read_file_lines, write_file, \
     write_file_lines
 from mig.shared.htmlgen import menu_items
@@ -187,8 +190,8 @@ def template_remove(template_file, remove_pattern):
 def generate_confs(
     # NOTE: make sure command line args with white-space are properly wrapped
     generateconfs_command=subprocess.list2cmdline(sys.argv),
-    source=os.path.dirname(sys.argv[0]),
-    destination=os.path.dirname(sys.argv[0]),
+    source=default_source,
+    destination=default_destination,
     destination_suffix="",
     base_fqdn='',
     public_fqdn='',
@@ -217,8 +220,9 @@ def generate_confs(
     jupyter_services_desc='{}',
     cloud_services='',
     cloud_services_desc='{}',
-    user='mig',
-    group='mig',
+    user=mig_user,
+    group=mig_group,
+    timezone=keyword_auto,
     apache_version='2.4',
     apache_etc='/etc/apache2',
     apache_run='/var/run',
@@ -358,6 +362,8 @@ def generate_confs(
     openid_port=8443,
     openid_show_port='',
     openid_session_lifetime=43200,
+    seafile_secret=keyword_auto,
+    seafile_ccnetid=keyword_auto,
     seafile_seahub_port=8000,
     seafile_seafhttp_port=8082,
     seafile_client_port=13419,
@@ -403,12 +409,31 @@ def generate_confs(
     quota_backend='lustre',
     quota_user_limit=(1024**4),
     quota_vgrid_limit=(1024**4),
+    _getpwnam=pwd.getpwnam,
 ):
     """Generate Apache and MiG server confs with specified variables"""
 
     # Read out dictionary of args with defaults and overrides
 
-    expanded = locals()
+    expanded = dict(locals())
+
+    # expand any directory path specific as "auto" relative to CWD
+    if source is keyword_auto:
+        expanded['source'] = os.path.dirname(sys.argv[0])
+        source = expanded['source']
+    if destination is keyword_auto:
+        expanded['destination'] = os.path.dirname(sys.argv[0])
+        destination = expanded['destination']
+
+    # expand any user information marked as "auto" based on the environment
+    if user is keyword_auto:
+        user = pwd.getpwuid(os.getuid())[0]
+    if group is keyword_auto:
+        group = grp.getgrgid(os.getgid())[0]
+
+    # finalize a destination path up-front
+    expanded['destination_path'] = "%s%s" % (destination, destination_suffix)
+    destination_path = expanded['destination_path']
 
     # Backwards compatibility with old name
     if public_port and not public_http_port:
@@ -656,7 +681,7 @@ def generate_confs(
     user_dict['__QUOTA_VGRID_LIMIT__'] = "%s" % quota_vgrid_limit
 
     # Needed for PAM/NSS
-    pw_info = pwd.getpwnam(user)
+    pw_info = _getpwnam(user)
     user_dict['__MIG_UID__'] = "%s" % (pw_info.pw_uid)
     user_dict['__MIG_GID__'] = "%s" % (pw_info.pw_gid)
 
@@ -916,22 +941,36 @@ def generate_confs(
             prio_duplicati_protocols.append('davs')
     user_dict['__DUPLICATI_PROTOCOLS__'] = ' '.join(prio_duplicati_protocols)
 
-    sys_timezone = 'UTC'
-    timezone_cmd = ["/usr/bin/timedatectl", "status"]
-    try:
-        timezone_proc = subprocess_popen(timezone_cmd, stdout=subprocess_pipe)
-        for line in timezone_proc.stdout.readlines():
-            line = line.strip()
-            if not line.startswith("Time zone: "):
-                continue
-            sys_timezone = line.replace("Time zone: ", "").split(" ", 1)[0]
-    except Exception as exc:
-        print("WARNING: failed to extract system time zone: %s" % exc)
-    user_dict['__SEAFILE_TIMEZONE__'] = sys_timezone
-    user_dict['__SEAFILE_SECRET_KEY__'] = base64.b64encode(
-        os.urandom(32)).lower()
-    user_dict['__SEAFILE_CCNET_ID__'] = base64.b16encode(
-        os.urandom(20)).lower()
+    if timezone is keyword_auto:
+        # attempt to detect the timezone
+        sys_timezone = None
+        try:
+            timezone_cmd = ["/usr/bin/timedatectl", "status"]
+            timezone_proc = subprocess_popen(timezone_cmd, stdout=subprocess_pipe)
+            for line in timezone_proc.stdout.readlines():
+                line = ensure_native_string(line.strip())
+                if not line.startswith("Time zone: "):
+                    continue
+                sys_timezone = line.replace("Time zone: ", "").split(" ", 1)[0]
+        except OSError as exc:
+            # warn about any issues executing the command but continue
+            pass
+        if sys_timezone is None:
+            print("WARNING: failed to extract system time zone; defaulting to UTC")
+            sys_timezone = 'UTC'
+
+        timezone = sys_timezone
+
+    user_dict['__SEAFILE_TIMEZONE__'] = timezone
+
+    if seafile_secret is keyword_auto:
+        seafile_secret = ensure_native_string(base64.b64encode(os.urandom(32))).lower()
+    user_dict['__SEAFILE_SECRET_KEY__'] = seafile_secret
+
+    if seafile_ccnetid is keyword_auto:
+        seafile_ccnetid = ensure_native_string(base64.b64encode(os.urandom(20))).lower()
+    user_dict['__SEAFILE_CCNET_ID__'] = seafile_ccnetid
+
     user_dict['__SEAFILE_SHORT_NAME__'] = short_title.replace(' ', '-')
     # IMPORTANT: we discriminate on local and remote seafile service
     #            for local ones we partly integrate directly with apache etc.
@@ -1584,14 +1623,16 @@ def generate_confs(
     user_dict['__ALL_OIDC_PROVIDER_META_URLS__'] = ' '.join(
         all_oidc_provider_meta_urls)
 
-    destination_path = "%s%s" % (destination, destination_suffix)
     if not os.path.islink(destination) and os.path.isdir(destination):
         print("ERROR: Legacy %s dir in the way - please remove first" %
               destination)
         sys.exit(1)
     try:
         os.makedirs(destination_path)
-        if os.path.exists(destination):
+    except OSError:
+        pass
+    try:
+        if os.path.lexists(destination):
             os.remove(destination)
         os.symlink(destination_path, destination)
     except OSError:
@@ -1691,16 +1732,17 @@ def generate_confs(
                                                      default_https_port])
             user_dict['__SID_URL__'] += ':%(__SID_PORT__)s' % user_dict
 
-    if digest_salt == keyword_auto:
+    if digest_salt is keyword_auto:
         # Generate random hex salt for scrambling saved digest credentials
-        digest_salt = base64.b16encode(os.urandom(16))
-    if crypto_salt == keyword_auto:
-        # Generate random hex salt for various crypto helpers
-        crypto_salt = base64.b16encode(os.urandom(16))
-
+        digest_salt = ensure_native_string(base64.b16encode(os.urandom(16)))
     user_dict['__DIGEST_SALT__'] = digest_salt
+
+    if crypto_salt is keyword_auto:
+        # Generate random hex salt for various crypto helpers
+        crypto_salt = ensure_native_string(base64.b16encode(os.urandom(16)))
     user_dict['__CRYPTO_SALT__'] = crypto_salt
 
+
     # Greedy match trailing space for all the values to uncomment stuff
     strip_trailing_space = ['__IF_SEPARATE_PORTS__', '__APACHE_PRE2.4__',
                             '__APACHE_RECENT__']
@@ -1989,7 +2031,7 @@ def generate_confs(
 sudo cp %(destination)s/migacctexpire /etc/cron.monthly/
 
 ''' % expanded
-    instructions_path = "%s/instructions.txt" % destination
+    instructions_path = "%s/instructions.txt" % destination_path
     if not write_file(instructions, instructions_path, None):
         print("could not write instructions ot %s" % instructions_path)
     return expanded

recommended.txt

@@ -14,7 +14,6 @@ requests
 #cracklib
 pdfkit
 xvfbwrapper
-pyotp
 openstackclient
 pyyaml
 nbformat

requirements.txt

@@ -1,6 +1,8 @@
 # migrid dependencies on a format suitable for pip install as described on 
 # https://pip.pypa.io/en/stable/reference/requirement-specifiers/
 future
+pyotp;python_version >= "3"
+pyotp<2.4;python_version < "3"
 pyyaml
 
 # NOTE: additional optional dependencies depending on site conf are listed

tests/fixture/confs-stdlocal/MiG-daemons-filter.conf

@@ -0,0 +1,28 @@
+# Custom MiG network daemon filter to detect attempts to brute-force guess user passwords 
+# and common scanning bots
+
+[DEFAULT]
+normal = ^.* WARNING IP: <HOST>, Protocol: .+, Type: .+, Username: .+, Message: Exceeded rate limit$
+
+# Authlog handles password crack detection
+pw_crack = ^.* CRITICAL IP: <HOST>, Protocol: .+, Type: .+, Username: .+, Message: (Crack username detected|Abuse limit reached)$
+
+# Repeated native sftp handshake errors from weak client security indicate a scan bot
+handshake = ^.* WARNING client negotiation errors for \('<HOST>', [0-9]+\): (Incompatible ssh (peer|server) \(no acceptable (kex algorithm|ciphers|macs|host key)\))?$
+
+# Common web vulnerability scans are a clear sign of malice
+webscan = ^.* ERROR got path from <HOST> with invalid root: /home/mig/state/webserver_home/((HNAP1|GponForm|provisioning|provision|prov|polycom|yealink|CertProv|phpmyadmin|admin|cfg|wp|wordpress|cms|blog|old|new|test|dev|tmp|temp|remote|mgmt|properties|authenticate|tmui|ddem|a2billing|vtigercrm|secure|rpc|recordings|dana-na)(/.*|)|.*(Login|login|logon|configuration|header|admin|index)\.(php|jsp|asp)|(api/v1/pods|Telerik.Web.UI.WebResource.axd))$
+
+
+[INCLUDES]
+before = common.conf
+
+[Definition]
+failregex = %(normal)s
+#ignoreregex =
+
+[Init]
+maxlines = 1
+# Use pyinotify or poll on native log file rather than systemd journal
+#journalmatch = _SYSTEMD_UNIT=migrid.service + _COMM=grid_webdavs.py
+journalmatch = 

tests/fixture/confs-stdlocal/MiG-daemons-handshake-filter.conf

@@ -0,0 +1,7 @@
+# Custom MiG network daemon filter to detect and ban repeated handshake errors common for bots
+
+[INCLUDES]
+before = MiG-daemons.conf
+
+[Definition]
+failregex = %(handshake)s