Merge remote-tracking branch 'origin/master' into edge

jonasbardino · jonasbardino · commit f88165de4d51 · 2024-06-10T12:53:51.000+02:00
diff --git a/mig/install/MiGserver-template.conf b/mig/install/MiGserver-template.conf
@@ -16,6 +16,15 @@ auto_add_oid_user = __AUTO_ADD_OID_USER__
 auto_add_oidc_user = __AUTO_ADD_OIDC_USER__
 # Auto create dedicated MiG resources from valid users
 #auto_add_resource = False
+# Apply filters to handle illegal characters e.g. in names during auto add
+# User ID fields to filter: full_name, organization, ...
+# Leave filter fields empty or unset to disable all filters and let input
+# validation simply reject user sign up if names contain such characters.
+auto_add_filter_fields = 
+# How to handle each illegal character in the configured filter fields. The
+# default is to skip each such character. Other valid options include hexlify
+# to encode each such character with the corresponding hex codepoint.
+auto_add_filter_method = skip
 # Default account expiry unless set. Renew and web login extends by default.
 cert_valid_days = __CERT_VALID_DAYS__
 oid_valid_days = __OID_VALID_DAYS__
diff --git a/mig/shared/configuration.py b/mig/shared/configuration.py
@@ -48,11 +48,11 @@
 try:
     from mig.shared.defaults import CSRF_MINIMAL, CSRF_WARN, CSRF_MEDIUM, \
         CSRF_FULL, POLICY_NONE, POLICY_WEAK, POLICY_MEDIUM, POLICY_HIGH, \
-        POLICY_MODERN, POLICY_CUSTOM, freeze_flavors, \
+        POLICY_MODERN, POLICY_CUSTOM, freeze_flavors, cert_field_order, \
         duplicati_protocol_choices, default_css_filename, keyword_any, \
         cert_valid_days, oid_valid_days, generic_valid_days, keyword_all, \
         keyword_file, keyword_env, DEFAULT_USER_ID_FORMAT, \
-        valid_user_id_formats, default_twofactor_auth_apps
+        valid_user_id_formats, valid_filter_methods, default_twofactor_auth_apps
     from mig.shared.logger import Logger, SYSLOG_GDP
     from mig.shared.html import menu_items, vgrid_items
     from mig.shared.fileio import read_file, load_json, write_file
@@ -137,6 +137,8 @@ def fix_missing(config_file, verbose=True):
         'auto_add_oid_user': False,
         'auto_add_oidc_user': False,
         'auto_add_resource': False,
+        'auto_add_filter_method': '',
+        'auto_add_filter_fields': '',
         'server_fqdn': fqdn,
         'support_email': '',
         'admin_email': '%s@%s' % (user, fqdn),
@@ -668,6 +670,8 @@ class Configuration:
     auto_add_oid_user = False
     auto_add_oidc_user = False
     auto_add_resource = False
+    auto_add_filter_method = ''
+    auto_add_filter_fields = []
 
     # ARC resource configuration (list)
     # wired-in shorthands in arcwrapper:
@@ -2521,6 +2525,18 @@ def reload_config(self, verbose, skip_log=False):
             self.auto_add_resource = config.getboolean('GLOBAL',
                                                        'auto_add_resource')
 
+        # Apply requested automatic filtering of selected auto add user fields
+        if config.has_option('GLOBAL', 'auto_add_filter_method'):
+            filter_method = config.get('GLOBAL', 'auto_add_filter_method')
+            if filter_method not in valid_filter_methods:
+                filter_method = ''
+            self.auto_add_filter_method = filter_method
+        if config.has_option('GLOBAL', 'auto_add_filter_fields'):
+            filter_fields = config.get('GLOBAL', 'auto_add_filter_fields')
+            valid_filter_fields = [i[0] for i in cert_field_order]
+            self.auto_add_filter_fields = [i for i in filter_fields.split()
+                                           if i in valid_filter_fields]
+
         # Allow override of account valid days from shared.defaults
         if config.has_option('GLOBAL', 'cert_valid_days'):
             self.cert_valid_days = config.getint('GLOBAL', 'cert_valid_days')
diff --git a/mig/shared/defaults.py b/mig/shared/defaults.py
@@ -110,6 +110,8 @@
     ('email', 'emailAddress'),
 ]
 
+valid_filter_methods = ['', 'skip', 'hexlify']
+
 sandbox_names = ['sandbox', 'oneclick', 'ps3live']
 
 email_keyword_list = ['mail', 'email']
diff --git a/mig/shared/functionality/autocreate.py b/mig/shared/functionality/autocreate.py
@@ -39,6 +39,7 @@
 
 from __future__ import absolute_import
 
+import base64
 import os
 import time
 
@@ -231,6 +232,66 @@ def split_comma_concat(value_list, sep=','):
     return result
 
 
+def lookup_filter_illegal_handler(filter_method):
+    """Get the illegal handler function to match filter_method. The output is
+    directly suitable for the filter_X helpers with illegal_handler argument.
+    """
+    # NOTE: the None value is a special case that means skip illegal values
+    if filter_method in ('', 'skip'):
+        return None
+    elif filter_method == 'hexlify':
+        def hex_wrap(val):
+            """Insert a clearly marked hex representation of val"""
+            # NOTE: use '.X' as '.x' will typically be capitalized in use anyway
+            return ".X%s" % base64.b16encode(val)
+        return hex_wrap
+    else:
+        raise ValueError("unsupported filter_method: %r" % filter_method)
+
+
+def populate_prefilters(configuration, prefilter_map, auth_type):
+    """Populate the prefilter map applied to input values before anything else
+    so that they can be used e.g. to mangle illegal values into compliance.
+    Particularly useful for making sure we keep file system names to something
+    we can actually safely handle.
+    """
+    _logger = configuration.logger
+    _logger.debug("populate prefilters for %s" % auth_type)
+    # TODO: add better reversible filters like punycode or base64 on whole name
+    filter_name = configuration.auto_add_filter_method
+    illegal_handler = lookup_filter_illegal_handler(filter_name)
+    _logger.debug("populate prefilters found filter illegal char handler %s" %
+                  illegal_handler)
+    if auth_type == AUTH_OPENID_V2:
+        if filter_name and 'full_name' in configuration.auto_add_filter_fields:
+            def _filter_helper(x):
+                return filter_commonname(x, illegal_handler)
+            # NOTE: KUIT OpenID 2.0 provides full name as 'fullname'
+            for name in ('openid.sreg.fullname', 'openid.sreg.full_name'):
+                prefilter_map[name] = _filter_helper
+    elif auth_type == AUTH_OPENID_CONNECT:
+        if configuration.auto_add_filter_method and \
+                'full_name' in configuration.auto_add_filter_fields:
+            def _filter_helper(x):
+                return filter_commonname(x, illegal_handler)
+            # NOTE: WAYF provides full name as 'name'
+            for name in ('oidc.claim.fullname', 'oidc.claim.full_name',
+                         'oidc.claim.name'):
+                prefilter_map[name] = _filter_helper
+    elif auth_type == AUTH_CERTIFICATE:
+        if configuration.auto_add_filter_method and \
+                'full_name' in configuration.auto_add_filter_fields:
+            def _filter_helper(x):
+                return filter_commonname(x, illegal_handler)
+            for name in ('cert_name', ):
+                prefilter_map[name] = _filter_helper
+    else:
+        raise ValueError("unsupported auth_type in populate_prefilters: %r" %
+                         auth_type)
+    _logger.debug("populate prefilters returns: %s" % prefilter_map)
+    return prefilter_map
+
+
 def main(client_id, user_arguments_dict, environ=None):
     """Main function used by front end"""
 
@@ -257,6 +318,8 @@ def main(client_id, user_arguments_dict, environ=None):
             output_objects.append({'object_type': 'error_text', 'text':
                                    '%s sign up not supported' % auth_flavor})
             return (output_objects, returnvalues.SYSTEM_ERROR)
+        # NOTE: simple filters to handle unsupported chars e.g. in full name
+        populate_prefilters(configuration, prefilter_map, auth_type)
     elif identity and auth_type == AUTH_OPENID_V2:
         if auth_flavor == AUTH_MIG_OID:
             base_url = configuration.migserver_https_mig_oid_url
@@ -267,9 +330,8 @@ def main(client_id, user_arguments_dict, environ=None):
             output_objects.append({'object_type': 'error_text', 'text':
                                    '%s sign up not supported' % auth_flavor})
             return (output_objects, returnvalues.SYSTEM_ERROR)
-        for name in ('openid.sreg.cn', 'openid.sreg.fullname',
-                     'openid.sreg.full_name'):
-            prefilter_map[name] = filter_commonname
+        # NOTE: simple filters to handle unsupported chars e.g. in full name
+        populate_prefilters(configuration, prefilter_map, auth_type)
     elif identity and auth_type == AUTH_OPENID_CONNECT:
         if auth_flavor == AUTH_MIG_OIDC:
             base_url = configuration.migserver_https_mig_oidc_url
@@ -286,6 +348,8 @@ def main(client_id, user_arguments_dict, environ=None):
             low_key = key.replace('OIDC_CLAIM_', 'oidc.claim.').lower()
             if low_key in oidc_keys:
                 user_arguments_dict[low_key] = [environ[key]]
+        # NOTE: simple filters to handle unsupported chars e.g. in full name
+        populate_prefilters(configuration, prefilter_map, auth_type)
     else:
         logger.error('autocreate without ID rejected for %s' % client_id)
         output_objects.append({'object_type': 'error_text',
diff --git a/mig/shared/functionality/reqpwresetaction.py b/mig/shared/functionality/reqpwresetaction.py
@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # reqpwresetaction - handle account password reset requests and send email to user
-# Copyright (C) 2003-2023  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -30,16 +30,20 @@
 from __future__ import absolute_import
 
 import os
-import time
 import tempfile
+import time
 
 from mig.shared import returnvalues
 from mig.shared.base import canonical_user_with_peers, generate_https_urls, \
     fill_distinguished_name, cert_field_map, auth_type_description, \
     mask_creds, is_gdp_user
 from mig.shared.defaults import keyword_auto, RESET_TOKEN_TTL
 from mig.shared.functional import validate_input, REJECT_UNSET
+from mig.shared.griddaemons.https import default_max_user_hits, \
+    default_user_abuse_hits, default_proto_abuse_hits, hit_rate_limit, \
+    expire_rate_limit, validate_auth_attempt
 from mig.shared.handlers import safe_handler, get_csrf_limit
+from mig.shared.html import themed_styles, themed_scripts
 from mig.shared.init import initialize_main_variables, find_entry
 from mig.shared.notification import send_email
 from mig.shared.pwcrypto import generate_reset_token
@@ -61,7 +65,22 @@ def main(client_id, user_arguments_dict):
     """Main function used by front end"""
 
     (configuration, logger, output_objects, op_name) = \
-        initialize_main_variables(client_id, op_header=False, op_menu=False)
+        initialize_main_variables(client_id, op_header=False, op_title=False,
+                                  op_menu=False)
+    # IMPORTANT: no title in init above so we MUST call it immediately here
+    #            or basic styling will break on e.g. the check user result.
+    styles = themed_styles(configuration)
+    scripts = themed_scripts(configuration, logged_in=False)
+    title_entry = {'object_type': 'title',
+                   'text': '%s reset account password request' %
+                   configuration.short_title,
+                   'skipmenu': True, 'style': styles, 'script': scripts}
+    output_objects.append(title_entry)
+    output_objects.append({'object_type': 'header', 'text':
+                           '%s reset account password request' %
+                           configuration.short_title
+                           })
+
     defaults = signature(configuration)[1]
     (validate_status, accepted) = validate_input(user_arguments_dict,
                                                  defaults, output_objects,
@@ -71,15 +90,20 @@ def main(client_id, user_arguments_dict):
         logger.warning('%s invalid input: %s' % (op_name, accepted))
         return (accepted, returnvalues.CLIENT_ERROR)
 
-    title_entry = find_entry(output_objects, 'title')
-    title_entry['text'] = '%s reset account password request' % \
-                          configuration.short_title
-    title_entry['skipmenu'] = True
-    output_objects.append({'object_type': 'header', 'text':
-                           '%s reset account password request' %
-                           configuration.short_title
-                           })
-
+    # Seconds to delay next reset attempt after hitting rate limit
+    delay_retry = 900
+    scripts['init'] += '''
+function update_reload_counter(cnt, delay) {
+    var remain = (delay - cnt);
+    $("#reload_counter").html(remain.toString());
+    if (cnt >= delay) {
+        /* Load previous page again without re-posting last attempt */
+        location = history.back();
+    } else {
+        setTimeout(function() { update_reload_counter(cnt+1, delay); }, 1000);
+    }
+}
+    '''
     smtp_server = configuration.smtp_server
 
     cert_id = accepted['cert_id'][-1].strip()
@@ -118,8 +142,68 @@ def main(client_id, user_arguments_dict):
         return (output_objects, returnvalues.CLIENT_ERROR)
 
     mig_user = os.environ.get('USER', 'mig')
+    client_addr = os.environ.get('REMOTE_ADDR', None)
+    tcp_port = int(os.environ.get('REMOTE_PORT', '0'))
     anon_migoid_url = configuration.migserver_https_sid_url
 
+    status = returnvalues.OK
+
+    # Rate limit password reset attempts for any cert_id from source addr
+    # to prevent excessive requests spamming users or overloading server.
+    # We do so no matter if cert_id matches a valid user to prevent disclosure.
+    # Rate limit does not affect reset for another ID from same address as
+    # that may be perfectly valid e.g. if behind a shared NAT-gateway.
+    proto = 'https'
+    disconnect, exceeded_rate_limit = False, False
+    # Clean up expired entries in persistent rate limit cache
+    expire_rate_limit(configuration, proto, fail_cache=delay_retry,
+                      expire_delay=delay_retry)
+    if hit_rate_limit(configuration, proto, client_addr, cert_id,
+                      max_user_hits=1):
+        exceeded_rate_limit = True
+    # Update rate limits and write to auth log
+    (authorized, disconnect) = validate_auth_attempt(
+        configuration,
+        proto,
+        op_name,
+        cert_id,
+        client_addr,
+        tcp_port,
+        secret=None,
+        authtype_enabled=True,
+        auth_reset=True,
+        exceeded_rate_limit=exceeded_rate_limit,
+        user_abuse_hits=default_user_abuse_hits,
+        proto_abuse_hits=default_proto_abuse_hits,
+        max_secret_hits=1,
+        skip_notify=True,
+    )
+
+    if exceeded_rate_limit or disconnect:
+        logger.warning('Throttle %s for %s from %s - past rate limit' %
+                       (op_name, cert_id, client_addr))
+        # NOTE: we keep actual result in plain text for json extract
+        output_objects.append({'object_type': 'html_form', 'text': '''
+<div class="vertical-spacer"></div>
+<div class="error leftpad errortext">
+'''})
+        output_objects.append({'object_type': 'text', 'text': """
+Invalid input or rate limit exceeded - please wait %d seconds before retrying.
+""" % delay_retry
+                               })
+        output_objects.append({'object_type': 'html_form', 'text': '''
+</div>
+<div class="vertical-spacer"></div>
+<div class="info leftpad">
+Origin will reload automatically in <span id="reload_counter">%d</span> seconds.
+</div>
+</div>
+''' % delay_retry})
+        scripts['ready'] += '''
+    setTimeout(function() { update_reload_counter(1, %d); }, 1000);
+''' % delay_retry
+        return (output_objects, status)
+
     search_filter = default_search()
     if '/' in cert_id:
         search_filter['distinguished_name'] = cert_id
@@ -215,4 +299,4 @@ def main(client_id, user_arguments_dict):
     output_objects.append(
         {'object_type': 'link', 'destination': 'javascript:history.back();',
          'class': 'genericbutton', 'text': "Back"})
-    return (output_objects, returnvalues.OK)
+    return (output_objects, status)
diff --git a/mig/shared/functionality/twofactor.py b/mig/shared/functionality/twofactor.py
@@ -48,7 +48,7 @@
 from mig.shared.defaults import twofactor_cookie_ttl, AUTH_MIG_OID, \
     AUTH_EXT_OID, AUTH_MIG_OIDC, AUTH_EXT_OIDC
 from mig.shared.functional import validate_input
-from mig.shared.griddaemons.openid import default_max_user_hits, \
+from mig.shared.griddaemons.https import default_max_user_hits, \
     default_user_abuse_hits, default_proto_abuse_hits, hit_rate_limit, \
     expire_rate_limit, validate_auth_attempt
 from mig.shared.init import initialize_main_variables
@@ -324,8 +324,8 @@ def main(client_id, user_arguments_dict, environ=None):
             )
 
         if exceeded_rate_limit or disconnect:
-            logger.warning('Throttle twofactor from %s (%s) - past rate limit'
-                           % (client_id, client_addr))
+            logger.warning('Throttle %s from %s (%s) - past rate limit' %
+                           (op_name, client_id, client_addr))
             # NOTE: we keep actual result in plain text for json extract
             output_objects.append({'object_type': 'html_form', 'text': '''
 <div class="vertical-spacer"></div>
diff --git a/mig/shared/griddaemons/auth.py b/mig/shared/griddaemons/auth.py
diff --git a/mig/shared/griddaemons/https.py b/mig/shared/griddaemons/https.py
diff --git a/mig/shared/safeinput.py b/mig/shared/safeinput.py

Original file line number	Diff line number	Diff line change
`@@ -110,6 +110,8 @@`
`110`	`110`	`('email', 'emailAddress'),`
`111`	`111`	`]`
`112`	`112`
	`113`	`+valid_filter_methods = ['', 'skip', 'hexlify']`
	`114`	`+`
`113`	`115`	`sandbox_names = ['sandbox', 'oneclick', 'ps3live']`
`114`	`116`
`115`	`117`	`email_keyword_list = ['mail', 'email']`