implement consistent handling of unsupported characters in auto add user backend (autocreate). Adds configuration option to define which ID fields to filter and how. By default no filtering is applied, but if enabled on full name the default is to skip unsupported characters no matter what auth type is used (OpenID 2.0, OpenID Connect or user certificates). One can set filter method to 'hexlify' instead to get a simple reversible hex translation of such unsupported chars. This should basically address migrid-sync issue 35, but we probably want to add more reversible translators later.

git-svn-id: svn+ssh://svn.code.sf.net/p/migrid/code/trunk@6047 b75ad72c-e7d7-11dd-a971-7dbc132099af

Author: jonasbardino

mig/install/MiGserver-template.conf

@@ -16,6 +16,15 @@ auto_add_oid_user = __AUTO_ADD_OID_USER__
 auto_add_oidc_user = __AUTO_ADD_OIDC_USER__
 # Auto create dedicated MiG resources from valid users
 #auto_add_resource = False
+# Apply filters to handle illegal characters e.g. in names during auto add
+# User ID fields to filter: full_name, organization, ...
+# Leave filter fields empty or unset to disable all filters and let input
+# validation simply reject user sign up if names contain such characters.
+auto_add_filter_fields = 
+# How to handle each illegal character in the configured filter fields. The
+# default is to skip each such character. Other valid options include hexlify
+# to encode each such character with the corresponding hex codepoint.
+auto_add_filter_method = skip
 # Default account expiry unless set. Renew and web login extends by default.
 cert_valid_days = __CERT_VALID_DAYS__
 oid_valid_days = __OID_VALID_DAYS__

mig/shared/configuration.py

@@ -48,11 +48,11 @@
 try:
     from mig.shared.defaults import CSRF_MINIMAL, CSRF_WARN, CSRF_MEDIUM, \
         CSRF_FULL, POLICY_NONE, POLICY_WEAK, POLICY_MEDIUM, POLICY_HIGH, \
-        POLICY_MODERN, POLICY_CUSTOM, freeze_flavors, \
+        POLICY_MODERN, POLICY_CUSTOM, freeze_flavors, cert_field_order, \
         duplicati_protocol_choices, default_css_filename, keyword_any, \
         cert_valid_days, oid_valid_days, generic_valid_days, keyword_all, \
         keyword_file, keyword_env, DEFAULT_USER_ID_FORMAT, \
-        valid_user_id_formats, default_twofactor_auth_apps
+        valid_user_id_formats, valid_filter_methods, default_twofactor_auth_apps
     from mig.shared.logger import Logger, SYSLOG_GDP
     from mig.shared.html import menu_items, vgrid_items
     from mig.shared.fileio import read_file, load_json, write_file
@@ -137,6 +137,8 @@ def fix_missing(config_file, verbose=True):
         'auto_add_oid_user': False,
         'auto_add_oidc_user': False,
         'auto_add_resource': False,
+        'auto_add_filter_method': '',
+        'auto_add_filter_fields': '',
         'server_fqdn': fqdn,
         'support_email': '',
         'admin_email': '%s@%s' % (user, fqdn),
@@ -668,6 +670,8 @@ class Configuration:
     auto_add_oid_user = False
     auto_add_oidc_user = False
     auto_add_resource = False
+    auto_add_filter_method = ''
+    auto_add_filter_fields = []
 
     # ARC resource configuration (list)
     # wired-in shorthands in arcwrapper:
@@ -2521,6 +2525,18 @@ def reload_config(self, verbose, skip_log=False):
             self.auto_add_resource = config.getboolean('GLOBAL',
                                                        'auto_add_resource')
 
+        # Apply requested automatic filtering of selected auto add user fields
+        if config.has_option('GLOBAL', 'auto_add_filter_method'):
+            filter_method = config.get('GLOBAL', 'auto_add_filter_method')
+            if filter_method not in valid_filter_methods:
+                filter_method = ''
+            self.auto_add_filter_method = filter_method
+        if config.has_option('GLOBAL', 'auto_add_filter_fields'):
+            filter_fields = config.get('GLOBAL', 'auto_add_filter_fields')
+            valid_filter_fields = [i[0] for i in cert_field_order]
+            self.auto_add_filter_fields = [i for i in filter_fields.split()
+                                           if i in valid_filter_fields]
+
         # Allow override of account valid days from shared.defaults
         if config.has_option('GLOBAL', 'cert_valid_days'):
             self.cert_valid_days = config.getint('GLOBAL', 'cert_valid_days')

mig/shared/defaults.py

@@ -110,6 +110,8 @@
     ('email', 'emailAddress'),
 ]
 
+valid_filter_methods = ['', 'skip', 'hexlify']
+
 sandbox_names = ['sandbox', 'oneclick', 'ps3live']
 
 email_keyword_list = ['mail', 'email']

mig/shared/functionality/autocreate.py

@@ -39,6 +39,7 @@
 
 from __future__ import absolute_import
 
+import base64
 import os
 import time
 
@@ -231,6 +232,66 @@ def split_comma_concat(value_list, sep=','):
     return result
 
 
+def lookup_filter_illegal_handler(filter_method):
+    """Get the illegal handler function to match filter_method. The output is
+    directly suitable for the filter_X helpers with illegal_handler argument.
+    """
+    # NOTE: the None value is a special case that means skip illegal values
+    if filter_method in ('', 'skip'):
+        return None
+    elif filter_method == 'hexlify':
+        def hex_wrap(val):
+            """Insert a clearly marked hex representation of val"""
+            # NOTE: use '.X' as '.x' will typically be capitalized in use anyway
+            return ".X%s" % base64.b16encode(val)
+        return hex_wrap
+    else:
+        raise ValueError("unsupported filter_method: %r" % filter_method)
+
+
+def populate_prefilters(configuration, prefilter_map, auth_type):
+    """Populate the prefilter map applied to input values before anything else
+    so that they can be used e.g. to mangle illegal values into compliance.
+    Particularly useful for making sure we keep file system names to something
+    we can actually safely handle.
+    """
+    _logger = configuration.logger
+    _logger.debug("populate prefilters for %s" % auth_type)
+    # TODO: add better reversible filters like punycode or base64 on whole name
+    filter_name = configuration.auto_add_filter_method
+    illegal_handler = lookup_filter_illegal_handler(filter_name)
+    _logger.debug("populate prefilters found filter illegal char handler %s" %
+                  illegal_handler)
+    if auth_type == AUTH_OPENID_V2:
+        if filter_name and 'full_name' in configuration.auto_add_filter_fields:
+            def _filter_helper(x):
+                return filter_commonname(x, illegal_handler)
+            # NOTE: KUIT OpenID 2.0 provides full name as 'fullname'
+            for name in ('openid.sreg.fullname', 'openid.sreg.full_name'):
+                prefilter_map[name] = _filter_helper
+    elif auth_type == AUTH_OPENID_CONNECT:
+        if configuration.auto_add_filter_method and \
+                'full_name' in configuration.auto_add_filter_fields:
+            def _filter_helper(x):
+                return filter_commonname(x, illegal_handler)
+            # NOTE: WAYF provides full name as 'name'
+            for name in ('oidc.claim.fullname', 'oidc.claim.full_name',
+                         'oidc.claim.name'):
+                prefilter_map[name] = _filter_helper
+    elif auth_type == AUTH_CERTIFICATE:
+        if configuration.auto_add_filter_method and \
+                'full_name' in configuration.auto_add_filter_fields:
+            def _filter_helper(x):
+                return filter_commonname(x, illegal_handler)
+            for name in ('cert_name', ):
+                prefilter_map[name] = _filter_helper
+    else:
+        raise ValueError("unsupported auth_type in populate_prefilters: %r" %
+                         auth_type)
+    _logger.debug("populate prefilters returns: %s" % prefilter_map)
+    return prefilter_map
+
+
 def main(client_id, user_arguments_dict, environ=None):
     """Main function used by front end"""
 
@@ -257,6 +318,8 @@ def main(client_id, user_arguments_dict, environ=None):
             output_objects.append({'object_type': 'error_text', 'text':
                                    '%s sign up not supported' % auth_flavor})
             return (output_objects, returnvalues.SYSTEM_ERROR)
+        # NOTE: simple filters to handle unsupported chars e.g. in full name
+        populate_prefilters(configuration, prefilter_map, auth_type)
     elif identity and auth_type == AUTH_OPENID_V2:
         if auth_flavor == AUTH_MIG_OID:
             base_url = configuration.migserver_https_mig_oid_url
@@ -267,9 +330,8 @@ def main(client_id, user_arguments_dict, environ=None):
             output_objects.append({'object_type': 'error_text', 'text':
                                    '%s sign up not supported' % auth_flavor})
             return (output_objects, returnvalues.SYSTEM_ERROR)
-        for name in ('openid.sreg.cn', 'openid.sreg.fullname',
-                     'openid.sreg.full_name'):
-            prefilter_map[name] = filter_commonname
+        # NOTE: simple filters to handle unsupported chars e.g. in full name
+        populate_prefilters(configuration, prefilter_map, auth_type)
     elif identity and auth_type == AUTH_OPENID_CONNECT:
         if auth_flavor == AUTH_MIG_OIDC:
             base_url = configuration.migserver_https_mig_oidc_url
@@ -286,6 +348,8 @@ def main(client_id, user_arguments_dict, environ=None):
             low_key = key.replace('OIDC_CLAIM_', 'oidc.claim.').lower()
             if low_key in oidc_keys:
                 user_arguments_dict[low_key] = [environ[key]]
+        # NOTE: simple filters to handle unsupported chars e.g. in full name
+        populate_prefilters(configuration, prefilter_map, auth_type)
     else:
         logger.error('autocreate without ID rejected for %s' % client_id)
         output_objects.append({'object_type': 'error_text',

mig/shared/safeinput.py

@@ -1371,10 +1371,11 @@ def filter_date(contents):
     return __filter_contents(contents, digits + '-/')
 
 
-def filter_commonname(contents):
+def filter_commonname(contents, illegal_handler=None):
     """Filter supplied contents to only contain valid commonname characters"""
 
-    return __filter_contents(contents, VALID_NAME_CHARACTERS, COMMON_ACCENTED)
+    return __filter_contents(contents, VALID_NAME_CHARACTERS, COMMON_ACCENTED,
+                             illegal_handler=illegal_handler)
 
 
 def filter_organization(contents):