Manually merge PR137 to rework the handling of storage_protocols and duplicati_protocols with default auto detection and elimination of redundancy for duplicati protocol order. Should wrap up the last bits of logic regarding the proposed workaround for github issue 119 and leave only the actual exposure of the new wwwserver_max_bytes conf option still missing.

git-svn-id: svn+ssh://svn.code.sf.net/p/migrid/code/trunk@6155 b75ad72c-e7d7-11dd-a971-7dbc132099af

Author: jonasbardino

mig/install/MiGserver-template.conf

@@ -187,9 +187,15 @@ mig_server_id = %(server_fqdn)s.0
 empty_job_name = no_grid_jobs_in_grid_scheduler
 notify_protocols = email
 smtp_server = __SMTP_SERVER__
-storage_protocols = __STORAGE_PROTOCOLS__
 gdp_email_notify = __GDP_EMAIL_NOTIFY__
 
+# Optional space-separated prioritized list of efficient storage access
+# protocols to advertize to clients. Leave to AUTO to use the ones actually
+# enabled with the corresponding enable_SERVICE options. Default is AUTO and
+# other allowed values are one or more of sftp, ftps and davs.
+# NOTE: the sftpsubsys service is advertized as just sftp to fit the protocol.
+storage_protocols = __STORAGE_PROTOCOLS__
+
 # Optional extra service interfaces with common structure
 # * user_X_address is the host address to listen on
 # * user_X_port is the host port to listen on

mig/shared/configuration.py

@@ -50,10 +50,10 @@
     from mig.shared.defaults import CSRF_MINIMAL, CSRF_WARN, CSRF_MEDIUM, \
         CSRF_FULL, POLICY_NONE, POLICY_WEAK, POLICY_MEDIUM, POLICY_HIGH, \
         POLICY_MODERN, POLICY_CUSTOM, freeze_flavors, cert_field_order, \
-        duplicati_protocol_choices, default_css_filename, keyword_any, \
-        cert_valid_days, oid_valid_days, generic_valid_days, keyword_all, \
-        keyword_file, keyword_env, DEFAULT_USER_ID_FORMAT, \
-        valid_user_id_formats, valid_filter_methods, default_twofactor_auth_apps
+        default_css_filename, keyword_any, keyword_auto, keyword_all, \
+        keyword_file, keyword_env, cert_valid_days, oid_valid_days, \
+        generic_valid_days, DEFAULT_USER_ID_FORMAT, valid_user_id_formats, \
+        valid_filter_methods, default_twofactor_auth_apps
     from mig.shared.logger import Logger, SYSLOG_GDP
     from mig.shared.htmlgen import menu_items, vgrid_items
     from mig.shared.fileio import read_file, load_json, write_file
@@ -264,7 +264,6 @@ def fix_missing(config_file, verbose=True):
         'user_seafile_auth': ['password'],
         'user_seafile_local_instance': False,
         'user_seafile_ro_access': False,
-        'user_duplicati_protocols': [],
         'user_cloud_console_access': [],
         'user_cloud_ssh_auth': ['publickey'],
         'user_cloud_alias': '',
@@ -528,7 +527,7 @@ def fix_missing(config_file, verbose=True):
     'user_seafile_alias': '',
     'user_seafile_local_instance': False,
     'user_seafile_ro_access': False,
-    'user_duplicati_protocols': [],
+    'user_duplicati_protocols': keyword_auto,
     'user_cloud_console_access': [],
     'user_cloud_ssh_auth': ['publickey'],
     'user_cloud_alias': '',
@@ -619,7 +618,7 @@ def fix_missing(config_file, verbose=True):
     'scriptlanguages': [],
     'jobtypes': [],
     'lrmstypes': [],
-    'storage_protocols': [],
+    'storage_protocols': keyword_auto,
     'server_cert': '',
     'server_key': '',
     'passphrase_file': '',
@@ -1284,6 +1283,17 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
                                               'user_ftps_alias')
         if config.has_option('GLOBAL', 'user_ftps_log'):
             self.user_ftps_log = config.get('GLOBAL', 'user_ftps_log')
+
+        # NOTE: prioritized order based on performance and robustness.
+        #       Needed by duplicati and storage protocols setup below.
+        prio_storage_protos = []
+        if self.site_enable_sftp_subsys or self.site_enable_sftp:
+            prio_storage_protos.append('sftp')
+        if self.site_enable_ftps:
+            prio_storage_protos.append('ftps')
+        if self.site_enable_davs:
+            prio_storage_protos.append('davs')
+
         if config.has_option('SITE', 'enable_seafile'):
             self.site_enable_seafile = config.getboolean(
                 'SITE', 'enable_seafile')
@@ -1321,10 +1331,21 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
         else:
             self.site_enable_duplicati = False
         if config.has_option('GLOBAL', 'user_duplicati_protocols'):
-            allowed_protos = [j for (i, j) in duplicati_protocol_choices]
-            protos = config.get('GLOBAL', 'user_duplicati_protocols').split()
+            allowed_protos = prio_storage_protos
+            plain_val = config.get('GLOBAL', 'user_duplicati_protocols')
+            protos = [i for i in plain_val.split() if i]
+            # Append missing supported protocols for AUTO and filter invalid
+            if keyword_auto in protos:
+                protos = [i for i in protos if i != keyword_auto] +\
+                         [i for i in allowed_protos if i not in protos]
             valid_protos = [i for i in protos if i in allowed_protos]
+            if protos != valid_protos:
+                invalid_protos = [i for i in protos if i not in valid_protos]
+                self.logger.warning("invalid duplicati_protocol value(s): %s" %
+                                    ', '.join(invalid_protos))
             self.user_duplicati_protocols = valid_protos
+        else:
+            self.user_duplicati_protocols = prio_storage_protos
         if config.has_option('SITE', 'enable_cloud'):
             self.site_enable_cloud = config.getboolean(
                 'SITE', 'enable_cloud')
@@ -1589,8 +1610,22 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
         else:
             self.notify_protocols = []
         if config.has_option('GLOBAL', 'storage_protocols'):
-            self.storage_protocols = config.get(
-                'GLOBAL', 'storage_protocols').split()
+            allowed_protos = prio_storage_protos
+            plain_val = config.get('GLOBAL', 'storage_protocols')
+            protos = [i for i in plain_val.split() if i]
+            # Append missing supported protocols for AUTO and filter invalid
+            if keyword_auto in protos:
+                protos = [i for i in protos if i != keyword_auto] +\
+                         [i for i in allowed_protos if i not in protos]
+            valid_protos = [i for i in protos if i in allowed_protos]
+            if protos != valid_protos:
+                invalid_protos = [i for i in protos if i not in valid_protos]
+                self.logger.warning("invalid storage_protocols value(s): %s" %
+                                    ', '.join(invalid_protos))
+            self.storage_protocols = valid_protos
+        else:
+            self.storage_protocols = prio_storage_protos
+
         if config.has_option('SITE', 'enable_jupyter'):
             self.site_enable_jupyter = config.getboolean(
                 'SITE', 'enable_jupyter')

mig/shared/defaults.py

@@ -424,9 +424,6 @@
                     'rsyncssh': 'RSYNC over SSH', 'rsyncd': 'RSYNC daemon',
                     'oid': 'OpenID 2.0', 'openid': 'OpenID 2.0',
                     'oidc': 'OpenID Connect', 'openidc': 'OpenID Connect'}
-# Prioritized protocol choices for duplicati - order matters!
-duplicati_protocol_choices = [(protocol_aliases[i], i) for i in
-                              ['sftp', 'ftps', 'davs']]
 # Prioritized schedule backup frequency choices and json values
 duplicati_schedule_choices = [('Daily', '1D'), ('Weekly', '1W'),
                               ('Monthly', '1M'), ('Never', '')]

mig/shared/duplicatikeywords.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # duplicatikeywords - keywords used in the duplicati settings file
-# Copyright (C) 2003-2021  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -29,8 +29,7 @@
 
 from __future__ import absolute_import
 
-from mig.shared.defaults import duplicati_protocol_choices, \
-    duplicati_schedule_choices
+from mig.shared.defaults import protocol_aliases, duplicati_schedule_choices
 from mig.shared.url import urlencode
 
 
@@ -54,17 +53,31 @@
     }'''
                             }
 
-protocol_map = dict(duplicati_protocol_choices)
 schedule_map = dict(duplicati_schedule_choices)
 
-# TODO: this function should probably move to shared.settings or something
+
+def get_duplicati_protocol_map(configuration, reverse=False):
+    """Simple helper to extract supported storage protocols from conf and map
+    them to user-friendly aliases. If the optional reverse argument is set the
+    map is reversed to map from aliases to protocols instead of vice-versa.
+    """
+    if reverse:
+        return {protocol_aliases[i]: i for i in configuration.storage_protocols}
+    else:
+        return {i: protocol_aliases[i] for i in configuration.storage_protocols}
 
 
+# TODO: this function should probably move to shared.settings or something
+
 def extract_duplicati_helper(configuration, client_id, duplicati_dict):
     """Fill helper dictionary with values used in duplicati_conf_templates"""
     # lookup fqdn, username, etc for specified protocol
-    default_protocol = duplicati_protocol_choices[0][0]
-    protocol_alias = duplicati_dict.get('PROTOCOL', default_protocol)
+    rev_proto_map = get_duplicati_protocol_map(configuration, reverse=True)
+    if configuration.storage_protocols:
+        default_alias = protocol_aliases[configuration.storage_protocols[0]]
+    else:
+        default_alias = ''
+    protocol_alias = duplicati_dict.get('PROTOCOL', default_alias)
     username = duplicati_dict.get('USERNAME')
     password = duplicati_dict.get('PASSWORD', '')
     credentials = [('auth-username', username)]
@@ -82,7 +95,7 @@ def extract_duplicati_helper(configuration, client_id, duplicati_dict):
     #       cert renew will change the hash and thus break existing confs.
     schedule_alias = duplicati_dict.get('SCHEDULE', '')
     schedule_freq = schedule_map.get(schedule_alias, '')
-    protocol = protocol_map[protocol_alias]
+    protocol = rev_proto_map[protocol_alias]
     if protocol in ('webdavs', 'davs'):
         # Duplicati client requires webdavs://BLA
         protocol = 'webdavs'

mig/shared/functionality/settings.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # settings - back end for the settings page
-# Copyright (C) 2003-2023  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -41,8 +41,9 @@
 from mig.shared.defaults import default_mrsl_filename, \
     default_css_filename, profile_img_max_kb, profile_img_extensions, \
     seafile_ro_dirname, duplicati_conf_dir, csrf_field, \
-    duplicati_protocol_choices, duplicati_schedule_choices
-from mig.shared.duplicatikeywords import get_duplicati_specs
+    duplicati_schedule_choices
+from mig.shared.duplicatikeywords import get_duplicati_specs, \
+    get_duplicati_protocol_map
 from mig.shared.editing import cm_css, cm_javascript, cm_options, wrap_edit_area
 from mig.shared.functional import validate_input_and_cert
 from mig.shared.handlers import get_csrf_limit, make_csrf_token
@@ -1675,9 +1676,8 @@ def main(client_id, user_arguments_dict):
         if configuration.user_duplicati_protocols:
             protocol_order = configuration.user_duplicati_protocols
         else:
-            protocol_order = [j for (i, j) in duplicati_protocol_choices]
-        reverse_proto_map = dict([(j, i) for (i, j) in
-                                  duplicati_protocol_choices])
+            protocol_order = configuration.storage_protocols
+        proto_map = get_duplicati_protocol_map(configuration)
 
         enabled_map = {
             'davs': configuration.site_enable_davs,
@@ -1691,7 +1691,7 @@ def main(client_id, user_arguments_dict):
             'ftps': extract_field(client_id, configuration.user_ftps_alias)
         }
         for proto in protocol_order:
-            pretty_proto = reverse_proto_map[proto]
+            pretty_proto = proto_map[proto]
             if not enabled_map[proto]:
                 continue
             if not pretty_proto in configuration.protocol:

mig/shared/functionality/setup.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # setup - back end for the client access setup page
-# Copyright (C) 2003-2023  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -38,9 +38,10 @@
 from mig.shared.base import client_alias, client_id_dir, extract_field, get_xgi_bin, \
     get_short_id, requested_url_base, requested_backend
 from mig.shared.defaults import seafile_ro_dirname, duplicati_conf_dir, csrf_field, \
-    duplicati_protocol_choices, duplicati_schedule_choices, keyword_all, \
-    AUTH_MIG_OID, AUTH_EXT_OID, AUTH_MIG_OIDC, AUTH_EXT_OIDC
-from mig.shared.duplicatikeywords import get_duplicati_specs
+    duplicati_schedule_choices, keyword_all, AUTH_MIG_OID, AUTH_EXT_OID, \
+    AUTH_MIG_OIDC, AUTH_EXT_OIDC
+from mig.shared.duplicatikeywords import get_duplicati_specs, \
+    get_duplicati_protocol_map
 from mig.shared.editing import cm_css, cm_javascript, cm_options, wrap_edit_area
 from mig.shared.functional import validate_input_and_cert
 from mig.shared.handlers import get_csrf_limit, make_csrf_token
@@ -1377,9 +1378,8 @@ def main(client_id, user_arguments_dict, target_op='settingsaction'):
         if configuration.user_duplicati_protocols:
             protocol_order = configuration.user_duplicati_protocols
         else:
-            protocol_order = [j for (i, j) in duplicati_protocol_choices]
-        reverse_proto_map = dict([(j, i) for (i, j) in
-                                  duplicati_protocol_choices])
+            protocol_order = configuration.storage_protocols
+        proto_map = get_duplicati_protocol_map(configuration)
 
         enabled_map = {
             'davs': configuration.site_enable_davs,
@@ -1393,7 +1393,7 @@ def main(client_id, user_arguments_dict, target_op='settingsaction'):
             'ftps': extract_field(client_id, configuration.user_ftps_alias)
         }
         for proto in protocol_order:
-            pretty_proto = reverse_proto_map[proto]
+            pretty_proto = proto_map[proto]
             if not enabled_map[proto]:
                 continue
             if not pretty_proto in configuration.protocol:

mig/shared/install.py

@@ -1054,6 +1054,8 @@ def _generate_confs_prepare(
     user_dict['__PERMANENT_FREEZE__'] = permanent_freeze
     user_dict['__FREEZE_TO_TAPE__'] = freeze_to_tape
     user_dict['__STATUS_SYSTEM_MATCH__'] = status_system_match
+    user_dict['__STORAGE_PROTOCOLS__'] = storage_protocols
+    user_dict['__DUPLICATI_PROTOCOLS__'] = duplicati_protocols
     user_dict['__IMNOTIFY_ADDRESS__'] = imnotify_address
     user_dict['__IMNOTIFY_CHANNEL__'] = imnotify_channel
     user_dict['__IMNOTIFY_USERNAME__'] = imnotify_username
@@ -1330,29 +1332,6 @@ def _generate_confs_prepare(
         if davs_show_port:
             fail2ban_daemon_ports.append(davs_show_port)
 
-    # NOTE: prioritized order based on performance and robustness
-    best_storage_svc = []
-    if enable_sftp_subsys or enable_sftp:
-        best_storage_svc.append('sftp')
-    if enable_ftps:
-        best_storage_svc.append('ftps')
-    if enable_davs:
-        best_storage_svc.append('davs')
-
-    if storage_protocols != keyword_auto:
-        storage_protocols = [i for i in storage_protocols.split() if i in
-                             best_storage_svc]
-    else:
-        storage_protocols = best_storage_svc
-    user_dict['__STORAGE_PROTOCOLS__'] = ' '.join(storage_protocols)
-
-    if duplicati_protocols != keyword_auto:
-        prio_duplicati_protocols = [i for i in duplicati_protocols.split() if i
-                                    in best_storage_svc]
-    else:
-        prio_duplicati_protocols = best_storage_svc
-    user_dict['__DUPLICATI_PROTOCOLS__'] = ' '.join(prio_duplicati_protocols)
-
     user_dict['__SEAFILE_TIMEZONE__'] = options['timezone']
 
     if seafile_secret == keyword_auto:

tests/data/MiGserver--storage_protocols.conf

@@ -187,9 +187,15 @@ mig_server_id = %(server_fqdn)s.0
 empty_job_name = no_grid_jobs_in_grid_scheduler
 notify_protocols = email
 smtp_server = localhost
-storage_protocols = xxx yyy zzz
 gdp_email_notify = False
 
+# Optional space-separated prioritized list of efficient storage access
+# protocols to advertize to clients. Leave to AUTO to use the ones actually
+# enabled with the corresponding enable_SERVICE options. Default is AUTO and
+# other allowed values are one or more of sftp, ftps and davs.
+# NOTE: the sftpsubsys service is advertized as just sftp to fit the protocol.
+storage_protocols = sftp
+
 # Optional extra service interfaces with common structure
 # * user_X_address is the host address to listen on
 # * user_X_port is the host port to listen on

tests/fixture/confs-stdlocal/MiGserver.conf

@@ -187,9 +187,15 @@ mig_server_id = %(server_fqdn)s.0
 empty_job_name = no_grid_jobs_in_grid_scheduler
 notify_protocols = email
 smtp_server = localhost
-storage_protocols = 
 gdp_email_notify = False
 
+# Optional space-separated prioritized list of efficient storage access
+# protocols to advertize to clients. Leave to AUTO to use the ones actually
+# enabled with the corresponding enable_SERVICE options. Default is AUTO and
+# other allowed values are one or more of sftp, ftps and davs.
+# NOTE: the sftpsubsys service is advertized as just sftp to fit the protocol.
+storage_protocols = AUTO
+
 # Optional extra service interfaces with common structure
 # * user_X_address is the host address to listen on
 # * user_X_port is the host port to listen on
@@ -290,7 +296,7 @@ user_seafile_local_instance = False
 # if local read-only mount is available for user home integration (default: False)
 user_seafile_ro_access = True
 # Priority list of protocols allowed in Duplicati backups (sftp, ftps, davs)
-user_duplicati_protocols = 
+user_duplicati_protocols = AUTO
 # Cloud settings for remote access - more in individual service sections
 # space separated list of cloud user authentication methods 
 # (default: publickey)

tests/fixture/mig_shared_configuration--new.json

@@ -207,7 +207,7 @@
   "smtp_server": "",
   "sss_home": "",
   "state_path": "/some/place/state",
-  "storage_protocols": [],
+  "storage_protocols": "AUTO",
   "submitui": [
     "fields",
     "textarea",
@@ -242,7 +242,7 @@
   "user_davs_show_address": "",
   "user_davs_show_port": 4443,
   "user_db_home": "",
-  "user_duplicati_protocols": [],
+  "user_duplicati_protocols": "AUTO",
   "user_events_log": "events.log",
   "user_ext_cert_title": "",
   "user_ext_oid_provider": "",