Merge remote-tracking branch 'origin/master' into edge

Author: jonasbardino

mig/install/MiGserver-template.conf

@@ -187,9 +187,15 @@ mig_server_id = %(server_fqdn)s.0
 empty_job_name = no_grid_jobs_in_grid_scheduler
 notify_protocols = email
 smtp_server = __SMTP_SERVER__
-storage_protocols = __STORAGE_PROTOCOLS__
 gdp_email_notify = __GDP_EMAIL_NOTIFY__
 
+# Optional space-separated prioritized list of efficient storage access
+# protocols to advertize to clients. Leave to AUTO to use the ones actually
+# enabled with the corresponding enable_SERVICE options. Default is AUTO and
+# other allowed values are one or more of sftp, ftps and davs.
+# NOTE: the sftpsubsys service is advertized as just sftp to fit the protocol.
+storage_protocols = __STORAGE_PROTOCOLS__
+
 # Optional extra service interfaces with common structure
 # * user_X_address is the host address to listen on
 # * user_X_port is the host port to listen on

mig/shared/configuration.py

@@ -50,10 +50,10 @@
     from mig.shared.defaults import CSRF_MINIMAL, CSRF_WARN, CSRF_MEDIUM, \
         CSRF_FULL, POLICY_NONE, POLICY_WEAK, POLICY_MEDIUM, POLICY_HIGH, \
         POLICY_MODERN, POLICY_CUSTOM, freeze_flavors, cert_field_order, \
-        duplicati_protocol_choices, default_css_filename, keyword_any, \
-        cert_valid_days, oid_valid_days, generic_valid_days, keyword_all, \
-        keyword_file, keyword_env, DEFAULT_USER_ID_FORMAT, \
-        valid_user_id_formats, valid_filter_methods, default_twofactor_auth_apps
+        default_css_filename, keyword_any, keyword_auto, keyword_all, \
+        keyword_file, keyword_env, cert_valid_days, oid_valid_days, \
+        generic_valid_days, DEFAULT_USER_ID_FORMAT, valid_user_id_formats, \
+        valid_filter_methods, default_twofactor_auth_apps
     from mig.shared.logger import Logger, SYSLOG_GDP
     from mig.shared.htmlgen import menu_items, vgrid_items
     from mig.shared.fileio import read_file, load_json, write_file
@@ -264,7 +264,6 @@ def fix_missing(config_file, verbose=True):
         'user_seafile_auth': ['password'],
         'user_seafile_local_instance': False,
         'user_seafile_ro_access': False,
-        'user_duplicati_protocols': [],
         'user_cloud_console_access': [],
         'user_cloud_ssh_auth': ['publickey'],
         'user_cloud_alias': '',
@@ -528,7 +527,7 @@ def fix_missing(config_file, verbose=True):
     'user_seafile_alias': '',
     'user_seafile_local_instance': False,
     'user_seafile_ro_access': False,
-    'user_duplicati_protocols': [],
+    'user_duplicati_protocols': keyword_auto,
     'user_cloud_console_access': [],
     'user_cloud_ssh_auth': ['publickey'],
     'user_cloud_alias': '',
@@ -619,7 +618,7 @@ def fix_missing(config_file, verbose=True):
     'scriptlanguages': [],
     'jobtypes': [],
     'lrmstypes': [],
-    'storage_protocols': [],
+    'storage_protocols': keyword_auto,
     'server_cert': '',
     'server_key': '',
     'passphrase_file': '',
@@ -1284,6 +1283,17 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
                                               'user_ftps_alias')
         if config.has_option('GLOBAL', 'user_ftps_log'):
             self.user_ftps_log = config.get('GLOBAL', 'user_ftps_log')
+
+        # NOTE: prioritized order based on performance and robustness.
+        #       Needed by duplicati and storage protocols setup below.
+        prio_storage_protos = []
+        if self.site_enable_sftp_subsys or self.site_enable_sftp:
+            prio_storage_protos.append('sftp')
+        if self.site_enable_ftps:
+            prio_storage_protos.append('ftps')
+        if self.site_enable_davs:
+            prio_storage_protos.append('davs')
+
         if config.has_option('SITE', 'enable_seafile'):
             self.site_enable_seafile = config.getboolean(
                 'SITE', 'enable_seafile')
@@ -1321,10 +1331,21 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
         else:
             self.site_enable_duplicati = False
         if config.has_option('GLOBAL', 'user_duplicati_protocols'):
-            allowed_protos = [j for (i, j) in duplicati_protocol_choices]
-            protos = config.get('GLOBAL', 'user_duplicati_protocols').split()
+            allowed_protos = prio_storage_protos
+            plain_val = config.get('GLOBAL', 'user_duplicati_protocols')
+            protos = [i for i in plain_val.split() if i]
+            # Append missing supported protocols for AUTO and filter invalid
+            if keyword_auto in protos:
+                protos = [i for i in protos if i != keyword_auto] +\
+                         [i for i in allowed_protos if i not in protos]
             valid_protos = [i for i in protos if i in allowed_protos]
+            if protos != valid_protos:
+                invalid_protos = [i for i in protos if i not in valid_protos]
+                self.logger.warning("invalid duplicati_protocol value(s): %s" %
+                                    ', '.join(invalid_protos))
             self.user_duplicati_protocols = valid_protos
+        else:
+            self.user_duplicati_protocols = prio_storage_protos
         if config.has_option('SITE', 'enable_cloud'):
             self.site_enable_cloud = config.getboolean(
                 'SITE', 'enable_cloud')
@@ -1589,8 +1610,22 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
         else:
             self.notify_protocols = []
         if config.has_option('GLOBAL', 'storage_protocols'):
-            self.storage_protocols = config.get(
-                'GLOBAL', 'storage_protocols').split()
+            allowed_protos = prio_storage_protos
+            plain_val = config.get('GLOBAL', 'storage_protocols')
+            protos = [i for i in plain_val.split() if i]
+            # Append missing supported protocols for AUTO and filter invalid
+            if keyword_auto in protos:
+                protos = [i for i in protos if i != keyword_auto] +\
+                         [i for i in allowed_protos if i not in protos]
+            valid_protos = [i for i in protos if i in allowed_protos]
+            if protos != valid_protos:
+                invalid_protos = [i for i in protos if i not in valid_protos]
+                self.logger.warning("invalid storage_protocols value(s): %s" %
+                                    ', '.join(invalid_protos))
+            self.storage_protocols = valid_protos
+        else:
+            self.storage_protocols = prio_storage_protos
+
         if config.has_option('SITE', 'enable_jupyter'):
             self.site_enable_jupyter = config.getboolean(
                 'SITE', 'enable_jupyter')

mig/shared/defaults.py

@@ -424,9 +424,6 @@
                     'rsyncssh': 'RSYNC over SSH', 'rsyncd': 'RSYNC daemon',
                     'oid': 'OpenID 2.0', 'openid': 'OpenID 2.0',
                     'oidc': 'OpenID Connect', 'openidc': 'OpenID Connect'}
-# Prioritized protocol choices for duplicati - order matters!
-duplicati_protocol_choices = [(protocol_aliases[i], i) for i in
-                              ['sftp', 'ftps', 'davs']]
 # Prioritized schedule backup frequency choices and json values
 duplicati_schedule_choices = [('Daily', '1D'), ('Weekly', '1W'),
                               ('Monthly', '1M'), ('Never', '')]

mig/shared/duplicatikeywords.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # duplicatikeywords - keywords used in the duplicati settings file
-# Copyright (C) 2003-2021  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -29,8 +29,7 @@
 
 from __future__ import absolute_import
 
-from mig.shared.defaults import duplicati_protocol_choices, \
-    duplicati_schedule_choices
+from mig.shared.defaults import protocol_aliases, duplicati_schedule_choices
 from mig.shared.url import urlencode
 
 
@@ -54,17 +53,31 @@
     }'''
                             }
 
-protocol_map = dict(duplicati_protocol_choices)
 schedule_map = dict(duplicati_schedule_choices)
 
-# TODO: this function should probably move to shared.settings or something
+
+def get_duplicati_protocol_map(configuration, reverse=False):
+    """Simple helper to extract supported storage protocols from conf and map
+    them to user-friendly aliases. If the optional reverse argument is set the
+    map is reversed to map from aliases to protocols instead of vice-versa.
+    """
+    if reverse:
+        return {protocol_aliases[i]: i for i in configuration.storage_protocols}
+    else:
+        return {i: protocol_aliases[i] for i in configuration.storage_protocols}
 
 
+# TODO: this function should probably move to shared.settings or something
+
 def extract_duplicati_helper(configuration, client_id, duplicati_dict):
     """Fill helper dictionary with values used in duplicati_conf_templates"""
     # lookup fqdn, username, etc for specified protocol
-    default_protocol = duplicati_protocol_choices[0][0]
-    protocol_alias = duplicati_dict.get('PROTOCOL', default_protocol)
+    rev_proto_map = get_duplicati_protocol_map(configuration, reverse=True)
+    if configuration.storage_protocols:
+        default_alias = protocol_aliases[configuration.storage_protocols[0]]
+    else:
+        default_alias = ''
+    protocol_alias = duplicati_dict.get('PROTOCOL', default_alias)
     username = duplicati_dict.get('USERNAME')
     password = duplicati_dict.get('PASSWORD', '')
     credentials = [('auth-username', username)]
@@ -82,7 +95,7 @@ def extract_duplicati_helper(configuration, client_id, duplicati_dict):
     #       cert renew will change the hash and thus break existing confs.
     schedule_alias = duplicati_dict.get('SCHEDULE', '')
     schedule_freq = schedule_map.get(schedule_alias, '')
-    protocol = protocol_map[protocol_alias]
+    protocol = rev_proto_map[protocol_alias]
     if protocol in ('webdavs', 'davs'):
         # Duplicati client requires webdavs://BLA
         protocol = 'webdavs'

mig/shared/functionality/settings.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # settings - back end for the settings page
-# Copyright (C) 2003-2023  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -41,8 +41,9 @@
 from mig.shared.defaults import default_mrsl_filename, \
     default_css_filename, profile_img_max_kb, profile_img_extensions, \
     seafile_ro_dirname, duplicati_conf_dir, csrf_field, \
-    duplicati_protocol_choices, duplicati_schedule_choices
-from mig.shared.duplicatikeywords import get_duplicati_specs
+    duplicati_schedule_choices
+from mig.shared.duplicatikeywords import get_duplicati_specs, \
+    get_duplicati_protocol_map
 from mig.shared.editing import cm_css, cm_javascript, cm_options, wrap_edit_area
 from mig.shared.functional import validate_input_and_cert
 from mig.shared.handlers import get_csrf_limit, make_csrf_token
@@ -1675,9 +1676,8 @@ def main(client_id, user_arguments_dict):
         if configuration.user_duplicati_protocols:
             protocol_order = configuration.user_duplicati_protocols
         else:
-            protocol_order = [j for (i, j) in duplicati_protocol_choices]
-        reverse_proto_map = dict([(j, i) for (i, j) in
-                                  duplicati_protocol_choices])
+            protocol_order = configuration.storage_protocols
+        proto_map = get_duplicati_protocol_map(configuration)
 
         enabled_map = {
             'davs': configuration.site_enable_davs,
@@ -1691,7 +1691,7 @@ def main(client_id, user_arguments_dict):
             'ftps': extract_field(client_id, configuration.user_ftps_alias)
         }
         for proto in protocol_order:
-            pretty_proto = reverse_proto_map[proto]
+            pretty_proto = proto_map[proto]
             if not enabled_map[proto]:
                 continue
             if not pretty_proto in configuration.protocol:

mig/shared/functionality/setup.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # setup - back end for the client access setup page
-# Copyright (C) 2003-2023  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -38,9 +38,10 @@
 from mig.shared.base import client_alias, client_id_dir, extract_field, get_xgi_bin, \
     get_short_id, requested_url_base, requested_backend
 from mig.shared.defaults import seafile_ro_dirname, duplicati_conf_dir, csrf_field, \
-    duplicati_protocol_choices, duplicati_schedule_choices, keyword_all, \
-    AUTH_MIG_OID, AUTH_EXT_OID, AUTH_MIG_OIDC, AUTH_EXT_OIDC
-from mig.shared.duplicatikeywords import get_duplicati_specs
+    duplicati_schedule_choices, keyword_all, AUTH_MIG_OID, AUTH_EXT_OID, \
+    AUTH_MIG_OIDC, AUTH_EXT_OIDC
+from mig.shared.duplicatikeywords import get_duplicati_specs, \
+    get_duplicati_protocol_map
 from mig.shared.editing import cm_css, cm_javascript, cm_options, wrap_edit_area
 from mig.shared.functional import validate_input_and_cert
 from mig.shared.handlers import get_csrf_limit, make_csrf_token
@@ -1377,9 +1378,8 @@ def main(client_id, user_arguments_dict, target_op='settingsaction'):
         if configuration.user_duplicati_protocols:
             protocol_order = configuration.user_duplicati_protocols
         else:
-            protocol_order = [j for (i, j) in duplicati_protocol_choices]
-        reverse_proto_map = dict([(j, i) for (i, j) in
-                                  duplicati_protocol_choices])
+            protocol_order = configuration.storage_protocols
+        proto_map = get_duplicati_protocol_map(configuration)
 
         enabled_map = {
             'davs': configuration.site_enable_davs,
@@ -1393,7 +1393,7 @@ def main(client_id, user_arguments_dict, target_op='settingsaction'):
             'ftps': extract_field(client_id, configuration.user_ftps_alias)
         }
         for proto in protocol_order:
-            pretty_proto = reverse_proto_map[proto]
+            pretty_proto = proto_map[proto]
             if not enabled_map[proto]:
                 continue
             if not pretty_proto in configuration.protocol:

mig/shared/install.py

@@ -1054,6 +1054,8 @@ def _generate_confs_prepare(
     user_dict['__PERMANENT_FREEZE__'] = permanent_freeze
     user_dict['__FREEZE_TO_TAPE__'] = freeze_to_tape
     user_dict['__STATUS_SYSTEM_MATCH__'] = status_system_match
+    user_dict['__STORAGE_PROTOCOLS__'] = storage_protocols
+    user_dict['__DUPLICATI_PROTOCOLS__'] = duplicati_protocols
     user_dict['__IMNOTIFY_ADDRESS__'] = imnotify_address
     user_dict['__IMNOTIFY_CHANNEL__'] = imnotify_channel
     user_dict['__IMNOTIFY_USERNAME__'] = imnotify_username
@@ -1330,29 +1332,6 @@ def _generate_confs_prepare(
         if davs_show_port:
             fail2ban_daemon_ports.append(davs_show_port)
 
-    # NOTE: prioritized order based on performance and robustness
-    best_storage_svc = []
-    if enable_sftp_subsys or enable_sftp:
-        best_storage_svc.append('sftp')
-    if enable_ftps:
-        best_storage_svc.append('ftps')
-    if enable_davs:
-        best_storage_svc.append('davs')
-
-    if storage_protocols != keyword_auto:
-        storage_protocols = [i for i in storage_protocols.split() if i in
-                             best_storage_svc]
-    else:
-        storage_protocols = best_storage_svc
-    user_dict['__STORAGE_PROTOCOLS__'] = ' '.join(storage_protocols)
-
-    if duplicati_protocols != keyword_auto:
-        prio_duplicati_protocols = [i for i in duplicati_protocols.split() if i
-                                    in best_storage_svc]
-    else:
-        prio_duplicati_protocols = best_storage_svc
-    user_dict['__DUPLICATI_PROTOCOLS__'] = ' '.join(prio_duplicati_protocols)
-
     user_dict['__SEAFILE_TIMEZONE__'] = options['timezone']
 
     if seafile_secret == keyword_auto:

tests/data/MiGserver--storage_protocols.conf

@@ -187,9 +187,15 @@ mig_server_id = %(server_fqdn)s.0
 empty_job_name = no_grid_jobs_in_grid_scheduler
 notify_protocols = email
 smtp_server = localhost
-storage_protocols = xxx yyy zzz
 gdp_email_notify = False
 
+# Optional space-separated prioritized list of efficient storage access
+# protocols to advertize to clients. Leave to AUTO to use the ones actually
+# enabled with the corresponding enable_SERVICE options. Default is AUTO and
+# other allowed values are one or more of sftp, ftps and davs.
+# NOTE: the sftpsubsys service is advertized as just sftp to fit the protocol.
+storage_protocols = sftp
+
 # Optional extra service interfaces with common structure
 # * user_X_address is the host address to listen on
 # * user_X_port is the host port to listen on

tests/fixture/confs-stdlocal/MiGserver.conf

@@ -187,9 +187,15 @@ mig_server_id = %(server_fqdn)s.0
 empty_job_name = no_grid_jobs_in_grid_scheduler
 notify_protocols = email
 smtp_server = localhost
-storage_protocols = 
 gdp_email_notify = False
 
+# Optional space-separated prioritized list of efficient storage access
+# protocols to advertize to clients. Leave to AUTO to use the ones actually
+# enabled with the corresponding enable_SERVICE options. Default is AUTO and
+# other allowed values are one or more of sftp, ftps and davs.
+# NOTE: the sftpsubsys service is advertized as just sftp to fit the protocol.
+storage_protocols = AUTO
+
 # Optional extra service interfaces with common structure
 # * user_X_address is the host address to listen on
 # * user_X_port is the host port to listen on
@@ -290,7 +296,7 @@ user_seafile_local_instance = False
 # if local read-only mount is available for user home integration (default: False)
 user_seafile_ro_access = True
 # Priority list of protocols allowed in Duplicati backups (sftp, ftps, davs)
-user_duplicati_protocols = 
+user_duplicati_protocols = AUTO
 # Cloud settings for remote access - more in individual service sections
 # space separated list of cloud user authentication methods 
 # (default: publickey)

tests/fixture/mig_shared_configuration--new.json

@@ -207,7 +207,7 @@
   "smtp_server": "",
   "sss_home": "",
   "state_path": "/some/place/state",
-  "storage_protocols": [],
+  "storage_protocols": "AUTO",
   "submitui": [
     "fields",
     "textarea",
@@ -242,7 +242,7 @@
   "user_davs_show_address": "",
   "user_davs_show_port": 4443,
   "user_db_home": "",
-  "user_duplicati_protocols": [],
+  "user_duplicati_protocols": "AUTO",
   "user_events_log": "events.log",
   "user_ext_cert_title": "",
   "user_ext_oid_provider": "",