Merge remote-tracking branch 'origin/master' into edge

Author: jonasbardino

mig/install/generateconfs.py

@@ -195,6 +195,7 @@ def main(argv, _generate_confs=generate_confs, _print=print):
         'permanent_freeze',
         'freeze_to_tape',
         'status_system_match',
+        'storage_protocols',
         'duplicati_protocols',
         'imnotify_address',
         'imnotify_channel',

mig/shared/defaults.py

@@ -415,9 +415,18 @@
 PASSWORD_POLICIES = [POLICY_NONE, POLICY_WEAK, POLICY_MEDIUM, POLICY_HIGH,
                      POLICY_MODERN, POLICY_CUSTOM]
 
-# Prioritized protocol choices and internal values
-duplicati_protocol_choices = [('SFTP', 'sftp'), ('FTPS', 'ftps'),
-                              ('WebDAVS', 'davs')]
+# Protocol aliases for pretty printing, etc.
+protocol_aliases = {'http': 'HTTP', 'https': 'HTTPS',
+                    'ftp': 'FTP', 'ftps': 'FTPS',
+                    'sftp': 'SFTP', 'sftp-subsys': 'SFTP',
+                    'dav': 'WebDAV', 'webdav': 'WebDAV',
+                    'davs': 'WebDAVS', 'webdavs': 'WebDAVS',
+                    'rsyncssh': 'RSYNC over SSH', 'rsyncd': 'RSYNC daemon',
+                    'oid': 'OpenID 2.0', 'openid': 'OpenID 2.0',
+                    'oidc': 'OpenID Connect', 'openidc': 'OpenID Connect'}
+# Prioritized protocol choices for duplicati - order matters!
+duplicati_protocol_choices = [(protocol_aliases[i], i) for i in
+                              ['sftp', 'ftps', 'davs']]
 # Prioritized schedule backup frequency choices and json values
 duplicati_schedule_choices = [('Daily', '1D'), ('Weekly', '1W'),
                               ('Monthly', '1M'), ('Never', '')]

mig/shared/functionality/cat.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # cat - show lines of one or more files
-# Copyright (C) 2003-2023  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2024  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -36,6 +36,7 @@
 
 from mig.shared import returnvalues
 from mig.shared.base import client_id_dir
+from mig.shared.defaults import protocol_aliases
 from mig.shared.fileio import read_file, read_file_lines, write_file, \
     write_file_lines
 from mig.shared.functional import validate_input_and_cert, REJECT_UNSET
@@ -71,10 +72,14 @@ def _check_serve_permitted(configuration, paths):
 def _render_error_text_for_serve_limit(configuration):
     """Helper to inform about suitable alternatives when above limit"""
     limit = configuration.wwwserve_max_bytes
-    alternatives = ', '.join(configuration.storage_protocols)
-    return ("Site configuration prevents web serving contents bigger than "
-            "%d bytes - please use better alternatives (%s) to retrieve large "
-            "data.") % (limit, alternatives)
+    msg = "Site configuration prevents web serving contents bigger than " \
+        "%d bytes" % limit
+    if configuration.storage_protocols:
+        aliased = [protocol_aliases[i] for i in configuration.storage_protocols
+                   if i in protocol_aliases]
+        msg += " - please use better alternatives (%s) to retrieve large data" \
+            % ', '.join(aliased)
+    return msg
 
 
 def signature():

mig/shared/functionality/datatransfer.py

@@ -39,7 +39,7 @@
 from mig.shared.base import client_id_dir, mask_creds, hexlify
 from mig.shared.conf import get_resource_exe
 from mig.shared.defaults import all_jobs, job_output_dir, default_pager_entries, \
-    csrf_field
+    csrf_field, protocol_aliases
 from mig.shared.fileio import read_tail_lines
 from mig.shared.functional import validate_input_and_cert
 from mig.shared.handlers import safe_handler, get_csrf_limit, make_csrf_token
@@ -67,10 +67,10 @@
 post_actions = transfer_actions + shuffling_actions + key_actions
 valid_actions = get_actions + post_actions
 # TODO: implement scp in backend and enable here?
-valid_proto = [("http", "HTTP"), ("https", "HTTPS"), ("ftp", "FTP"),
-               ("ftps", "FTPS"), ("webdav", "WebDAV"), ("webdavs", "WebDAVS"),
-               ("sftp", "SFTP"), ("rsyncssh", "RSYNC over SSH"),
-               ("rsyncd", "RSYNC daemon")]
+valid_proto_list = ["http", "https", "ftp", "ftps", "webdav", "webdavs", "sftp",
+                    "rsyncssh", "rsyncd"]
+valid_proto = [(proto, protocol_aliases[proto]) for proto in valid_proto_list
+               if proto in protocol_aliases]
 valid_proto_map = dict(valid_proto)
 warn_anon = [i for (i, _) in valid_proto if not i in ('http', 'https', 'ftp',
                                                       'rsyncd')]
@@ -489,7 +489,7 @@ def main(client_id, user_arguments_dict):
 
         # Make page with manage transfers tab and manage keys tab
 
-        output_objects.append({'object_type': 'html_form', 'text':  '''
+        output_objects.append({'object_type': 'html_form', 'text': '''
     <div id="quiet-mode-content" class="hidden">
         <p>
         Accept data %(transfer_action)s of %(transfer_src_string)s from
@@ -509,7 +509,7 @@ def main(client_id, user_arguments_dict):
 
         # Display external transfers, log and form to add new ones
 
-        output_objects.append({'object_type': 'html_form', 'text':  '''
+        output_objects.append({'object_type': 'html_form', 'text': '''
 <div id="transfer-tab">
 '''})
 
@@ -671,13 +671,13 @@ def main(client_id, user_arguments_dict):
 '''
         output_objects.append(
             {'object_type': 'html_form', 'text': transfer_html % fill_helpers})
-        output_objects.append({'object_type': 'html_form', 'text':  '''
+        output_objects.append({'object_type': 'html_form', 'text': '''
 </div>
 '''})
 
         # Display key management
 
-        output_objects.append({'object_type': 'html_form', 'text':  '''
+        output_objects.append({'object_type': 'html_form', 'text': '''
 <div id="keys-tab">
 '''})
         output_objects.append(
@@ -737,11 +737,11 @@ def main(client_id, user_arguments_dict):
 ''' % (restrict_template % 'ssh-rsa AAAAB3NzaC...', configuration.short_title)
         output_objects.append(
             {'object_type': 'html_form', 'text': key_html % fill_helpers})
-        output_objects.append({'object_type': 'html_form', 'text':  '''
+        output_objects.append({'object_type': 'html_form', 'text': '''
 </div>
 '''})
 
-        output_objects.append({'object_type': 'html_form', 'text':  '''
+        output_objects.append({'object_type': 'html_form', 'text': '''
 </div>
 '''})
         return (output_objects, returnvalues.OK)

mig/shared/griddaemons/auth.py

@@ -32,7 +32,7 @@
 
 from mig.shared.auth import active_twofactor_session
 from mig.shared.base import extract_field, expand_openid_alias
-from mig.shared.defaults import CRACK_USERNAME_REGEX
+from mig.shared.defaults import CRACK_USERNAME_REGEX, protocol_aliases
 from mig.shared.gdp.all import get_client_id_from_project_client_id
 from mig.shared.griddaemons.ratelimits import default_user_abuse_hits, \
     default_proto_abuse_hits, default_max_secret_hits, update_rate_limit
@@ -284,11 +284,8 @@ def validate_auth_attempt(configuration,
 
     # Log auth attempt and set (authorized, disconnect) return values
 
-    proto_names = {'sftp': 'SFTP', 'sftp-subsys': 'SFTP', 'ftps': 'FTPS',
-                   'webdavs': 'WebDAVS', 'davs': 'WebDAVS',
-                   'openid': 'OpenID 2.0', 'openidc': 'OpenID Connect'}
     mountable_protos = ['sftp', 'sftp-subsys', 'ftps', 'webdavs', 'davs']
-    proto_alias = proto_names.get(protocol, protocol.upper())
+    proto_alias = protocol_aliases.get(protocol, protocol.upper())
     if exceeded_rate_limit:
         disconnect = True
         auth_msg = "Exceeded rate limit"

mig/shared/install.py

@@ -488,7 +488,8 @@ def generate_confs(
     permanent_freeze='no',
     freeze_to_tape='',
     status_system_match=keyword_any,
-    duplicati_protocols='',
+    storage_protocols=keyword_auto,
+    duplicati_protocols=keyword_auto,
     imnotify_address='',
     imnotify_channel='',
     imnotify_username='',
@@ -803,6 +804,7 @@ def _generate_confs_prepare(
     permanent_freeze,
     freeze_to_tape,
     status_system_match,
+    storage_protocols,
     duplicati_protocols,
     imnotify_address,
     imnotify_channel,
@@ -1328,17 +1330,27 @@ def _generate_confs_prepare(
         if davs_show_port:
             fail2ban_daemon_ports.append(davs_show_port)
 
-    if duplicati_protocols:
-        prio_duplicati_protocols = duplicati_protocols.split()
+    # NOTE: prioritized order based on performance and robustness
+    best_storage_svc = []
+    if enable_sftp_subsys or enable_sftp:
+        best_storage_svc.append('sftp')
+    if enable_ftps:
+        best_storage_svc.append('ftps')
+    if enable_davs:
+        best_storage_svc.append('davs')
+
+    if storage_protocols != keyword_auto:
+        storage_protocols = [i for i in storage_protocols.split() if i in
+                             best_storage_svc]
     else:
-        # NOTE: prioritized order based on performance and robustness
-        prio_duplicati_protocols = []
-        if enable_sftp_subsys or enable_sftp:
-            prio_duplicati_protocols.append('sftp')
-        if enable_ftps:
-            prio_duplicati_protocols.append('ftps')
-        if enable_davs:
-            prio_duplicati_protocols.append('davs')
+        storage_protocols = best_storage_svc
+    user_dict['__STORAGE_PROTOCOLS__'] = ' '.join(storage_protocols)
+
+    if duplicati_protocols != keyword_auto:
+        prio_duplicati_protocols = [i for i in duplicati_protocols.split() if i
+                                    in best_storage_svc]
+    else:
+        prio_duplicati_protocols = best_storage_svc
     user_dict['__DUPLICATI_PROTOCOLS__'] = ' '.join(prio_duplicati_protocols)
 
     user_dict['__SEAFILE_TIMEZONE__'] = options['timezone']

tests/test_mig_shared_functionality_cat.py

@@ -151,7 +151,7 @@ def test_file_serving_over_limit(self):
         self.assertEqual(relevant_obj['text'],
                          "Site configuration prevents web serving contents "
                          "bigger than 3896 bytes - please use better "
-                         "alternatives (sftp) to retrieve large data.")
+                         "alternatives (SFTP) to retrieve large data")
 
 
 if __name__ == '__main__':