Merge remote-tracking branch 'origin/master' into edge

Author: jonasbardino

mig/install/MiGserver-template.conf

@@ -196,6 +196,18 @@ gdp_email_notify = __GDP_EMAIL_NOTIFY__
 # NOTE: the sftpsubsys service is advertized as just sftp to fit the protocol.
 storage_protocols = __STORAGE_PROTOCOLS__
 
+# Optional limit which instructs the web backends to deliver at most N bytes of
+# output to the users in order to avoid excessive memory use when serving files
+# which are subject to in-memory buffering. Defaults to a value of -1 which
+# disables the limit completely while 0 or any positive integer is intepreted
+# as a total number of bytes to allow serving in a single request.
+#
+# For now the limit is only enforced on cat.py, which is used implicitly for
+# certain downloads depending on site settings as well as for explicit client
+# requests e.g. xmlrpc requests or user scripts. The limit may or may not go
+# away in the future if better streaming of such requests can be made.
+wwwserve_max_bytes = __WWWSERVE_MAX_BYTES__
+
 # Optional extra service interfaces with common structure
 # * user_X_address is the host address to listen on
 # * user_X_port is the host port to listen on

mig/install/generateconfs.py

@@ -247,6 +247,7 @@ def main(argv, _generate_confs=generate_confs, _print=print):
         'seafile_quota',
         'quota_user_limit',
         'quota_vgrid_limit',
+        'wwwserve_max_bytes',
     ]
     bool_names = [
         'auto_add_cert_user',

mig/shared/configuration.py

@@ -181,7 +181,6 @@ def fix_missing(config_file, verbose=True):
         'mig_system_storage': '~/state/mig_system_storage',
         'mig_system_run': '~/state/mig_system_run/',
         'wwwpublic': '~/state/wwwpublic/',
-        'wwwserve_max_bytes': -1,
         'vm_home': '~/state/vm_home',
         'server_cert': '~/certs/cert.pem',
         'server_key': '~/certs/key.pem',
@@ -1626,6 +1625,10 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
         else:
             self.storage_protocols = prio_storage_protos
 
+        if config.has_option('GLOBAL', 'wwwserve_max_bytes'):
+            self.wwwserve_max_bytes = config.getint(
+                'GLOBAL', 'wwwserve_max_bytes')
+
         if config.has_option('SITE', 'enable_jupyter'):
             self.site_enable_jupyter = config.getboolean(
                 'SITE', 'enable_jupyter')

mig/shared/install.py

@@ -505,6 +505,7 @@ def generate_confs(
     ca_smtp='localhost',
     datasafety_link='',
     datasafety_text='',
+    wwwserve_max_bytes=-1,
     _getpwnam=pwd.getpwnam,
     _prepare=None,
     _writefiles=None,
@@ -821,6 +822,7 @@ def _generate_confs_prepare(
     ca_smtp,
     datasafety_link,
     datasafety_text,
+    wwwserve_max_bytes,
 ):
     """Prepate conf generator run"""
     user_dict = {}
@@ -1073,6 +1075,7 @@ def _generate_confs_prepare(
     user_dict['__CA_SMTP__'] = ca_smtp
     user_dict['__DATASAFETY_LINK__'] = datasafety_link
     user_dict['__DATASAFETY_TEXT__'] = datasafety_text
+    user_dict['__WWWSERVE_MAX_BYTES__'] = "%d" % (wwwserve_max_bytes)
 
     user_dict['__MIG_USER__'] = "%s" % (options['user_uname'])
     user_dict['__MIG_GROUP__'] = "%s" % (options['user_group'])

tests/data/MiGserver--storage_protocols.conf renamed to tests/data/MiGserver--customised.conf

@@ -196,6 +196,18 @@ gdp_email_notify = False
 # NOTE: the sftpsubsys service is advertized as just sftp to fit the protocol.
 storage_protocols = sftp
 
+# Optional limit which instructs the web backends to deliver at most N bytes of
+# output to the users in order to avoid excessive memory use when serving files
+# which are subject to in-memory buffering. Defaults to a value of -1 which
+# disables the limit completely while 0 or any positive integer is intepreted
+# as a total number of bytes to allow serving in a single request.
+#
+# For now the limit is only enforced on cat.py, which is used implicitly for
+# certain downloads depending on site settings as well as for explicit client
+# requests e.g. xmlrpc requests or user scripts. The limit may or may not go
+# away in the future if better streaming of such requests can be made.
+wwwserve_max_bytes = 43211234
+
 # Optional extra service interfaces with common structure
 # * user_X_address is the host address to listen on
 # * user_X_port is the host port to listen on

tests/fixture/confs-stdlocal/MiGserver.conf

@@ -196,6 +196,18 @@ gdp_email_notify = False
 # NOTE: the sftpsubsys service is advertized as just sftp to fit the protocol.
 storage_protocols = AUTO
 
+# Optional limit which instructs the web backends to deliver at most N bytes of
+# output to the users in order to avoid excessive memory use when serving files
+# which are subject to in-memory buffering. Defaults to a value of -1 which
+# disables the limit completely while 0 or any positive integer is intepreted
+# as a total number of bytes to allow serving in a single request.
+#
+# For now the limit is only enforced on cat.py, which is used implicitly for
+# certain downloads depending on site settings as well as for explicit client
+# requests e.g. xmlrpc requests or user scripts. The limit may or may not go
+# away in the future if better streaming of such requests can be made.
+wwwserve_max_bytes = -1
+
 # Optional extra service interfaces with common structure
 # * user_X_address is the host address to listen on
 # * user_X_port is the host port to listen on

tests/test_mig_install_generateconfs.py

@@ -99,6 +99,22 @@ def test_option_storage_protocols(self):
         self.assertIn('storage_protocols', settings)
         self.assertEqual(settings['storage_protocols'], 'proto1 proto2 proto3')
 
+    def test_option_wwwserve_max_bytes(self):
+        expected_generated_dir = cleanpath('confs-stdlocal', self,
+                                           ensure_dir=True)
+        with open(os.path.join(expected_generated_dir, "instructions.txt"),
+                  "w"):
+            pass
+        fake_generate_confs = create_fake_generate_confs(
+            dict(destination_dir=expected_generated_dir))
+        test_arguments = ['--wwwserve_max_bytes', '43211234']
+
+        exit_code = main(
+            test_arguments, _generate_confs=fake_generate_confs, _print=noop)
+        settings = fake_generate_confs.settings
+        self.assertIn('wwwserve_max_bytes', settings)
+        self.assertEqual(settings['wwwserve_max_bytes'], 43211234)
+
 
 if __name__ == '__main__':
     testmain()

tests/test_mig_shared_configuration.py

@@ -49,7 +49,7 @@ class MigSharedConfiguration(MigTestCase):
 
     def test_argument_storage_protocols(self):
         test_conf_file = os.path.join(
-            TEST_DATA_DIR, 'MiGserver--storage_protocols.conf')
+            TEST_DATA_DIR, 'MiGserver--customised.conf')
 
         configuration = Configuration(
             test_conf_file, skip_log=True, disable_auth_log=True)
@@ -60,6 +60,15 @@ def test_argument_storage_protocols(self):
         #self.assertEqual(configuration.storage_protocols, ['sftp'])
         self.assertEqual(configuration.storage_protocols, [])
 
+    def test_argument_wwwserve_max_bytes(self):
+        test_conf_file = os.path.join(
+            TEST_DATA_DIR, 'MiGserver--customised.conf')
+
+        configuration = Configuration(
+            test_conf_file, skip_log=True, disable_auth_log=True)
+
+        self.assertEqual(configuration.wwwserve_max_bytes, 43211234)
+
     @unittest.skipIf(PY2, "Python 3 only")
     def test_default_object(self):
         expected_values = fixturefile(