Manully merge PR104 to expose permanent_freeze conf option in conf generator and add unit tests for it. Reworks generateconfs script to wrap actual call in a main function for easier external testing. Adjusted with comments from review at github.

git-svn-id: svn+ssh://svn.code.sf.net/p/migrid/code/trunk@6115 b75ad72c-e7d7-11dd-a971-7dbc132099af

Author: jonasbardino

mig/install/MiGserver-template.conf

@@ -660,7 +660,7 @@ transfers_from = __PUBLIC_FQDN__ __MIG_CERT_FQDN__ __EXT_CERT_FQDN__ __MIG_OID_F
 enable_freeze = __ENABLE_FREEZE__
 # Which frozen archive flavors can be deleted (True for all, False or empty for
 # none and a space-separated list of flavors for individual control.
-#permanent_freeze = 
+permanent_freeze = __PERMANENT_FREEZE__
 # Delay before frozen archives are expected to hit tape (e.g. 5m, 4d or 2w).
 # Leave unset or empty if no tape archiving is available.
 freeze_to_tape = __FREEZE_TO_TAPE__

mig/install/generateconfs.py

@@ -64,7 +64,7 @@ def usage(options):
 ''' % (sys.argv[0], '\n'.join(lines)))
 
 
-if '__main__' == __name__:
+def main(argv, _generate_confs=generate_confs):
     str_names = [
         'source',
         'destination',
@@ -289,6 +289,7 @@ def usage(options):
         'enable_sitestatus',
         'daemon_pubkey_from_dns',
         'seafile_ro_access',
+        'permanent_freeze',
         'public_use_https',
         'prefer_python3',
         'io_account_expire',
@@ -318,23 +319,23 @@ def usage(options):
         else:
             print('Error: environment options %r not supported!' % opt_name)
             usage(names)
-            sys.exit(1)
+            return 1
 
     # apply values from CLI parameters
     flag_str = 'h'
     opts_str = ["%s=" % key for key in names] + ["help"]
     try:
-        (opts, args) = getopt.getopt(sys.argv[1:], flag_str, opts_str)
+        (opts, args) = getopt.getopt(argv, flag_str, opts_str)
     except getopt.GetoptError as exc:
         print('Error: ', exc.msg)
         usage(names)
-        sys.exit(1)
+        return 1
 
     for (opt, val) in opts:
         opt_name = opt.lstrip('-')
         if opt in ('-h', '--help'):
             usage(names)
-            sys.exit(0)
+            return 0
         elif opt_name in str_names:
             settings[opt_name] = val
         elif opt_name in int_names:
@@ -350,7 +351,7 @@ def usage(options):
         print('Error: non-option arguments are no longer supported!')
         print(" ... found: %s" % args)
         usage(names)
-        sys.exit(1)
+        return 1
     if settings['destination_suffix'] == 'DEFAULT':
         suffix = "-%s" % datetime.datetime.now().isoformat()
         settings['destination_suffix'] = suffix
@@ -371,7 +372,7 @@ def usage(options):
         if val == 'DEFAULT':
             del settings[key]
 
-    options = generate_confs(output_path, **settings)
+    options = _generate_confs(output_path, **settings)
 
     # TODO: avoid reconstructing this path (also done inside generate_confs)
     instructions_path = os.path.join(options['destination_dir'],
@@ -383,5 +384,10 @@ def usage(options):
         print(instructions)
     except Exception as exc:
         print("ERROR: could not read generated instructions: %s" % exc)
-        sys.exit(1)
-    sys.exit(0)
+        return 1
+    return 0
+
+
+if '__main__' == __name__:
+    exit_code = main(sys.argv[1:])
+    sys.exit(exit_code)

mig/server/MiGserver-localhost.conf

@@ -343,7 +343,7 @@ enable_transfers = True
 enable_freeze = True
 # Which frozen archive flavors can be deleted (True for all, False or empty for
 # none and a space-separated list of flavors for individual control.
-#permanent_freeze = 
+permanent_freeze = 
 # Delay before frozen archives are expected to hit tape (e.g. 5m, 4d or 2w).
 # Leave unset or empty if no tape archiving is available.
 #freeze_to_tape = 

mig/shared/install.py

@@ -35,6 +35,7 @@
 
 from __future__ import print_function
 from __future__ import absolute_import
+from past.builtins import basestring
 
 import ast
 import base64
@@ -482,6 +483,7 @@ def generate_confs(
     smtp_server='localhost',
     smtp_sender='',
     log_level='info',
+    permanent_freeze='no',
     freeze_to_tape='',
     status_system_match=keyword_any,
     duplicati_protocols='',
@@ -796,6 +798,7 @@ def _generate_confs_prepare(
     smtp_server,
     smtp_sender,
     log_level,
+    permanent_freeze,
     freeze_to_tape,
     status_system_match,
     duplicati_protocols,
@@ -1043,6 +1046,13 @@ def _generate_confs_prepare(
     user_dict['__SMTP_SERVER__'] = smtp_server
     user_dict['__SMTP_SENDER__'] = smtp_sender
     user_dict['__LOG_LEVEL__'] = log_level
+
+    if isinstance(permanent_freeze, basestring):
+        permanent_freeze = permanent_freeze.split(' ')
+    elif isinstance(permanent_freeze, bool):
+        permanent_freeze = ['yes' if permanent_freeze else 'no']
+    user_dict['__PERMANENT_FREEZE__'] = ' '.join(permanent_freeze)
+
     user_dict['__FREEZE_TO_TAPE__'] = freeze_to_tape
     user_dict['__STATUS_SYSTEM_MATCH__'] = status_system_match
     user_dict['__IMNOTIFY_ADDRESS__'] = imnotify_address

tests/fixture/confs-stdlocal/MiGserver.conf

@@ -660,7 +660,7 @@ transfers_from =
 enable_freeze = True
 # Which frozen archive flavors can be deleted (True for all, False or empty for
 # none and a space-separated list of flavors for individual control.
-#permanent_freeze = 
+permanent_freeze = no
 # Delay before frozen archives are expected to hit tape (e.g. 5m, 4d or 2w).
 # Leave unset or empty if no tape archiving is available.
 freeze_to_tape = 

tests/support.py

@@ -278,9 +278,14 @@ def is_path_within(path, start=None, _msg=None):
     return not relative.startswith('..')
 
 
-def cleanpath(relative_path, test_case):
+def cleanpath(relative_path, test_case, ensure_dir=False):
     assert isinstance(test_case, MigTestCase)
     tmp_path = os.path.join(TEST_OUTPUT_DIR, relative_path)
+    if ensure_dir:
+        try:
+            os.mkdir(tmp_path)
+        except FileExistsError:
+            raise AssertionError("ABORT: use of unclean output path: %s" % relative_path)
     test_case._cleanup_paths.add(tmp_path)
     return tmp_path
 

tests/test_mig_install_generateconfs.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# test_mig_install_generateconfs - unit test of the corresponding mig module
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+"""Unit tests for the migrid module pointed to in the filename"""
+
+from __future__ import print_function
+
+import importlib
+import os
+import sys
+
+from tests.support import MIG_BASE, MigTestCase, testmain, cleanpath
+
+
+def _import_generateconfs():
+    """Internal helper to work around non-package import location"""
+    sys.path.append(os.path.join(MIG_BASE, 'mig/install'))
+    mod = importlib.import_module('generateconfs')
+    sys.path.pop(-1)
+    return mod
+
+
+# workaround for generatconfs being placed witin a non-module directory
+generateconfs = _import_generateconfs()
+main = generateconfs.main
+
+
+def create_fake_generate_confs(return_dict=None):
+    """Fake generate confs helper"""
+    def _generate_confs(*args, **kwargs):
+        if return_dict:
+            return return_dict
+        else:
+            return {}
+    return _generate_confs
+
+
+class MigInstallGenerateconfs__main(MigTestCase):
+    """Unit test helper for the migrid code pointed to in class name"""
+
+    def test_option_permanent_freeze(self):
+        expected_generated_dir = cleanpath('confs-stdlocal', self,
+                                           ensure_dir=True)
+        with open(os.path.join(expected_generated_dir, "instructions.txt"),
+                  "w"):
+            pass
+        fake_generate_confs = create_fake_generate_confs(dict(
+            destination_dir=expected_generated_dir
+        ))
+        test_arguments = ['--permanent_freeze', 'yes']
+
+        exit_code = main(test_arguments, _generate_confs=fake_generate_confs)
+        self.assertEqual(exit_code, 0)
+
+
+if __name__ == '__main__':
+    testmain()

tests/test_mig_shared_install.py

@@ -36,7 +36,8 @@
 import pwd
 import sys
 
-sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__), "..")))
+sys.path.append(os.path.realpath(
+    os.path.join(os.path.dirname(__file__), "..")))
 
 from support import MIG_BASE, TEST_OUTPUT_DIR, MigTestCase, \
     testmain, temppath, cleanpath, fixturepath, is_path_within, outputpath
@@ -231,6 +232,26 @@ def test_creates_output_files_with_datasafety(self):
         self.assertConfigKey(
             actual_file, 'SITE', 'datasafety_text', expected='TEST_DATASAFETY_TEXT')
 
+    def test_creates_output_files_with_permanent_freeze(self):
+        fixture_dir = fixturepath("confs-stdlocal")
+        expected_generated_dir = cleanpath('confs-stdlocal', self)
+        symlink_path = temppath('confs', self)
+
+        generate_confs(
+            self.output_path,
+            destination=symlink_path,
+            destination_suffix='-stdlocal',
+            permanent_freeze=('foo', 'bar', 'baz'),
+            _getpwnam=create_dummy_gpwnam(4321, 1234),
+        )
+
+        relative_file = 'confs-stdlocal/MiGserver.conf'
+        self.assertPathExists('confs-stdlocal/MiGserver.conf')
+
+        actual_file = outputpath(relative_file)
+        self.assertConfigKey(
+            actual_file, 'SITE', 'permanent_freeze', expected='foo bar baz')
+
     def test_options_for_source_auto(self):
         options = generate_confs(
             '/some/arbitrary/path',