From 2126a5d0de7c214816e3cf309cad667ca57d8bcb Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Mon, 15 Jul 2024 10:18:19 +0200 Subject: [PATCH] Break the installation defaults out into their structure. Add tests that assert the consistency of the options that are accepted by the generateconfs command line with the internal library function. Additionally assert that the library routine itself matches the defaults structure thus making the _structure_ the definitive source of truth. Doing so highlighted the following missing command line options which are added as of this commit: --seafile_secret --seafile_ccnetid --- mig/install/__init__.py | 0 mig/install/generateconfs.py | 16 +- mig/shared/compat.py | 10 + mig/shared/install.py | 261 ++++++++++++++++++++++-- tests/test_mig_install_generateconfs.py | 9 + tests/test_mig_shared_install.py | 11 +- 6 files changed, 290 insertions(+), 17 deletions(-) create mode 100644 mig/install/__init__.py diff --git a/mig/install/__init__.py b/mig/install/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mig/install/generateconfs.py b/mig/install/generateconfs.py index 098d17188..eb3e097dd 100755 --- a/mig/install/generateconfs.py +++ b/mig/install/generateconfs.py @@ -48,7 +48,7 @@ # NOTE: moved mig imports into try/except to avoid autopep8 moving to top! try: from mig.shared.defaults import MIG_BASE, MIG_ENV - from mig.shared.install import generate_confs + from mig.shared.install import generate_confs, _GENERATE_CONFS_PARAMETERS except ImportError: print("ERROR: the migrid modules must be in PYTHONPATH") sys.exit(1) @@ -65,7 +65,7 @@ def usage(options): ''' % (sys.argv[0], '\n'.join(lines))) -def main(argv, _generate_confs=generate_confs, _print=print): +def _make_parameters(): str_names = [ 'source', 'destination', @@ -247,6 +247,8 @@ def main(argv, _generate_confs=generate_confs, _print=print): 'openid_port', 'openid_show_port', 'openid_session_lifetime', + 'seafile_secret', + 'seafile_ccnetid', 'seafile_seahub_port', 'seafile_seafhttp_port', 'seafile_client_port', @@ -303,7 +305,15 @@ def main(argv, _generate_confs=generate_confs, _print=print): 'io_account_expire', 'gdp_email_notify', ] - names = str_names + int_names + bool_names + return (str_names, int_names, bool_names) + + +str_names, int_names, bool_names = _make_parameters() +_PARAMETERS = str_names + int_names + bool_names + + +def main(argv, _generate_confs=generate_confs, _print=print): + names = _PARAMETERS settings, options, result = {}, {}, {} default_val = 'DEFAULT' # Force values to expected type diff --git a/mig/shared/compat.py b/mig/shared/compat.py index 01069ce43..0c0f46334 100644 --- a/mig/shared/compat.py +++ b/mig/shared/compat.py @@ -34,6 +34,7 @@ from past.builtins import basestring import codecs +import inspect import io import sys # NOTE: StringIO is only available in python2 @@ -93,6 +94,15 @@ def ensure_native_string(string_or_bytes): return textual_output +def inspect_args(func): + """Wrapper to return the arguments of a function.""" + + if PY2: + return inspect.getargspec(func).args + else: + return inspect.getfullargspec(func).args + + def NativeStringIO(initial_value=''): """Mock StringIO pseudo-class to create a StringIO matching the native string coding form. That is a BytesIO with utf8 on python 2 and unicode diff --git a/mig/shared/install.py b/mig/shared/install.py index 0d916e911..2f537ce19 100644 --- a/mig/shared/install.py +++ b/mig/shared/install.py @@ -50,15 +50,16 @@ import sys from mig.shared.base import force_native_str, force_utf8 +from mig.shared.compat import ensure_native_string, inspect_args, \ + SimpleNamespace from mig.shared.defaults import default_http_port, default_https_port, \ auth_openid_mig_db, auth_openid_ext_db, MIG_BASE, STRONG_TLS_CIPHERS, \ STRONG_TLS_CURVES, STRONG_SSH_HOSTKEYALGOS, STRONG_SSH_KEXALGOS, \ STRONG_SSH_CIPHERS, STRONG_SSH_MACS, LEGACY_SSH_HOSTKEYALGOS, \ LEGACY_SSH_KEXALGOS, LEGACY_SSH_CIPHERS, LEGACY_SSH_MACS, \ FALLBACK_SSH_HOSTKEYALGOS, FALLBACK_SSH_KEXALGOS, FALLBACK_SSH_CIPHERS, \ - FALLBACK_SSH_MACS, CRACK_USERNAME_REGEX, CRACK_WEB_REGEX, keyword_any, \ - keyword_auto -from mig.shared.compat import ensure_native_string + FALLBACK_SSH_MACS, CRACK_USERNAME_REGEX, CRACK_WEB_REGEX, \ + keyword_any, keyword_auto from mig.shared.fileio import read_file, read_file_lines, write_file, \ write_file_lines from mig.shared.htmlgen import menu_items @@ -283,16 +284,7 @@ def template_remove(template_file, remove_pattern): ] -def generate_confs( - generateconfs_output_path, - # NOTE: make sure command line args with white-space are properly wrapped - generateconfs_command=subprocess.list2cmdline(sys.argv), - source=keyword_auto, - destination=keyword_auto, - user=keyword_auto, - group=keyword_auto, - timezone=keyword_auto, - destination_suffix="", +_DEFAULTS = SimpleNamespace( base_fqdn='', public_fqdn='', public_alias_fqdn='', @@ -520,6 +512,246 @@ def generate_confs( datasafety_link='', datasafety_text='', wwwserve_max_bytes=-1, +) + + +def generate_confs( + generateconfs_output_path, + # NOTE: make sure command line args with white-space are properly wrapped + generateconfs_command=subprocess.list2cmdline(sys.argv), + source=keyword_auto, + destination=keyword_auto, + user=keyword_auto, + group=keyword_auto, + timezone=keyword_auto, + destination_suffix="", + base_fqdn=_DEFAULTS.base_fqdn, + public_fqdn=_DEFAULTS.public_fqdn, + public_alias_fqdn=_DEFAULTS.public_alias_fqdn, + public_sec_fqdn=_DEFAULTS.public_sec_fqdn, + status_alias_fqdn=_DEFAULTS.status_alias_fqdn, + mig_cert_fqdn=_DEFAULTS.mig_cert_fqdn, + ext_cert_fqdn=_DEFAULTS.ext_cert_fqdn, + mig_oid_fqdn=_DEFAULTS.mig_oid_fqdn, + ext_oid_fqdn=_DEFAULTS.ext_oid_fqdn, + mig_oidc_fqdn=_DEFAULTS.mig_oidc_fqdn, + ext_oidc_fqdn=_DEFAULTS.ext_oidc_fqdn, + sid_fqdn=_DEFAULTS.sid_fqdn, + io_fqdn=_DEFAULTS.io_fqdn, + cert_fqdn_extras=_DEFAULTS.cert_fqdn_extras, + cloud_fqdn=_DEFAULTS.cloud_fqdn, + seafile_fqdn=_DEFAULTS.seafile_fqdn, + seafile_base=_DEFAULTS.seafile_base, + seafmedia_base=_DEFAULTS.seafmedia_base, + seafhttp_base=_DEFAULTS.seafhttp_base, + openid_address=_DEFAULTS.openid_address, + sftp_address=_DEFAULTS.sftp_address, + sftp_subsys_address=_DEFAULTS.sftp_subsys_address, + ftps_address=_DEFAULTS.ftps_address, + davs_address=_DEFAULTS.davs_address, + jupyter_services=_DEFAULTS.jupyter_services, + jupyter_services_desc=_DEFAULTS.jupyter_services_desc, + cloud_services=_DEFAULTS.cloud_services, + cloud_services_desc=_DEFAULTS.cloud_services_desc, + apache_version=_DEFAULTS.apache_version, + apache_etc=_DEFAULTS.apache_etc, + apache_run=_DEFAULTS.apache_run, + apache_lock=_DEFAULTS.apache_lock, + apache_log=_DEFAULTS.apache_log, + apache_worker_procs=_DEFAULTS.apache_worker_procs, + openssh_version=_DEFAULTS.openssh_version, + mig_code=_DEFAULTS.mig_code, + mig_state=_DEFAULTS.mig_state, + mig_certs=_DEFAULTS.mig_certs, + auto_add_cert_user=_DEFAULTS.auto_add_cert_user, + auto_add_oid_user=_DEFAULTS.auto_add_oid_user, + auto_add_oidc_user=_DEFAULTS.auto_add_oidc_user, + auto_add_filter_fields=_DEFAULTS.auto_add_filter_fields, + auto_add_filter_method=_DEFAULTS.auto_add_filter_method, + auto_add_user_permit=_DEFAULTS.auto_add_user_permit, + auto_add_user_with_peer=_DEFAULTS.auto_add_user_with_peer, + cert_valid_days=_DEFAULTS.cert_valid_days, + oid_valid_days=_DEFAULTS.oid_valid_days, + oidc_valid_days=_DEFAULTS.oidc_valid_days, + generic_valid_days=_DEFAULTS.generic_valid_days, + enable_migadmin=_DEFAULTS.enable_migadmin, + enable_sftp=_DEFAULTS.enable_sftp, + enable_sftp_subsys=_DEFAULTS.enable_sftp_subsys, + sftp_subsys_auth_procs=_DEFAULTS.sftp_subsys_auth_procs, + enable_davs=_DEFAULTS.enable_davs, + enable_ftps=_DEFAULTS.enable_ftps, + enable_wsgi=_DEFAULTS.enable_wsgi, + wsgi_procs=_DEFAULTS.wsgi_procs, + enable_gdp=_DEFAULTS.enable_gdp, + enable_jobs=_DEFAULTS.enable_jobs, + enable_resources=_DEFAULTS.enable_resources, + enable_workflows=_DEFAULTS.enable_workflows, + enable_events=_DEFAULTS.enable_events, + enable_sharelinks=_DEFAULTS.enable_sharelinks, + enable_transfers=_DEFAULTS.enable_transfers, + enable_freeze=_DEFAULTS.enable_freeze, + enable_sandboxes=_DEFAULTS.enable_sandboxes, + enable_vmachines=_DEFAULTS.enable_vmachines, + enable_preview=_DEFAULTS.enable_preview, + enable_jupyter=_DEFAULTS.enable_jupyter, + enable_cloud=_DEFAULTS.enable_cloud, + enable_hsts=_DEFAULTS.enable_hsts, + enable_vhost_certs=_DEFAULTS.enable_vhost_certs, + enable_verify_certs=_DEFAULTS.enable_verify_certs, + enable_seafile=_DEFAULTS.enable_seafile, + enable_duplicati=_DEFAULTS.enable_duplicati, + enable_crontab=_DEFAULTS.enable_crontab, + enable_notify=_DEFAULTS.enable_notify, + enable_imnotify=_DEFAULTS.enable_imnotify, + enable_dev_accounts=_DEFAULTS.enable_dev_accounts, + enable_twofactor=_DEFAULTS.enable_twofactor, + twofactor_mandatory_protos=_DEFAULTS.twofactor_mandatory_protos, + enable_twofactor_strict_address=_DEFAULTS.enable_twofactor_strict_address, + twofactor_auth_apps=_DEFAULTS.twofactor_auth_apps, + enable_peers=_DEFAULTS.enable_peers, + peers_mandatory=_DEFAULTS.peers_mandatory, + peers_explicit_fields=_DEFAULTS.peers_explicit_fields, + peers_contact_hint=_DEFAULTS.peers_contact_hint, + enable_cracklib=_DEFAULTS.enable_cracklib, + enable_openid=_DEFAULTS.enable_openid, + enable_gravatars=_DEFAULTS.enable_gravatars, + enable_sitestatus=_DEFAULTS.enable_sitestatus, + enable_quota=_DEFAULTS.enable_quota, + prefer_python3=_DEFAULTS.prefer_python3, + io_account_expire=_DEFAULTS.io_account_expire, + gdp_email_notify=_DEFAULTS.gdp_email_notify, + user_interface=_DEFAULTS.user_interface, + mig_oid_title=_DEFAULTS.mig_oid_title, + mig_oid_provider=_DEFAULTS.mig_oid_provider, + ext_oid_title=_DEFAULTS.ext_oid_title, + ext_oid_provider=_DEFAULTS.ext_oid_provider, + mig_oidc_title=_DEFAULTS.mig_oidc_title, + mig_oidc_provider_meta_url=_DEFAULTS.mig_oidc_provider_meta_url, + ext_oidc_title=_DEFAULTS.ext_oidc_title, + ext_oidc_provider_meta_url=_DEFAULTS.ext_oidc_provider_meta_url, + ext_oidc_provider_issuer=_DEFAULTS.ext_oidc_provider_issuer, + ext_oidc_provider_authorization_endpoint=_DEFAULTS.ext_oidc_provider_authorization_endpoint, + ext_oidc_provider_verify_cert_files=_DEFAULTS.ext_oidc_provider_verify_cert_files, + ext_oidc_provider_token_endpoint=_DEFAULTS.ext_oidc_provider_token_endpoint, + ext_oidc_provider_token_endpoint_auth=_DEFAULTS.ext_oidc_provider_token_endpoint_auth, + ext_oidc_provider_user_info_endpoint=_DEFAULTS.ext_oidc_provider_user_info_endpoint, + ext_oidc_scope=_DEFAULTS.ext_oidc_scope, + ext_oidc_user_info_token_method=_DEFAULTS.ext_oidc_user_info_token_method, + ext_oidc_public_key_files=_DEFAULTS.ext_oidc_public_key_files, + ext_oidc_private_key_files=_DEFAULTS.ext_oidc_private_key_files, + ext_oidc_response_type=_DEFAULTS.ext_oidc_response_type, + ext_oidc_response_mode=_DEFAULTS.ext_oidc_response_mode, + ext_oidc_client_id=_DEFAULTS.ext_oidc_client_id, + ext_oidc_client_name=_DEFAULTS.ext_oidc_client_name, + ext_oidc_pkce_method=_DEFAULTS.ext_oidc_pkce_method, + ext_oidc_id_token_encrypted_response_alg=_DEFAULTS.ext_oidc_id_token_encrypted_response_alg, + ext_oidc_id_token_encrypted_response_enc=_DEFAULTS.ext_oidc_id_token_encrypted_response_enc, + ext_oidc_user_info_signed_response_alg=_DEFAULTS.ext_oidc_user_info_signed_response_alg, + ext_oidc_cookie_same_site=_DEFAULTS.ext_oidc_cookie_same_site, + ext_oidc_pass_cookies=_DEFAULTS.ext_oidc_pass_cookies, + ext_oidc_remote_user_claim=_DEFAULTS.ext_oidc_remote_user_claim, + ext_oidc_pass_claim_as=_DEFAULTS.ext_oidc_pass_claim_as, + ext_oidc_rewrite_cookie=_DEFAULTS.ext_oidc_rewrite_cookie, + dhparams_path=_DEFAULTS.dhparams_path, + daemon_keycert=_DEFAULTS.daemon_keycert, + daemon_keycert_sha256=_DEFAULTS.daemon_keycert_sha256, + daemon_pubkey=_DEFAULTS.daemon_pubkey, + daemon_pubkey_from_dns=_DEFAULTS.daemon_pubkey_from_dns, + daemon_pubkey_md5=_DEFAULTS.daemon_pubkey_md5, + daemon_pubkey_sha256=_DEFAULTS.daemon_pubkey_sha256, + daemon_show_address=_DEFAULTS.daemon_show_address, + alias_field=_DEFAULTS.alias_field, + peers_permit=_DEFAULTS.peers_permit, + vgrid_creators=_DEFAULTS.vgrid_creators, + vgrid_managers=_DEFAULTS.vgrid_managers, + signup_methods=_DEFAULTS.signup_methods, + login_methods=_DEFAULTS.login_methods, + digest_salt=_DEFAULTS.digest_salt, + crypto_salt=_DEFAULTS.crypto_salt, + csrf_protection=_DEFAULTS.csrf_protection, + password_policy=_DEFAULTS.password_policy, + password_legacy_policy=_DEFAULTS.password_legacy_policy, + hg_path=_DEFAULTS.hg_path, + hgweb_scripts=_DEFAULTS.hgweb_scripts, + trac_admin_path=_DEFAULTS.trac_admin_path, + trac_ini_path=_DEFAULTS.trac_ini_path, + public_port=_DEFAULTS.public_port, + public_http_port=_DEFAULTS.public_http_port, + public_https_port=_DEFAULTS.public_https_port, + mig_cert_port=_DEFAULTS.mig_cert_port, + ext_cert_port=_DEFAULTS.ext_cert_port, + mig_oid_port=_DEFAULTS.mig_oid_port, + ext_oid_port=_DEFAULTS.ext_oid_port, + mig_oidc_port=_DEFAULTS.mig_oidc_port, + ext_oidc_port=_DEFAULTS.ext_oidc_port, + sid_port=_DEFAULTS.sid_port, + sftp_port=_DEFAULTS.sftp_port, + sftp_subsys_port=_DEFAULTS.sftp_subsys_port, + sftp_show_port=_DEFAULTS.sftp_show_port, + sftp_max_sessions=_DEFAULTS.sftp_max_sessions, + davs_port=_DEFAULTS.davs_port, + davs_show_port=_DEFAULTS.davs_show_port, + ftps_ctrl_port=_DEFAULTS.ftps_ctrl_port, + ftps_ctrl_show_port=_DEFAULTS.ftps_ctrl_show_port, + ftps_pasv_ports=_DEFAULTS.ftps_pasv_ports, + openid_port=_DEFAULTS.openid_port, + openid_show_port=_DEFAULTS.openid_show_port, + openid_session_lifetime=_DEFAULTS.openid_session_lifetime, + seafile_secret=_DEFAULTS.seafile_secret, + seafile_ccnetid=_DEFAULTS.seafile_ccnetid, + seafile_seahub_port=_DEFAULTS.seafile_seahub_port, + seafile_seafhttp_port=_DEFAULTS.seafile_seafhttp_port, + seafile_client_port=_DEFAULTS.seafile_client_port, + seafile_quota=_DEFAULTS.seafile_quota, + seafile_ro_access=_DEFAULTS.seafile_ro_access, + public_use_https=_DEFAULTS.public_use_https, + user_clause=_DEFAULTS.user_clause, + group_clause=_DEFAULTS.group_clause, + listen_clause=_DEFAULTS.listen_clause, + serveralias_clause=_DEFAULTS.serveralias_clause, + distro=_DEFAULTS.distro, + autolaunch_page=_DEFAULTS.autolaunch_page, + landing_page=_DEFAULTS.landing_page, + skin=_DEFAULTS.skin, + title=_DEFAULTS.title, + short_title=_DEFAULTS.short_title, + extra_userpage_scripts=_DEFAULTS.extra_userpage_scripts, + extra_userpage_styles=_DEFAULTS.extra_userpage_styles, + external_doc=_DEFAULTS.external_doc, + vgrid_label=_DEFAULTS.vgrid_label, + secscan_addr=_DEFAULTS.secscan_addr, + default_menu=_DEFAULTS.default_menu, + user_menu=_DEFAULTS.user_menu, + collaboration_links=_DEFAULTS.collaboration_links, + default_vgrid_links=_DEFAULTS.default_vgrid_links, + advanced_vgrid_links=_DEFAULTS.advanced_vgrid_links, + support_email=_DEFAULTS.support_email, + admin_email=_DEFAULTS.admin_email, + admin_list=_DEFAULTS.admin_list, + smtp_server=_DEFAULTS.smtp_server, + smtp_sender=_DEFAULTS.smtp_sender, + permanent_freeze=_DEFAULTS.permanent_freeze, + log_level=_DEFAULTS.log_level, + freeze_to_tape=_DEFAULTS.freeze_to_tape, + status_system_match=_DEFAULTS.status_system_match, + storage_protocols=_DEFAULTS.storage_protocols, + duplicati_protocols=_DEFAULTS.duplicati_protocols, + imnotify_address=_DEFAULTS.imnotify_address, + imnotify_channel=_DEFAULTS.imnotify_channel, + imnotify_username=_DEFAULTS.imnotify_username, + imnotify_password=_DEFAULTS.imnotify_password, + gdp_data_categories=_DEFAULTS.gdp_data_categories, + gdp_id_scramble=_DEFAULTS.gdp_id_scramble, + gdp_path_scramble=_DEFAULTS.gdp_path_scramble, + quota_backend=_DEFAULTS.quota_backend, + quota_user_limit=_DEFAULTS.quota_user_limit, + quota_vgrid_limit=_DEFAULTS.quota_vgrid_limit, + ca_fqdn=_DEFAULTS.ca_fqdn, + ca_user=_DEFAULTS.ca_user, + ca_smtp=_DEFAULTS.ca_smtp, + datasafety_link=_DEFAULTS.datasafety_link, + datasafety_text=_DEFAULTS.datasafety_text, + wwwserve_max_bytes=_DEFAULTS.wwwserve_max_bytes, _getpwnam=pwd.getpwnam, _prepare=None, _writefiles=None, @@ -612,6 +844,9 @@ def generate_confs( return (options, user_dict) +_GENERATE_CONFS_PARAMETERS = set(inspect_args(generate_confs)) - set(_GENERATE_CONFS_NOFORWARD_KEYS) + + def _generate_confs_prepare( options, # forwarded arguments diff --git a/tests/test_mig_install_generateconfs.py b/tests/test_mig_install_generateconfs.py index e8ed90241..4590b5d58 100644 --- a/tests/test_mig_install_generateconfs.py +++ b/tests/test_mig_install_generateconfs.py @@ -35,6 +35,8 @@ from tests.support import MIG_BASE, MigTestCase, testmain, cleanpath +from mig.install.generateconfs import _PARAMETERS +from mig.shared.install import generate_confs, _GENERATE_CONFS_PARAMETERS def _import_generateconfs(): """Internal helper to work around non-package import location""" @@ -65,6 +67,13 @@ def noop(*args): pass +class MigInstallGenerateconfs(MigTestCase): + def test_consistent_parameters(self): + mismatched = _GENERATE_CONFS_PARAMETERS - set(_PARAMETERS) + self.assertEqual(len(mismatched), 0, + "defined parameters do not match generate_confs()") + + class MigInstallGenerateconfs__main(MigTestCase): """Unit test helper for the migrid code pointed to in class name""" diff --git a/tests/test_mig_shared_install.py b/tests/test_mig_shared_install.py index 5c7e567a1..acd10dd6d 100644 --- a/tests/test_mig_shared_install.py +++ b/tests/test_mig_shared_install.py @@ -39,8 +39,9 @@ from tests.support import MIG_BASE, TEST_OUTPUT_DIR, MigTestCase, \ testmain, temppath, cleanpath, fixturepath, is_path_within +from mig.install.generateconfs import _GENERATE_CONFS_PARAMETERS from mig.shared.defaults import keyword_auto -from mig.shared.install import determine_timezone, generate_confs +from mig.shared.install import determine_timezone, generate_confs, _DEFAULTS class DummyPwInfo: @@ -64,6 +65,14 @@ def noop(*args, **kwargs): return None +class MigSharedInstall(MigTestCase): + def test_consistent_parameters(self): + install_defaults_keys = set(_DEFAULTS.__dict__.keys()) + mismatched = _GENERATE_CONFS_PARAMETERS - install_defaults_keys + self.assertEqual(len(mismatched), 0, + "install defaults do not match generate_confs()") + + class MigSharedInstall__determine_timezone(MigTestCase): """Coverage of timezone determination."""