diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f86bfba88..95c6e4b56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: make dependencies - name: Run tests run: | - make test + make unittest python3-rocky9ish: runs-on: ubuntu-22.04 @@ -51,7 +51,7 @@ jobs: make dependencies - name: Run tests run: | - make test + make unittest python3-rocky8ish: runs-on: ubuntu-20.04 @@ -67,7 +67,7 @@ jobs: make dependencies - name: Run tests run: | - make test + make unittest python2-latest: runs-on: ubuntu-latest @@ -80,8 +80,7 @@ jobs: uses: actions/checkout@v4 - name: Setup environment run: | - pip install --no-cache-dir -r requirements.txt -r local-requirements.txt + make PYTHON_BIN=python PY=2 dependencies - name: Run tests run: | - PYTHON_BIN=python ./envhelp/makeconfig test --python2 - MIG_ENV='local' python -m unittest discover -s tests/ + make PYTHON_BIN=python PY=2 unittest diff --git a/Makefile b/Makefile index 7e39f2149..1863a9ffc 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,21 @@ ifndef MIG_ENV MIG_ENV = 'local' endif -ifeq ($(PY),2) + +ifndef PY + PY = 3 +endif + +LOCAL_PYTHON_BIN = './envhelp/lpython' + +ifdef PYTHON_BIN + LOCAL_PYTHON_BIN = $(PYTHON_BIN) +else ifeq ($(PY),2) PYTHON_BIN = './envhelp/python2' else PYTHON_BIN = './envhelp/python3' endif + ifeq ($(ALLDEPS),1) REQS_PATH = ./recommended.txt else @@ -17,8 +27,9 @@ info: @echo @echo "The following should help you get started:" @echo - @echo "'make test' - run the test suite" - @echo "'make PY=2 test' - run the test suite (python 2)" + @echo "'make test' - run the test suite (default python 3)" + @echo "'make PY=2 test' - run the test suite (default python 2)" + @echo "'make unittest' - execute tests locally for development" .PHONY: fmt fmt: @@ -26,12 +37,13 @@ ifneq ($(MIG_ENV),'local') @echo "unavailable outside local development environment" @exit 1 endif - $(PYTHON_BIN) -m autopep8 --ignore E402 -i + $(LOCAL_PYTHON_BIN) -m autopep8 --ignore E402 -i .PHONY: clean clean: @rm -f ./envhelp/py2.imageid - @rm -f ./envhelp/py3.depends + @rm -f ./envhelp/py3.imageid + @rm -f ./envhelp/local.depends .PHONY: distclean distclean: clean @@ -44,37 +56,41 @@ distclean: clean test: dependencies testconfig @$(PYTHON_BIN) -m unittest discover -s tests/ +.PHONY: unittest +unittest: dependencies testconfig + @$(LOCAL_PYTHON_BIN) -m unittest discover -s tests/ + .PHONY: dependencies -dependencies: ./envhelp/venv/pyvenv.cfg ./envhelp/py3.depends +ifeq ($(PY),2) +dependencies: ./envhelp/local.depends +else +dependencies: ./envhelp/venv/pyvenv.cfg ./envhelp/local.depends +endif .PHONY: testconfig testconfig: ./envhelp/output/testconfs ./envhelp/output/testconfs: - @./envhelp/makeconfig test --python2 + @./envhelp/makeconfig test --docker @./envhelp/makeconfig test - @mkdir -p ./envhelp/output/certs - @mkdir -p ./envhelp/output/state - @mkdir -p ./envhelp/output/state/log ifeq ($(MIG_ENV),'local') -./envhelp/py3.depends: $(REQS_PATH) local-requirements.txt +./envhelp/local.depends: $(REQS_PATH) local-requirements.txt else -./envhelp/py3.depends: $(REQS_PATH) +./envhelp/local.depends: $(REQS_PATH) endif - @rm -f ./envhelp/py3.depends - @echo "upgrading venv pip as required for some dependencies" - @./envhelp/venv/bin/pip3 install --upgrade pip @echo "installing dependencies from $(REQS_PATH)" - @./envhelp/venv/bin/pip3 install -r $(REQS_PATH) + @$(LOCAL_PYTHON_BIN) -m pip install -r $(REQS_PATH) ifeq ($(MIG_ENV),'local') @echo "" @echo "installing development dependencies" - @./envhelp/venv/bin/pip3 install -r local-requirements.txt + @$(LOCAL_PYTHON_BIN) -m pip install -r local-requirements.txt endif - @touch ./envhelp/py3.depends + @touch ./envhelp/local.depends ./envhelp/venv/pyvenv.cfg: @echo "provisioning environment" @/usr/bin/env python3 -m venv ./envhelp/venv - @rm -f ./envhelp/py3.depends + @rm -f ./envhelp/local.depends + @echo "upgrading venv pip as required for some dependencies" + @./envhelp/venv/bin/pip3 install --upgrade pip diff --git a/envhelp/docker/Dockerfile.python2 b/envhelp/docker/Dockerfile.py2 similarity index 100% rename from envhelp/docker/Dockerfile.python2 rename to envhelp/docker/Dockerfile.py2 diff --git a/envhelp/docker/Dockerfile.py3 b/envhelp/docker/Dockerfile.py3 new file mode 100644 index 000000000..c0aeeb9d6 --- /dev/null +++ b/envhelp/docker/Dockerfile.py3 @@ -0,0 +1,8 @@ +FROM python:3.9 + +WORKDIR /usr/src/app + +COPY requirements.txt local-requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt -r local-requirements.txt + +CMD [ "python", "--version" ] diff --git a/envhelp/docker/Dockerfile.pyver b/envhelp/docker/Dockerfile.pyver new file mode 100644 index 000000000..e90a17ef8 --- /dev/null +++ b/envhelp/docker/Dockerfile.pyver @@ -0,0 +1,9 @@ +ARG pyver +FROM python:${pyver} + +WORKDIR /usr/src/app + +COPY requirements.txt local-requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt -r local-requirements.txt + +CMD [ "python", "--version" ] diff --git a/envhelp/dpython b/envhelp/dpython new file mode 100755 index 000000000..cedd5f778 --- /dev/null +++ b/envhelp/dpython @@ -0,0 +1,88 @@ +#!/bin/sh +# +# --- BEGIN_HEADER --- +# +# dpython - wrapper to invoke a containerised python +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +set -e + +SCRIPT_PATH=$(realpath "$0") +SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH") +MIG_BASE=$(realpath "$SCRIPT_BASE/..") + +if [ -n "${PY}" ]; then + PYVER="$PY" + PYTHON_SUFFIX="py$PY" + DOCKER_FILE_SUFFIX="$PYTHON_SUFFIX" +elif [ -n "${PYVER}" ]; then + PY=3 + PYTHON_SUFFIX="pyver-$PYVER" + DOCKER_FILE_SUFFIX="pyver" +else + echo "No python version specified - please supply a PY env var" + exit 1 +fi + +DOCKER_FILE="$SCRIPT_BASE/docker/Dockerfile.$DOCKER_FILE_SUFFIX" +DOCKER_IMAGEID_FILE="$SCRIPT_BASE/$PYTHON_SUFFIX.imageid" + +# NOTE: portable dynamic lookup with docker as default and fallback to podman +DOCKER_BIN=$(command -v docker || command -v podman || echo "") +if [ -z "${DOCKER_BIN}" ]; then + echo "No docker binary found - cannot use for python $PY tests" + exit 1 +fi + +# default PYTHONPATH such that directly executing files in the repo "just works" +# NOTE: this is hard-coded to the mount point used within the container +PYTHONPATH='/usr/src/app' + +# default any variables for container development +MIG_ENV=${MIG_ENV:-'docker'} + +# determine if the image has changed +echo -n "validating python $PY container.. " + +# load a previously written docker image id if present +IMAGEID_STORED=$(cat "$DOCKER_IMAGEID_FILE" 2>/dev/null || echo "") + +IMAGEID=$(${DOCKER_BIN} build -f "$DOCKER_FILE" . -q --build-arg "pyver=$PYVER") +if [ "$IMAGEID" != "$IMAGEID_STORED" ]; then + echo "rebuilt for changes" + + # reset the image id so the next call finds no changes + echo "$IMAGEID" > "$DOCKER_IMAGEID_FILE" +else + echo "no changes needed" +fi + +echo "using image id $IMAGEID" + +# execute python2 within the image passing the supplied arguments + +${DOCKER_BIN} run -it --rm \ + --mount "type=bind,source=$MIG_BASE,target=/usr/src/app" \ + --env "PYTHONPATH=$PYTHONPATH" \ + --env "MIG_ENV=$MIG_ENV" \ + "$IMAGEID" python$PY $@ diff --git a/envhelp/lpython b/envhelp/lpython new file mode 100755 index 000000000..9d42a83d3 --- /dev/null +++ b/envhelp/lpython @@ -0,0 +1,46 @@ +#!/bin/sh +# +# --- BEGIN_HEADER --- +# +# python3 - wrapper to invoke a local python3 virtual environment +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +set -e + +SCRIPT_PATH=$(realpath "$0") +SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH") +MIG_BASE=$(realpath "$SCRIPT_BASE/..") + +PYTHON_BIN=${PYTHON_BIN:-"$SCRIPT_BASE/venv/bin/python3"} +if [ ! -f "${PYTHON_BIN}" ]; then + echo "No python binary found - perhaps the virtual env was not created" + exit 1 +fi + +# default PYTHONPATH such that directly executing files in the repo "just works" +PYTHONPATH=${PYTHONPATH:-"$MIG_BASE"} + +# default any variables for local development +MIG_ENV=${MIG_ENV:-'local'} + +PYTHONPATH="$PYTHONPATH" MIG_ENV="$MIG_ENV" "$PYTHON_BIN" "$@" diff --git a/envhelp/makeconfig.py b/envhelp/makeconfig.py index 703b62e07..0d1941b83 100644 --- a/envhelp/makeconfig.py +++ b/envhelp/makeconfig.py @@ -38,8 +38,9 @@ from mig.shared.install import MIG_BASE, generate_confs -_LOCAL_ENVHELP_OUTPUT_DIR = os.path.realpath( - os.path.join(os.path.dirname(__file__), "output")) +_LOCAL_MIG_BASE = os.path.normpath( + os.path.join(os.path.dirname(__file__), "..")) +_LOCAL_ENVHELP_OUTPUT_DIR = os.path.join(_LOCAL_MIG_BASE, "envhelp/output") _MAKECONFIG_ALLOWED = ["local", "test"] @@ -51,21 +52,27 @@ def _at(sequence, index=-1, default=None): return default -def write_testconfig(env_name, is_py2=False): - confs_name = 'confs' if env_name == 'local' else '%sconfs' % (env_name,) - confs_suffix = 'py2' if is_py2 else 'py3' +def write_testconfig(env_name, is_docker=False): + is_predefined = env_name == 'test' + confs_name = '%sconfs' % (env_name,) + if is_predefined: + confs_suffix = 'docker' if is_docker else 'local' + else: + confs_suffix = 'py3' overrides = { 'destination': os.path.join(_LOCAL_ENVHELP_OUTPUT_DIR, confs_name), 'destination_suffix': "-%s" % (confs_suffix,), } - # determine the paths by which we will access the various configured dirs - if is_py2: + # determine the paths b which we will access the various configured dirs + # the tests output directory - when invoked within + + if is_predefined and is_docker: env_mig_base = '/usr/src/app' else: - env_mig_base = MIG_BASE - conf_dir_path = os.path.join(env_mig_base, "envhelp/output") + env_mig_base = _LOCAL_MIG_BASE + conf_dir_path = os.path.join(env_mig_base, "tests/output") overrides.update(**{ 'mig_code': os.path.join(conf_dir_path, 'mig'), @@ -85,7 +92,7 @@ def write_testconfig(env_name, is_py2=False): def main_(argv): env_name = _at(argv, index=1, default='') - arg_is_py2 = '--python2' in argv + arg_is_docker = '--docker' in argv if env_name == '': raise RuntimeError( @@ -94,7 +101,7 @@ def main_(argv): raise RuntimeError('environment must be one of %s' % (_MAKECONFIG_ALLOWED,)) - write_testconfig(env_name, is_py2=arg_is_py2) + write_testconfig(env_name, is_docker=arg_is_docker) def main(argv=sys.argv): diff --git a/envhelp/python2 b/envhelp/python2 index c0c0351a5..d0b0d04a8 100755 --- a/envhelp/python2 +++ b/envhelp/python2 @@ -29,46 +29,5 @@ set -e SCRIPT_PATH=$(realpath "$0") SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH") -DOCKER_BASE="$SCRIPT_BASE/docker" -DOCKER_IMAGEID_FILE="$SCRIPT_BASE/py2.imageid" -# NOTE: portable dynamic lookup with docker as default and fallback to podman -DOCKER_BIN=$(command -v docker || command -v podman || echo "") -if [ -z "${DOCKER_BIN}" ]; then - echo "No docker binary found - cannot use for python2 tests" - exit 1 -fi - -# default PYTHONPATH such that directly executing files in the repo "just works" -# NOTE: this is hard-coded to the mount point used within the container -PYTHONPATH='/usr/app/src' - -# default any variables for local development -MIG_ENV=${MIG_ENV:-'local'} - -# determine if the image has changed -echo -n "validating container.. " - -# load a previously written docker image id if present -IMAGEID_STORED=$(cat "$DOCKER_IMAGEID_FILE" 2>/dev/null || echo "") - -IMAGEID=$(${DOCKER_BIN} build -f "$DOCKER_BASE/Dockerfile.python2" . -q) -if [ "$IMAGEID" != "$IMAGEID_STORED" ]; then - echo "rebuilt for changes" - - # reset the image id so the next call finds no changes - echo "$IMAGEID" > "$DOCKER_IMAGEID_FILE" -else - echo "no changes needed" -fi - -echo "running with MIG_ENV='$MIG_ENV' under python 2" -echo - -# execute python2 within the image passing the supplied arguments - -${DOCKER_BIN} run -it --rm \ - --mount type=bind,source=.,target=/usr/src/app \ - --env "PYTHONPATH=$PYTHON_PATH" \ - --env "MIG_ENV=$MIG_ENV" \ - "$IMAGEID" python2 "$@" +PY=2 $SCRIPT_BASE/dpython "$@" diff --git a/envhelp/python3 b/envhelp/python3 index 309356a03..9584a6dc5 100755 --- a/envhelp/python3 +++ b/envhelp/python3 @@ -2,7 +2,7 @@ # # --- BEGIN_HEADER --- # -# python3 - wrap python3 virtual environment for testing +# python2 - wrap python2 docker container for testing # Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. @@ -29,21 +29,5 @@ set -e SCRIPT_PATH=$(realpath "$0") SCRIPT_BASE=$(dirname -- "$SCRIPT_PATH") -MIG_BASE=$(realpath "$SCRIPT_BASE/..") -PYTHON3_BIN="$SCRIPT_BASE/venv/bin/python3" -if [ ! -f "${PYTHON3_BIN}" ]; then - echo "No python3 binary found - perhaps the virtual env was not created" - exit 1 -fi - -# default PYTHONPATH such that directly executing files in the repo "just works" -PYTHONPATH=${PYTHONPATH:-"$MIG_BASE"} - -# default any variables for local development -MIG_ENV=${MIG_ENV:-'local'} - -echo "running with MIG_ENV='$MIG_ENV' under python 3" -echo - -PYTHONPATH="$PYTHONPATH" MIG_ENV="$MIG_ENV" "$PYTHON3_BIN" "$@" +PY=3 $SCRIPT_BASE/dpython "$@" diff --git a/local-requirements.txt b/local-requirements.txt index da83dd4b7..7faf3b026 100644 --- a/local-requirements.txt +++ b/local-requirements.txt @@ -7,3 +7,4 @@ autopep8;python_version >= "3" # NOTE: paramiko-3.0.0 dropped python2 and python3.6 support paramiko;python_version >= "3.7" paramiko<3;python_version < "3.7" +werkzeug diff --git a/mig/shared/base.py b/mig/shared/base.py index 64f12b370..372c9bc30 100644 --- a/mig/shared/base.py +++ b/mig/shared/base.py @@ -36,6 +36,7 @@ import re # IMPORTANT: do not import any other MiG modules here - to avoid import loops +from mig.shared.compat import PY2 from mig.shared.defaults import default_str_coding, default_fs_coding, \ keyword_all, keyword_auto, sandbox_names, _user_invisible_files, \ _user_invisible_dirs, _vgrid_xgi_scripts, cert_field_order, csrf_field, \ @@ -496,7 +497,7 @@ def is_unicode(val): return (type(u"") == type(val)) -def force_utf8(val, highlight=''): +def _force_utf8_py2(val, highlight=''): """Internal helper to encode unicode strings to utf8 version. Actual changes are marked out with the highlight string if given. """ @@ -507,6 +508,31 @@ def force_utf8(val, highlight=''): return val return "%s%s%s" % (highlight, val.encode("utf8"), highlight) +def _force_utf8_py3(val, highlight='', stringify=True): + """Internal helper to encode unicode strings to utf8 version. Actual + changes are marked out with the highlight string if given. + The optional stringify turns ALL values including numbers into string. + """ + # We run into all kind of nasty encoding problems if we mix + if not isinstance(val, basestring): + if stringify: + val = "%s" % val + else: + return val + if not is_unicode(val): + return val + if is_unicode(highlight): + hl_utf = highlight.encode("utf8") + else: + hl_utf = highlight + return (b"%s%s%s" % (hl_utf, val.encode("utf8"), hl_utf)) + + +if PY2: + force_utf8 = _force_utf8_py2 +else: + force_utf8 = _force_utf8_py3 + def force_utf8_rec(input_obj, highlight=''): """Recursive object conversion from unicode to utf8: useful to convert e.g. @@ -516,8 +542,9 @@ def force_utf8_rec(input_obj, highlight=''): if isinstance(input_obj, dict): return {force_utf8_rec(i, highlight): force_utf8_rec(j, highlight) for (i, j) in input_obj.items()} - elif isinstance(input_obj, list): - return [force_utf8_rec(i, highlight) for i in input_obj] + elif isinstance(input_obj, (list, tuple)): + thetype = type(input_obj) + return thetype(force_utf8_rec(i, highlight) for i in input_obj) elif is_unicode(input_obj): return force_utf8(input_obj, highlight) else: @@ -544,8 +571,9 @@ def force_unicode_rec(input_obj, highlight=''): if isinstance(input_obj, dict): return {force_unicode_rec(i, highlight): force_unicode_rec(j, highlight) for (i, j) in input_obj.items()} - elif isinstance(input_obj, list): - return [force_unicode_rec(i, highlight) for i in input_obj] + elif isinstance(input_obj, (list, tuple)): + thetype = type(input_obj) + return thetype(force_utf8_rec(i, highlight) for i in input_obj) elif not is_unicode(input_obj): return force_unicode(input_obj, highlight) else: diff --git a/mig/shared/safeinput.py b/mig/shared/safeinput.py index 592250755..e91937d8c 100644 --- a/mig/shared/safeinput.py +++ b/mig/shared/safeinput.py @@ -58,7 +58,7 @@ from html import escape as escape_html assert escape_html is not None -from mig.shared.base import force_unicode, force_utf8 +from mig.shared.base import force_unicode, force_native_str from mig.shared.defaults import src_dst_sep, username_charset, \ username_max_length, session_id_charset, session_id_length, \ subject_id_charset, subject_id_min_length, subject_id_max_length, \ @@ -2294,7 +2294,7 @@ def __init__(self, value): def __str__(self): """Return string representation""" - return force_utf8(force_unicode(self.value)) + return force_native_str(self.value) def main(_exit=sys.exit, _print=print): diff --git a/mig/unittest/testcore.py b/mig/unittest/testcore.py index 34998b8db..8944fd45b 100644 --- a/mig/unittest/testcore.py +++ b/mig/unittest/testcore.py @@ -42,18 +42,17 @@ invisible_path, allow_script, brief_list -_LOCAL_MIG_BASE = '/usr/src/app' if PY2 else MIG_BASE # account for execution in container -_PYTHON_MAJOR = '2' if PY2 else '3' -_TEST_CONF_DIR = os.path.join(MIG_BASE, "envhelp/output/testconfs-py%s" % (_PYTHON_MAJOR,)) -_TEST_CONF_FILE = os.path.join(_TEST_CONF_DIR, "MiGserver.conf") +_TEST_CONF_FILE = os.environ['MIG_CONF'] +_TEST_CONF_DIR = os.path.dirname(_TEST_CONF_FILE) _TEST_CONF_SYMLINK = os.path.join(MIG_BASE, "envhelp/output/testconfs") def _assert_local_config(): try: - link_stat = os.lstat(_TEST_CONF_SYMLINK) - assert stat.S_ISLNK(link_stat.st_mode) - configdir_stat = os.stat(_TEST_CONF_DIR) + #link_stat = os.lstat(_TEST_CONF_SYMLINK) + #assert stat.S_ISLNK(link_stat.st_mode) + _test_conf_dir = os.path.dirname(_TEST_CONF_DIR) + configdir_stat = os.stat(_test_conf_dir) assert stat.S_ISDIR(configdir_stat.st_mode) config = ConfigParser() config.read([_TEST_CONF_FILE]) @@ -67,23 +66,16 @@ def _assert_local_config_global_values(config): for path in ('mig_path', 'certs_path', 'state_path'): path_value = config_global_values.get(path) - if not is_path_within(path_value, start=_LOCAL_MIG_BASE): + if not is_path_within(path_value, start=MIG_BASE): raise AssertionError('local config contains bad path: %s=%s' % (path, path_value)) return config_global_values -def main(_exit=sys.exit): +def main(configuration, _exit=sys.exit): config = _assert_local_config() config_global_values = _assert_local_config_global_values(config) - from mig.shared.conf import get_configuration_object - configuration = get_configuration_object(_TEST_CONF_FILE, skip_log=True, - disable_auth_log=True) - logging.basicConfig(filename=None, level=logging.INFO, - format="%(asctime)s %(levelname)s %(message)s") - configuration.logger = logging - print("Running unit test on shared core functions ..") short_alias = 'email' @@ -192,4 +184,5 @@ def main(_exit=sys.exit): _exit(0) if __name__ == "__main__": - main() + from mig.shared.conf import get_configuration_object + main(get_configuration_object()) diff --git a/mig/wsgi-bin/migwsgi.py b/mig/wsgi-bin/migwsgi.py index 73987133e..b7b0bb2ce 100755 --- a/mig/wsgi-bin/migwsgi.py +++ b/mig/wsgi-bin/migwsgi.py @@ -34,7 +34,7 @@ from mig.shared import returnvalues from mig.shared.bailout import bailout_helper, crash_helper, compact_string from mig.shared.base import requested_backend, allow_script, \ - is_default_str_coding, force_default_str_coding_rec + is_default_str_coding, force_default_str_coding_rec, force_utf8 from mig.shared.defaults import download_block_size, default_fs_coding from mig.shared.conf import get_configuration_object from mig.shared.objecttypes import get_object_type_info @@ -43,14 +43,19 @@ from mig.shared.scriptinput import fieldstorage_to_dict +def _import_backend(backend): + import_path = 'mig.shared.functionality.%s' % backend + module_handle = importlib.import_module(import_path) + return module_handle.main + + def object_type_info(object_type): """Lookup object type""" return get_object_type_info(object_type) -def stub(configuration, client_id, import_path, backend, user_arguments_dict, - environ): +def stub(configuration, client_id, user_arguments_dict, environ, _retrieve_handler): """Run backend on behalf of client_id with supplied user_arguments_dict. I.e. import main from import_path and execute it with supplied arguments. """ @@ -61,6 +66,7 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict, before_time = time.time() output_objects = [] + backend = 'UNKNOWN' main = dummy_main # _logger.debug("stub for backend %r" % backend) @@ -69,10 +75,12 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict, # NEVER print/output it verbatim before it is validated below. try: + default_page = configuration.site_landing_page # TODO: avoid doing this work a second time + backend = requested_backend(environ, fallback=default_page) valid_backend_name(backend) except InputException as iex: - _logger.error("%s refused to import invalid backend %r (%s): %s" % - (_addr, backend, import_path, iex)) + _logger.error("%s refused to import invalid backend %r: %s" % + (_addr, backend, iex)) bailout_helper(configuration, backend, output_objects, header_text='User Error') output_objects.extend([ @@ -81,18 +89,17 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict, {'object_type': 'link', 'text': 'Go to default interface', 'destination': configuration.site_landing_page} ]) - return (output_objects, returnvalues.CLIENT_ERROR) + return backend, (output_objects, returnvalues.CLIENT_ERROR) try: # Import main from backend module # _logger.debug("import main from %r" % import_path) # NOTE: dynamic module loading to find corresponding main function - module_handle = importlib.import_module(import_path) - main = module_handle.main + main = _retrieve_handler(backend) except Exception as err: - _logger.error("%s could not import %r (%s): %s" % - (_addr, backend, import_path, err)) + _logger.error("%s could not import %r: %s" % + (_addr, backend, err)) bailout_helper(configuration, backend, output_objects) output_objects.extend([ {'object_type': 'error_text', 'text': @@ -100,22 +107,22 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict, {'object_type': 'link', 'text': 'Go to default interface', 'destination': configuration.site_landing_page} ]) - return (output_objects, returnvalues.SYSTEM_ERROR) + return backend, (output_objects, returnvalues.SYSTEM_ERROR) # _logger.debug("imported main %s" % main) # Now backend value is validated to be safe for output if not isinstance(user_arguments_dict, dict): - _logger.error("%s invalid user args %s for %s" % (_addr, + _logger.error("%s invalid user args %s for backend %r" % (_addr, user_arguments_dict, - import_path)) + backend)) bailout_helper(configuration, backend, output_objects, header_text='Input Error') output_objects.append( {'object_type': 'error_text', 'text': 'User input is not on expected format!'}) - return (output_objects, returnvalues.INVALID_ARGUMENT) + return backend, (output_objects, returnvalues.INVALID_ARGUMENT) try: (output_objects, (ret_code, ret_msg)) = main(client_id, @@ -125,7 +132,7 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict, _logger.error("%s script crashed:\n%s" % (_addr, traceback.format_exc())) crash_helper(configuration, backend, output_objects) - return (output_objects, returnvalues.ERROR) + return backend, (output_objects, returnvalues.ERROR) (val_ret, val_msg) = validate(output_objects) if not val_ret: @@ -138,7 +145,7 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict, after_time = time.time() output_objects.append({'object_type': 'timing_info', 'text': "done in %.3fs" % (after_time - before_time)}) - return (output_objects, (ret_code, ret_msg)) + return backend, (output_objects, (ret_code, ret_msg)) def wrap_wsgi_errors(environ, configuration, max_line_len=100): @@ -193,6 +200,14 @@ def application(environ, start_response): *start_response* is a helper function used to deliver the client response. """ + def _set_os_environ(value): + os.environ = value + + return _application(None, environ, start_response, _set_environ=_set_os_environ, _wrap_wsgi_errors=wrap_wsgi_errors) + + +def _application(configuration, environ, start_response, _set_environ, _fieldstorage_to_dict=fieldstorage_to_dict, _format_output=format_output, _retrieve_handler=_import_backend, _wrap_wsgi_errors=True, _config_file=None, _skip_log=False): + # NOTE: pass app environ including apache and query args on to sub handlers # through the usual 'os.environ' channel expected in functionality # handlers. Special care is needed to avoid various sub-interpreter @@ -235,18 +250,20 @@ def application(environ, start_response): os_env_value)) # Assign updated environ to LOCAL os.environ for the rest of this session - os.environ = environ + _set_environ(environ) # NOTE: redirect stdout to stderr in python 2 only. It breaks logger in 3 # and stdout redirection apparently is already handled there. if sys.version_info[0] < 3: sys.stdout = sys.stderr - configuration = get_configuration_object() + if configuration is None: + configuration = get_configuration_object(_config_file, _skip_log) + _logger = configuration.logger # NOTE: replace default wsgi errors to apache error log with our own logs - wrap_wsgi_errors(environ, configuration) + _wrap_wsgi_errors(environ, configuration) for line in env_sync_status: _logger.debug(line) @@ -298,22 +315,20 @@ def application(environ, start_response): default_page = configuration.site_landing_page script_name = requested_backend(environ, fallback=default_page, strip_ext=False) - backend = requested_backend(environ, fallback=default_page) # _logger.debug('DEBUG: wsgi found backend %s and script %s' % # (backend, script_name)) fieldstorage = cgi.FieldStorage(fp=environ['wsgi.input'], environ=environ) - user_arguments_dict = fieldstorage_to_dict(fieldstorage) + user_arguments_dict = _fieldstorage_to_dict(fieldstorage) if 'output_format' in user_arguments_dict: output_format = user_arguments_dict['output_format'][0] - module_path = 'mig.shared.functionality.%s' % backend (allow, msg) = allow_script(configuration, script_name, client_id) if allow: # _logger.debug("wsgi handling script: %s" % script_name) - (output_objs, ret_val) = stub(configuration, client_id, - module_path, backend, - user_arguments_dict, environ) + backend, (output_objs, ret_val) = stub(configuration, client_id, + user_arguments_dict, environ, + _retrieve_handler) else: _logger.warning("wsgi handling refused script:%s" % script_name) (output_objs, ret_val) = reject_main(client_id, @@ -363,7 +378,7 @@ def application(environ, start_response): output_objs.append(wsgi_entry) _logger.debug("call format %r output to %s" % (backend, output_format)) - output = format_output(configuration, backend, ret_code, ret_msg, + output = _format_output(configuration, backend, ret_code, ret_msg, output_objs, output_format) # _logger.debug("formatted %s output to %s" % (backend, output_format)) # _logger.debug("output:\n%s" % [output]) @@ -372,7 +387,7 @@ def application(environ, start_response): _logger.error( "Formatted output is NOT on default str coding: %s" % [output[:100]]) err_mark = '__****__' - output = format_output(configuration, backend, ret_code, ret_msg, + output = _format_output(configuration, backend, ret_code, ret_msg, force_default_str_coding_rec( output_objs, highlight=err_mark), output_format) @@ -396,7 +411,14 @@ def application(environ, start_response): # NOTE: send response to client but don't crash e.g. on closed connection try: start_response(status, response_headers) + except IOError as ioe: + _logger.warning("WSGI %s for %s could not deliver output: %s" % + (backend, client_id, ioe)) + except Exception as exc: + _logger.error("WSGI %s for %s crashed during response: %s" % + (backend, client_id, exc)) + try: # NOTE: we consistently hit download error for archive files reaching ~2GB # with showfreezefile.py on wsgi but the same on cgi does NOT suffer # the problem for the exact same files. It seems wsgi has a limited @@ -410,12 +432,15 @@ def application(environ, start_response): _logger.info("WSGI %s yielding %d output parts (%db)" % (backend, chunk_parts, content_length)) # _logger.debug("send chunked %r response to client" % backend) - for i in xrange(chunk_parts): + for i in list(range(chunk_parts)): # _logger.debug("WSGI %s yielding part %d / %d output parts" % # (backend, i+1, chunk_parts)) # end index may be after end of content - but no problem part = output[i*download_block_size:(i+1)*download_block_size] - yield part + if output_format == 'file': + yield part + else: + yield force_utf8(part) if chunk_parts > 1: _logger.info("WSGI %s finished yielding all %d output parts" % (backend, chunk_parts)) diff --git a/tests/__init__.py b/tests/__init__.py index e69de29bb..bcec2ab8a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,10 @@ +def _print_identity(): + import os + import sys + python_version_string = sys.version.split(' ')[0] + mig_env = os.environ.get('MIG_ENV', 'local') + print("running with MIG_ENV='%s' under Python %s" % + (mig_env, python_version_string)) + print("") + +_print_identity() diff --git a/tests/support/__init__.py b/tests/support/__init__.py index 499ea005c..422182b4a 100644 --- a/tests/support/__init__.py +++ b/tests/support/__init__.py @@ -42,27 +42,30 @@ from tests.support.configsupp import FakeConfiguration from tests.support.suppconst import MIG_BASE, TEST_BASE, TEST_FIXTURE_DIR, \ - TEST_OUTPUT_DIR, TEST_DATA_DIR + TEST_DATA_DIR, TEST_OUTPUT_DIR, ENVHELP_OUTPUT_DIR -PY2 = (sys.version_info[0] == 2) +from tests.support._env import MIG_ENV, PY2 -# force defaults to a local environment -os.environ['MIG_ENV'] = 'local' +# Provide access to a configuration file for the active environment. -# expose the configured environment as a constant -MIG_ENV = os.environ['MIG_ENV'] - -if MIG_ENV == 'local': - # force testconfig as the conig file path - is_py2 = PY2 - _conf_dir_suffix = "-py%s" % ('2' if is_py2 else '3',) - _conf_dir = "testconfs%s" % (_conf_dir_suffix,) - _local_conf = os.path.join( - MIG_BASE, 'envhelp/output', _conf_dir, 'MiGserver.conf') +if MIG_ENV in ('local', 'docker'): + # force local testconfig + _output_dir = os.path.join(MIG_BASE, 'envhelp/output') + _conf_dir_name = "testconfs-%s" % (MIG_ENV,) + _conf_dir = os.path.join(_output_dir, _conf_dir_name) + _local_conf = os.path.join(_conf_dir, 'MiGserver.conf') _config_file = os.getenv('MIG_CONF', None) if _config_file is None: os.environ['MIG_CONF'] = _local_conf + # adjust the link through which confs are accessed to suit the environment + _conf_link = os.path.join(_output_dir, 'testconfs') + assert os.path.lexists(_conf_link) # it must already exist + os.remove(_conf_link) # blow it away + os.symlink(_conf_dir, _conf_link) # recreate it using the active MIG_BASE +else: + raise NotImplementedError() + # All MiG related code will at some point include bits from the mig module # namespace. Rather than have this knowledge spread through every test file, # make the sole responsbility of test files to find the support file and @@ -75,7 +78,10 @@ os.mkdir(TEST_OUTPUT_DIR) except EnvironmentError as enverr: if enverr.errno == errno.EEXIST: # FileExistsError - shutil.rmtree(TEST_OUTPUT_DIR) + try: + shutil.rmtree(TEST_OUTPUT_DIR) + except Exception as exc: + raise os.mkdir(TEST_OUTPUT_DIR) # Exports to expose at the top level from the support library. @@ -146,7 +152,11 @@ def tearDown(self): if os.path.islink(path): os.remove(path) elif os.path.isdir(path): - shutil.rmtree(path) + try: + shutil.rmtree(path) + except Exception as exc: + print(path) + raise elif os.path.exists(path): os.remove(path) else: @@ -164,6 +174,11 @@ def before_each(self): def _register_check(self, check_callable): self._cleanup_checks.append(check_callable) + def _register_path(self, cleanup_path): + assert os.path.isabs(cleanup_path) + self._cleanup_paths.add(cleanup_path) + return cleanup_path + def _reset_logging(self, stream): root_logger = logging.getLogger() root_handler = root_logger.handlers[0] @@ -188,11 +203,26 @@ def _provide_configuration(self): @property def configuration(self): """Init a fake configuration if not already done""" - if self._configuration is None: - configuration_to_make = self._provide_configuration() - self._configuration = self._make_configuration_instance( - configuration_to_make) - return self._configuration + + if self._configuration is not None: + return self._configuration + + configuration_to_make = self._provide_configuration() + configuration_instance = self._make_configuration_instance( + configuration_to_make) + + if configuration_to_make == 'testconfig': + # use the paths defined by the loaded configuration to create + # the directories which are expected to be present by the code + os.mkdir(self._register_path(configuration_instance.certs_path)) + os.mkdir(self._register_path(configuration_instance.state_path)) + log_path = os.path.join(configuration_instance.state_path, "log") + os.mkdir(self._register_path(log_path)) + + self._configuration = configuration_instance + + return configuration_instance + @property def logger(self): @@ -361,18 +391,31 @@ def fixturepath(relative_path): return tmp_path -def temppath(relative_path, test_case, ensure_dir=False, skip_clean=False, - skip_output_anchor=False): +def temppath(relative_path, test_case, ensure_dir=False, skip_clean=False): """Register relative_path as a temp path and schedule automatic clean up after unit tests unless skip_clean is set. Anchors the temp path in internal test output dir unless skip_output_anchor is set. Returns resulting temp path. """ assert isinstance(test_case, MigTestCase) - if not skip_output_anchor: - tmp_path = os.path.join(TEST_OUTPUT_DIR, relative_path) - else: + + if os.path.isabs(relative_path): + # the only permitted paths are those within the output directory set + # aside for execution of the test suite: this will be enforced below + # so effectively submit the supplied path for scrutiny tmp_path = relative_path + else: + tmp_path = os.path.join(TEST_OUTPUT_DIR, relative_path) + + # failsafe path checking that supplied paths are rooted within valid paths + is_tmp_path_within_safe_dir = False + for start in (ENVHELP_OUTPUT_DIR): + is_tmp_path_within_safe_dir = is_path_within(tmp_path, start=start) + if is_tmp_path_within_safe_dir: + break + if not is_tmp_path_within_safe_dir: + raise AssertionError("ABORT: corrupt test path=%s" % (tmp_path,)) + if ensure_dir: try: os.mkdir(tmp_path) diff --git a/tests/support/_env.py b/tests/support/_env.py new file mode 100644 index 000000000..2c71386a4 --- /dev/null +++ b/tests/support/_env.py @@ -0,0 +1,11 @@ +import os +import sys + +# expose the configured environment as a constant +MIG_ENV = os.environ.get('MIG_ENV', 'local') + +# force the chosen environment globally +os.environ['MIG_ENV'] = MIG_ENV + +# expose a boolean indicating whether we are executing on Python 2 +PY2 = (sys.version_info[0] == 2) diff --git a/tests/support/htmlsupp.py b/tests/support/htmlsupp.py new file mode 100644 index 000000000..61fcadbee --- /dev/null +++ b/tests/support/htmlsupp.py @@ -0,0 +1,84 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# htmlsupp - test support library for HTML +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +"""Test support library for HTML.""" + + +class HtmlAssertMixin: + """Custom assertions for HTML containing strings.""" + + def assertHtmlElement(self, value, tag_name): + """Check that an occurrence of the specifid tag within an HTML input + string can be found. Returns the textual content of the first match. + """ + + self.assertIsValidHtmlDocument(value) + + # TODO: this is a definitively stop-gap way of finding a tag within the HTML + # and is used purely to keep this initial change to a reasonable size. + + tag_open = ''.join(['<', tag_name, '>']) + tag_open_index = value.index(tag_open) + tag_open_index_after = tag_open_index + len(tag_open) + + tag_close = ''.join(['']) + tag_close_index = value.index(tag_close, tag_open_index_after) + + return value[tag_open_index_after:tag_close_index] + + def assertHtmlElementTextContent(self, value, tag_name, expected_text, trim_newlines=True): + """Check there is an occurrence of a tag within an HTML input string + and check the text it encloses equals exactly the expecatation. + """ + + self.assertIsValidHtmlDocument(value) + + # TODO: this is a definitively stop-gap way of finding a tag within the HTML + # and is used purely to keep this initial change to a reasonable size. + + actual_text = self.assertHtmlElement(value, tag_name) + if trim_newlines: + actual_text = actual_text.strip('\n') + self.assertEqual(actual_text, expected_text) + + def assertIsValidHtmlDocument(self, value): + """Check that the input string contains a valid HTML document. + """ + + assert isinstance(value, type(u"")), "input string was not utf8" + + error = None + try: + has_doctype = value.startswith("') + maybe_document_end = value[end_html_tag_idx:].rstrip() + assert maybe_document_end == '', "no valid document closer" + except Exception as exc: + error = exc + if error: + raise AssertionError("failed to verify input string as HTML: %s", str(error)) diff --git a/tests/support/suppconst.py b/tests/support/suppconst.py index 15912e933..148303f0d 100644 --- a/tests/support/suppconst.py +++ b/tests/support/suppconst.py @@ -27,11 +27,33 @@ import os +from tests.support._env import MIG_ENV -# Use abspath for __file__ on Py2 -_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_BASE = os.path.normpath(os.path.join(_SUPPORT_DIR, "..")) +if MIG_ENV == 'local': + # Use abspath for __file__ on Py2 + _SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__)) +elif MIG_ENV == 'docker': + _SUPPORT_DIR = '/usr/src/app/tests/support' +else: + raise NotImplementedError("ABORT: unsupported environment: %s" % (MIG_ENV,)) + +MIG_BASE = os.path.realpath(os.path.join(_SUPPORT_DIR, "../..")) +TEST_BASE = os.path.join(MIG_BASE, "tests") TEST_DATA_DIR = os.path.join(TEST_BASE, "data") TEST_FIXTURE_DIR = os.path.join(TEST_BASE, "fixture") TEST_OUTPUT_DIR = os.path.join(TEST_BASE, "output") -MIG_BASE = os.path.realpath(os.path.join(TEST_BASE, "..")) +ENVHELP_DIR = os.path.join(MIG_BASE, "envhelp") +ENVHELP_OUTPUT_DIR = os.path.join(ENVHELP_DIR, "output") + + +if __name__ == '__main__': + def print_root_relative(prefix, path): + print("%s = /%s" % (prefix, os.path.relpath(path, MIG_BASE))) + + print("# base paths") + print("root=%s" % (MIG_BASE,)) + print("# envhelp paths") + print_root_relative("output", ENVHELP_OUTPUT_DIR) + print("# test paths") + print_root_relative("fixture", TEST_FIXTURE_DIR) + print_root_relative("output", TEST_OUTPUT_DIR) diff --git a/tests/support/wsgibinsupp.py b/tests/support/wsgibinsupp.py new file mode 100644 index 000000000..efd98fab1 --- /dev/null +++ b/tests/support/wsgibinsupp.py @@ -0,0 +1,206 @@ +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# htmlsupp - test support library for WSGI +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +from collections import namedtuple +import codecs +from io import BytesIO + +from mig.shared.output import format_output +import mig.shared.returnvalues as returnvalues + + +def _is_return_value(return_value): + defined_return_values = returnvalues.__dict__.values() + return return_value in defined_return_values + + +def create_instrumented_fieldstorage_to_dict(): + def _instrumented_fieldstorage_to_dict(fieldstorage): + return _instrumented_fieldstorage_to_dict._result + + _instrumented_fieldstorage_to_dict._result = { + 'output_format': ('html',) + } + + def set_result(result): + _instrumented_fieldstorage_to_dict._result = result + + _instrumented_fieldstorage_to_dict.set_result = set_result + + return _instrumented_fieldstorage_to_dict + + +def create_instrumented_format_output(): + def _instrumented_format_output( + configuration, + backend, + ret_val, + ret_msg, + out_obj, + outputformat, + ): + # record the call args + # capture the original before altering it + call_args_out_obj = list(out_obj) + call_args = (configuration, backend, ret_val, ret_msg, + call_args_out_obj, outputformat,) + _instrumented_format_output.calls.append({'args': call_args}) + + if _instrumented_format_output._file: + return format_output( + configuration, + backend, + ret_val, + ret_msg, + out_obj, + outputformat, + ) + + # FIXME: the following is a workaround for a bug that exists between the WSGI wrapper + # and the output formatter - specifically, the latter adds default header and + # title if start does not exist, but the former ensures that start always exists + # meaning that a default response under WSGI is missing half the HTML. + start_obj_idx = next((i for i, obj in enumerate( + out_obj) if obj['object_type'] == 'start')) + insertion_idx = start_obj_idx + + # FIXME: format_output() is sensitive to ordering and MUST see a title object _before_ + # anything else otherwise the preamble ends up written above the header and thus + # an invalid HTML page is served. + insertion_idx += 1 + out_obj.insert(insertion_idx, { + 'object_type': 'title', + 'text': _instrumented_format_output.values['title_text'], + 'meta': '', + 'style': {}, + 'script': {}, + }) + + insertion_idx += 1 + out_obj.insert(insertion_idx, { + 'object_type': 'header', + 'text': _instrumented_format_output.values['header_text'] + }) + + return format_output( + configuration, + backend, + ret_val, + ret_msg, + out_obj, + outputformat, + ) + _instrumented_format_output.calls = [] + _instrumented_format_output._file = False + _instrumented_format_output.values = dict( + title_text='', + header_text='', + ) + + def _set_file(is_enabled): + _instrumented_format_output._file = is_enabled + + setattr(_instrumented_format_output, 'set_file', _set_file) + + def _program_values(**kwargs): + _instrumented_format_output.values.update(kwargs) + + _instrumented_format_output.set_values = _program_values + + return _instrumented_format_output + + +def create_instrumented_retrieve_handler(): + def _simulated_action(*args): + return _simulated_action.returning or ([], returnvalues.ERROR) + _simulated_action.calls = [] + _simulated_action.returning = None + + def _program_response(output_objects=None, return_value=None): + assert _is_return_value( + return_value), "return value must be present in returnvalues" + assert isinstance(output_objects, list) + _simulated_action.returning = (output_objects, return_value) + + def _instrumented_retrieve_handler(*args): + _instrumented_retrieve_handler.calls.append(tuple(args)) + return _simulated_action + _instrumented_retrieve_handler.calls = [] + + _instrumented_retrieve_handler.program = _program_response + _instrumented_retrieve_handler.simulated = _simulated_action + + return _instrumented_retrieve_handler + + +class WsgibinInstrumentation: + def __init__(self): + self.fieldstorage_to_dict = create_instrumented_fieldstorage_to_dict() + self.format_output = create_instrumented_format_output() + self.retrieve_handler = create_instrumented_retrieve_handler() + + def _set_response_content(self, content, returnvalue): + self.retrieve_handler.program(content, returnvalue) + + def _set_response_file(self, returnbytes, returnvalue): + self.fieldstorage_to_dict.set_result({ + 'output_format': ('file',) + }) + self.format_output.set_file(True) + file_obj = {'object_type': 'binary', 'data': returnbytes} + self.set_response([file_obj], returnvalue) + + def set_response(self, content, returnvalue, responding_with='objects'): + assert not (content is not None and file is not None) + + if responding_with == 'file': + assert isinstance( + returnvalue, bytes), "file response demands bytes" + self._set_response_file(content, returnvalue) + elif responding_with == 'objects': + self._set_response_content(content, returnvalue) + else: + raise NotImplementedError() + + +class WsgibinAssertMixin: + def assertWsgibinInstrumentation(self, instrumentation=None): + if instrumentation is None: + instrumentation = getattr(self, 'wsgibin_instrumentation', None) + assert isinstance(instrumentation, WsgibinInstrumentation) + + simulated_action = instrumentation.retrieve_handler.simulated + self.assertIsNotNone(simulated_action.returning, + "no response programmed") + + def was_called(fake): + assert hasattr(fake, 'calls') + return len(fake.calls) > 0 + + self.assertTrue(was_called( + instrumentation.format_output), "no output generated") + self.assertTrue(was_called( + instrumentation.retrieve_handler), "no output generated") diff --git a/tests/support/wsgisupp.py b/tests/support/wsgisupp.py new file mode 100644 index 000000000..5776bfce3 --- /dev/null +++ b/tests/support/wsgisupp.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# htmlsupp - test support library for WSGI +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +from collections import namedtuple +import codecs +from io import BytesIO +from werkzeug.datastructures import MultiDict + +from tests.support._env import PY2 + +if PY2: + from urllib import urlencode + from urlparse import urlparse +else: + from urllib.parse import urlencode, urlparse + +"""Test support library for WSGI.""" + + +_PreparedWsgi = namedtuple('_PreparedWsgi', ['environ', 'start_response']) + + +class FakeWsgiStartResponse: + """Glue object that conforms to the same interface as the start_response() + in the WSGI specs but records the calls to it such that they can be + inspected and, for our purposes, asserted against.""" + + def __init__(self): + self.calls = [] + + def __call__(self, status, headers, exc=None): + self.calls.append((status, headers, exc)) + + +def create_wsgi_environ(configuration, wsgi_url, method='GET', query=None, headers=None, form=None): + parsed_url = urlparse(wsgi_url) + + if query: + method = 'GET' + + request_query = urlencode(query) + wsgi_input = () + elif form: + method = 'POST' + request_query = '' + + body = urlencode(MultiDict(form)).encode('ascii') + + headers = headers or {} + if not 'Content-Type' in headers: + headers['Content-Type'] = 'application/x-www-form-urlencoded' + + headers['Content-Length'] = str(len(body)) + wsgi_input = BytesIO(body) + else: + request_query = parsed_url.query + wsgi_input = () + + environ = {} + environ['wsgi.input'] = wsgi_input + environ['wsgi.url_scheme'] = parsed_url.scheme + environ['wsgi.version'] = (1, 0) + environ['MIG_CONF'] = configuration.config_file + environ['HTTP_HOST'] = parsed_url.netloc + environ['PATH_INFO'] = parsed_url.path + environ['QUERY_STRING'] = request_query + environ['REQUEST_METHOD'] = method + environ['SCRIPT_URI'] = ''.join(('http://', environ['HTTP_HOST'], environ['PATH_INFO'])) + + if headers: + for k, v in headers.items(): + header_key = k.replace('-', '_').upper() + if header_key.startswith('CONTENT'): + # Content-* headers must not be prefixed in WSGI + pass + else: + header_key = "HTTP_%s" % (header_key), + environ[header_key] = v + + return environ + + +def create_wsgi_start_response(): + return FakeWsgiStartResponse() + + +def prepare_wsgi(configuration, url, **kwargs): + return _PreparedWsgi( + create_wsgi_environ(configuration, url, **kwargs), + create_wsgi_start_response() + ) + + +def _trigger_and_unpack_result(wsgi_result, content_kind='textual'): + assert content_kind in ('textual', 'binary') + + chunks = list(wsgi_result) + assert len(chunks) > 0, "invocation returned no output" + complete_value = b''.join(chunks) + if content_kind == 'binary': + decoded_value = complete_value + else: + decoded_value = codecs.decode(complete_value, 'utf8') + return decoded_value + + +class WsgiAssertMixin: + """Custom assertions for verifying server code executed under test.""" + + def assertWsgiResponse(self, wsgi_result, fake_wsgi, expected_status_code, + content_kind='textual'): + assert isinstance(fake_wsgi, _PreparedWsgi) + + content = _trigger_and_unpack_result(wsgi_result, content_kind=content_kind) + + def called_once(fake): + assert hasattr(fake, 'calls') + return len(fake.calls) == 1 + + fake_start_response = fake_wsgi.start_response + + try: + self.assertTrue(called_once(fake_start_response)) + except AssertionError: + if len(fake.calls) == 0: + raise AssertionError("WSGI handler did not respond") + else: + raise AssertionError("WSGI handler responded more than once") + + wsgi_call = fake_start_response.calls[0] + + # check for expected HTTP status code + wsgi_status = wsgi_call[0] + actual_status_code = int(wsgi_status[0:3]) + self.assertEqual(actual_status_code, expected_status_code) + + headers = dict(wsgi_call[1]) + + return content, headers diff --git a/tests/test_booleans.py b/tests/test_booleans.py index 3c37c1ce3..5246197ee 100644 --- a/tests/test_booleans.py +++ b/tests/test_booleans.py @@ -1,11 +1,14 @@ from __future__ import print_function -import sys -from unittest import TestCase +from tests.support import MigTestCase, testmain -class TestBooleans(TestCase): +class TestBooleans(MigTestCase): def test_true(self): self.assertEqual(True, True) def test_false(self): self.assertEqual(False, False) + + +if __name__ == '__main__': + testmain() diff --git a/tests/test_mig_shared_base.py b/tests/test_mig_shared_base.py new file mode 100644 index 000000000..82145cb20 --- /dev/null +++ b/tests/test_mig_shared_base.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# test_mig_shared_base - unit test of the corresponding mig shared module +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""Unit test base functions""" + +import binascii +import codecs +import os +import sys + +from tests.support import PY2, MigTestCase, testmain + +from mig.shared.base import force_utf8 + +DUMMY_STRING = "foo bÆr baz" +DUMMY_UNICODE = u'UniCode123½¾µßðþđŋħĸþł@ª€£$¥©®' + + +class MigSharedBase(MigTestCase): + """Unit tests of fucntions within the mig.shared.base module.""" + + def test_force_utf8_on_string(self): + actual = force_utf8(DUMMY_STRING) + + self.assertIsInstance(actual, bytes) + self.assertEqual(binascii.hexlify(actual), b'666f6f2062c386722062617a') + + def test_force_utf8_on_unicode(self): + actual = force_utf8(DUMMY_UNICODE) + + self.assertIsInstance(actual, bytes) + self.assertEqual(actual, codecs.encode(DUMMY_UNICODE, 'utf8')) + + +if __name__ == '__main__': + testmain() diff --git a/tests/test_mig_shared_functionality_cat.py b/tests/test_mig_shared_functionality_cat.py index b7edaab5a..e02af8896 100644 --- a/tests/test_mig_shared_functionality_cat.py +++ b/tests/test_mig_shared_functionality_cat.py @@ -75,19 +75,20 @@ def before_each(self): test_user_dir = os.path.join(conf_user_home, test_client_dir) # ensure a user db that includes our test user + conf_user_db_home = ensure_dirs_exist(self.configuration.user_db_home) - temppath(conf_user_db_home, self, skip_output_anchor=True) + temppath(conf_user_db_home, self) db_fixture, db_fixture_file = fixturefile('MiG-users.db--example', fixture_format='binary', include_path=True) test_db_file = temppath(fixturefile_normname('MiG-users.db--example', prefix=conf_user_db_home), - self, skip_output_anchor=True) + self) shutil.copyfile(db_fixture_file, test_db_file) # create the test user home directory self.test_user_dir = ensure_dirs_exist(test_user_dir) - temppath(self.test_user_dir, self, skip_output_anchor=True) + temppath(self.test_user_dir, self) self.test_environ = create_http_environ(self.configuration) def assertSingleOutputObject(self, output_objects, with_object_type=None): diff --git a/tests/test_mig_unittest_testcore.py b/tests/test_mig_unittest_testcore.py index a7621812a..b27a74d33 100644 --- a/tests/test_mig_unittest_testcore.py +++ b/tests/test_mig_unittest_testcore.py @@ -38,6 +38,9 @@ class MigUnittestTestcore(MigTestCase): + def _provide_configuration(self): + return 'testconfig' + def test_existing_main(self): def raise_on_error_exit(exit_code, identifying_message=None): if exit_code != 0: @@ -48,7 +51,7 @@ def raise_on_error_exit(exit_code, identifying_message=None): print("") # account for wrapped tests printing to console - testcore_main(_exit=raise_on_error_exit) + testcore_main(self.configuration, _exit=raise_on_error_exit) if __name__ == '__main__': diff --git a/tests/test_mig_wsgi-bin.py b/tests/test_mig_wsgi-bin.py new file mode 100644 index 000000000..4b759cd07 --- /dev/null +++ b/tests/test_mig_wsgi-bin.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""Unit tests for the MiG WSGI glue.""" + +import codecs +from configparser import ConfigParser +import importlib +import os +import stat +import sys + +from tests.support import PY2, MIG_BASE, TEST_DATA_DIR, \ + MigTestCase, testmain, is_path_within +from tests.support.htmlsupp import HtmlAssertMixin +from tests.support.wsgisupp import prepare_wsgi, WsgiAssertMixin +from tests.support.wsgibinsupp import WsgibinInstrumentation, WsgibinAssertMixin + +from mig.shared.base import client_id_dir, client_dir_id, get_short_id, \ + invisible_path, allow_script, brief_list +import mig.shared.returnvalues as returnvalues + +# workaround for files within non-module directories + + +def _import_forcibly(module_name, relative_module_dir=None): + module_path = os.path.join(MIG_BASE, 'mig') + if relative_module_dir is not None: + module_path = os.path.join(module_path, relative_module_dir) + sys.path.append(module_path) + mod = importlib.import_module(module_name) + sys.path.pop(-1) # do not leave the forced module path + return mod + + +migwsgi = _import_forcibly('migwsgi', relative_module_dir='wsgi-bin') + + +def noop(*args): + pass + + +class MigWsgibin(MigTestCase, HtmlAssertMixin, + WsgiAssertMixin, WsgibinAssertMixin): + + def _provide_configuration(self): + return 'testconfig' + + def before_each(self): + self.fake_wsgi = prepare_wsgi(self.configuration, 'http://localhost/') + self.wsgibin_instrumentation = WsgibinInstrumentation() + + self.application_args = ( + self.configuration, + self.fake_wsgi.environ, + self.fake_wsgi.start_response, + ) + self.application_kwargs = dict( + _wrap_wsgi_errors=noop, + _fieldstorage_to_dict=self.wsgibin_instrumentation.fieldstorage_to_dict, + _format_output=self.wsgibin_instrumentation.format_output, + _retrieve_handler=self.wsgibin_instrumentation.retrieve_handler, + _set_environ=noop, + ) + + def test_return_value_ok_returns_status_200(self): + self.wsgibin_instrumentation.set_response([], returnvalues.OK) + + wsgi_result = migwsgi._application( + *self.application_args, + **self.application_kwargs + ) + + self.assertWsgiResponse(wsgi_result, self.fake_wsgi, 200) + self.assertWsgibinInstrumentation() + + def test_return_value_ok_returns_valid_html_page(self): + self.wsgibin_instrumentation.set_response([], returnvalues.OK) + + wsgi_result = migwsgi._application( + *self.application_args, + **self.application_kwargs + ) + + output, _ = self.assertWsgiResponse(wsgi_result, self.fake_wsgi, 200) + self.assertWsgibinInstrumentation() + self.assertIsValidHtmlDocument(output) + + def test_return_value_ok_returns_expected_title(self): + self.wsgibin_instrumentation.set_response([], returnvalues.OK) + self.wsgibin_instrumentation.format_output.set_values( + title_text='TEST') + + wsgi_result = migwsgi._application( + *self.application_args, + **self.application_kwargs + ) + + output, _ = self.assertWsgiResponse(wsgi_result, self.fake_wsgi, 200) + self.assertWsgibinInstrumentation() + self.assertHtmlElementTextContent( + output, 'title', 'TEST', trim_newlines=True) + + def test_return_value_ok_serving_a_binary_file(self): + test_binary_file = os.path.join(TEST_DATA_DIR, 'loading.gif') + with open(test_binary_file, 'rb') as f: + test_binary_data = f.read() + self.wsgibin_instrumentation.set_response( + test_binary_data, returnvalues.OK, responding_with='file') + + wsgi_result = migwsgi._application( + *self.application_args, + **self.application_kwargs + ) + + output, _ = self.assertWsgiResponse( + wsgi_result, self.fake_wsgi, 200, content_kind='binary') + self.assertWsgibinInstrumentation() + self.assertEqual(output, test_binary_data) + + +if __name__ == '__main__': + testmain()