Basic coverage of migwsgi.

This PR does the principle things required to allow exercising the
central component responsible for glueing named MiG "functionality"
files to WSGI and have the result execute to completion under Python 3.

Included is a small tactical change to allow the structural force
recursive functions to iterate tuples preserving their type such that
subsequence output behaves correctly under Py3.

=======

wip

fixup

fixup

updare and relocate a comment

start tightening up the code

shift things around a little

work to make it readable with a nod towards further tests

assert the response status

allow programming the response

repair previous

assert that a programmed title ends up in the page

line naming up with other recent work in grid_openid

fixup

fixup

fixup

fixup

split the testing infrastructure across multiple files

collect common default kwargs

use noop for set environ

make the generic WSGI handling setup code more uniform

bring over improvements to hmtlsupp from another branch

simplify

fixup

Author: albu-diku

mig/shared/base.py

@@ -542,8 +542,9 @@ def force_utf8_rec(input_obj, highlight=''):
     if isinstance(input_obj, dict):
         return {force_utf8_rec(i, highlight): force_utf8_rec(j, highlight) for (i, j) in
                 input_obj.items()}
-    elif isinstance(input_obj, list):
-        return [force_utf8_rec(i, highlight) for i in input_obj]
+    elif isinstance(input_obj, (list, tuple)):
+        thetype = type(input_obj)
+        return thetype(force_utf8_rec(i, highlight) for i in input_obj)
     elif is_unicode(input_obj):
         return force_utf8(input_obj, highlight)
     else:
@@ -570,8 +571,9 @@ def force_unicode_rec(input_obj, highlight=''):
     if isinstance(input_obj, dict):
         return {force_unicode_rec(i, highlight): force_unicode_rec(j, highlight) for (i, j) in
                 input_obj.items()}
-    elif isinstance(input_obj, list):
-        return [force_unicode_rec(i, highlight) for i in input_obj]
+    elif isinstance(input_obj, (list, tuple)):
+        thetype = type(input_obj)
+        return thetype(force_utf8_rec(i, highlight) for i in input_obj)
     elif not is_unicode(input_obj):
         return force_unicode(input_obj, highlight)
     else:

mig/wsgi-bin/migwsgi.py

@@ -34,7 +34,7 @@
 from mig.shared import returnvalues
 from mig.shared.bailout import bailout_helper, crash_helper, compact_string
 from mig.shared.base import requested_backend, allow_script, \
-    is_default_str_coding, force_default_str_coding_rec
+    is_default_str_coding, force_default_str_coding_rec, force_utf8
 from mig.shared.defaults import download_block_size, default_fs_coding
 from mig.shared.conf import get_configuration_object
 from mig.shared.objecttypes import get_object_type_info
@@ -43,14 +43,19 @@
 from mig.shared.scriptinput import fieldstorage_to_dict
 
 
+def _import_backend(backend):
+    import_path = 'mig.shared.functionality.%s' % backend
+    module_handle = importlib.import_module(import_path)
+    return module_handle.main
+
+
 def object_type_info(object_type):
     """Lookup object type"""
 
     return get_object_type_info(object_type)
 
 
-def stub(configuration, client_id, import_path, backend, user_arguments_dict,
-         environ):
+def stub(configuration, client_id, user_arguments_dict, environ, _retrieve_handler):
     """Run backend on behalf of client_id with supplied user_arguments_dict.
     I.e. import main from import_path and execute it with supplied arguments.
     """
@@ -61,6 +66,7 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict,
     before_time = time.time()
 
     output_objects = []
+    backend = 'UNKNOWN'
     main = dummy_main
 
     # _logger.debug("stub for backend %r" % backend)
@@ -69,10 +75,12 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict,
     #            NEVER print/output it verbatim before it is validated below.
 
     try:
+        default_page = configuration.site_landing_page # TODO: avoid doing this work a second time
+        backend = requested_backend(environ, fallback=default_page)
         valid_backend_name(backend)
     except InputException as iex:
-        _logger.error("%s refused to import invalid backend %r (%s): %s" %
-                      (_addr, backend, import_path, iex))
+        _logger.error("%s refused to import invalid backend %r: %s" %
+                      (_addr, backend, iex))
         bailout_helper(configuration, backend, output_objects,
                        header_text='User Error')
         output_objects.extend([
@@ -81,41 +89,40 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict,
             {'object_type': 'link', 'text': 'Go to default interface',
              'destination': configuration.site_landing_page}
         ])
-        return (output_objects, returnvalues.CLIENT_ERROR)
+        return backend, (output_objects, returnvalues.CLIENT_ERROR)
 
     try:
         # Import main from backend module
 
         # _logger.debug("import main from %r" % import_path)
         # NOTE: dynamic module loading to find corresponding main function
-        module_handle = importlib.import_module(import_path)
-        main = module_handle.main
+        main = _retrieve_handler(backend)
     except Exception as err:
-        _logger.error("%s could not import %r (%s): %s" %
-                      (_addr, backend, import_path, err))
+        _logger.error("%s could not import %r: %s" %
+                      (_addr, backend, err))
         bailout_helper(configuration, backend, output_objects)
         output_objects.extend([
             {'object_type': 'error_text', 'text':
              'Could not load backend: %s' % html_escape(backend)},
             {'object_type': 'link', 'text': 'Go to default interface',
              'destination': configuration.site_landing_page}
         ])
-        return (output_objects, returnvalues.SYSTEM_ERROR)
+        return backend, (output_objects, returnvalues.SYSTEM_ERROR)
 
     # _logger.debug("imported main %s" % main)
 
     # Now backend value is validated to be safe for output
 
     if not isinstance(user_arguments_dict, dict):
-        _logger.error("%s invalid user args %s for %s" % (_addr,
+        _logger.error("%s invalid user args %s for backend %r" % (_addr,
                                                           user_arguments_dict,
-                                                          import_path))
+                                                          backend))
         bailout_helper(configuration, backend, output_objects,
                        header_text='Input Error')
         output_objects.append(
             {'object_type': 'error_text', 'text':
              'User input is not on expected format!'})
-        return (output_objects, returnvalues.INVALID_ARGUMENT)
+        return backend, (output_objects, returnvalues.INVALID_ARGUMENT)
 
     try:
         (output_objects, (ret_code, ret_msg)) = main(client_id,
@@ -125,7 +132,7 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict,
         _logger.error("%s script crashed:\n%s" % (_addr,
                                                   traceback.format_exc()))
         crash_helper(configuration, backend, output_objects)
-        return (output_objects, returnvalues.ERROR)
+        return backend, (output_objects, returnvalues.ERROR)
 
     (val_ret, val_msg) = validate(output_objects)
     if not val_ret:
@@ -138,7 +145,7 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict,
     after_time = time.time()
     output_objects.append({'object_type': 'timing_info', 'text':
                            "done in %.3fs" % (after_time - before_time)})
-    return (output_objects, (ret_code, ret_msg))
+    return backend, (output_objects, (ret_code, ret_msg))
 
 
 def wrap_wsgi_errors(environ, configuration, max_line_len=100):
@@ -193,6 +200,14 @@ def application(environ, start_response):
     *start_response* is a helper function used to deliver the client response.
     """
 
+    def _set_os_environ(value):
+        os.environ = value
+
+    return _application(None, environ, start_response, _set_environ=_set_os_environ, _wrap_wsgi_errors=wrap_wsgi_errors)
+
+
+def _application(configuration, environ, start_response, _set_environ, _format_output=format_output, _retrieve_handler=_import_backend, _wrap_wsgi_errors=True, _config_file=None, _skip_log=False):
+
     # NOTE: pass app environ including apache and query args on to sub handlers
     #       through the usual 'os.environ' channel expected in functionality
     #       handlers. Special care is needed to avoid various sub-interpreter
@@ -235,18 +250,20 @@ def application(environ, start_response):
                                                              os_env_value))
 
     # Assign updated environ to LOCAL os.environ for the rest of this session
-    os.environ = environ
+    _set_environ(environ)
 
     # NOTE: redirect stdout to stderr in python 2 only. It breaks logger in 3
     #       and stdout redirection apparently is already handled there.
     if sys.version_info[0] < 3:
         sys.stdout = sys.stderr
 
-    configuration = get_configuration_object()
+    if configuration is None:
+        configuration = get_configuration_object(_config_file, _skip_log)
+
     _logger = configuration.logger
 
     # NOTE: replace default wsgi errors to apache error log with our own logs
-    wrap_wsgi_errors(environ, configuration)
+    _wrap_wsgi_errors(environ, configuration)
 
     for line in env_sync_status:
         _logger.debug(line)
@@ -298,7 +315,6 @@ def application(environ, start_response):
         default_page = configuration.site_landing_page
         script_name = requested_backend(environ, fallback=default_page,
                                         strip_ext=False)
-        backend = requested_backend(environ, fallback=default_page)
         # _logger.debug('DEBUG: wsgi found backend %s and script %s' %
         #              (backend, script_name))
         fieldstorage = cgi.FieldStorage(fp=environ['wsgi.input'],
@@ -307,13 +323,12 @@ def application(environ, start_response):
         if 'output_format' in user_arguments_dict:
             output_format = user_arguments_dict['output_format'][0]
 
-        module_path = 'mig.shared.functionality.%s' % backend
         (allow, msg) = allow_script(configuration, script_name, client_id)
         if allow:
             # _logger.debug("wsgi handling script: %s" % script_name)
-            (output_objs, ret_val) = stub(configuration, client_id,
-                                          module_path, backend,
-                                          user_arguments_dict, environ)
+            backend, (output_objs, ret_val) = stub(configuration, client_id,
+                                          user_arguments_dict, environ,
+                                          _retrieve_handler)
         else:
             _logger.warning("wsgi handling refused script:%s" % script_name)
             (output_objs, ret_val) = reject_main(client_id,
@@ -363,7 +378,7 @@ def application(environ, start_response):
     output_objs.append(wsgi_entry)
 
     _logger.debug("call format %r output to %s" % (backend, output_format))
-    output = format_output(configuration, backend, ret_code, ret_msg,
+    output = _format_output(configuration, backend, ret_code, ret_msg,
                            output_objs, output_format)
     # _logger.debug("formatted %s output to %s" % (backend, output_format))
     # _logger.debug("output:\n%s" % [output])
@@ -372,7 +387,7 @@ def application(environ, start_response):
         _logger.error(
             "Formatted output is NOT on default str coding: %s" % [output[:100]])
         err_mark = '__****__'
-        output = format_output(configuration, backend, ret_code, ret_msg,
+        output = _format_output(configuration, backend, ret_code, ret_msg,
                                force_default_str_coding_rec(
                                    output_objs, highlight=err_mark),
                                output_format)
@@ -396,7 +411,14 @@ def application(environ, start_response):
     # NOTE: send response to client but don't crash e.g. on closed connection
     try:
         start_response(status, response_headers)
+    except IOError as ioe:
+        _logger.warning("WSGI %s for %s could not deliver output: %s" %
+                        (backend, client_id, ioe))
+    except Exception as exc:
+        _logger.error("WSGI %s for %s crashed during response: %s" %
+                      (backend, client_id, exc))
 
+    try:
         # NOTE: we consistently hit download error for archive files reaching ~2GB
         #       with showfreezefile.py on wsgi but the same on cgi does NOT suffer
         #       the problem for the exact same files. It seems wsgi has a limited
@@ -410,12 +432,12 @@ def application(environ, start_response):
             _logger.info("WSGI %s yielding %d output parts (%db)" %
                          (backend, chunk_parts, content_length))
         # _logger.debug("send chunked %r response to client" % backend)
-        for i in xrange(chunk_parts):
+        for i in list(range(chunk_parts)):
             # _logger.debug("WSGI %s yielding part %d / %d output parts" %
             #              (backend, i+1, chunk_parts))
             # end index may be after end of content - but no problem
             part = output[i*download_block_size:(i+1)*download_block_size]
-            yield part
+            yield force_utf8(part)
         if chunk_parts > 1:
             _logger.info("WSGI %s finished yielding all %d output parts" %
                          (backend, chunk_parts))

tests/support/htmlsupp.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# htmlsupp - test support library for HTML
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# -- END_HEADER ---
+#
+
+"""Test support library for HTML."""
+
+
+class HtmlAssertMixin:
+    """Custom assertions for HTML containing strings."""
+
+    def assertHtmlElement(self, value, tag_name):
+        """Check that an occurrence of the specifid tag within an HTML input
+        string can be found. Returns the textual content of the first match.
+        """
+
+        self.assertIsValidHtmlDocument(value)
+
+        # TODO: this is a definitively stop-gap way of finding a tag within the HTML
+        #       and is used purely to keep this initial change to a reasonable size.
+
+        tag_open = ''.join(['<', tag_name, '>'])
+        tag_open_index = value.index(tag_open)
+        tag_open_index_after = tag_open_index + len(tag_open)
+
+        tag_close = ''.join(['</', tag_name, '>'])
+        tag_close_index = value.index(tag_close, tag_open_index_after)
+
+        return value[tag_open_index_after:tag_close_index]
+
+    def assertHtmlElementTextContent(self, value, tag_name, expected_text, trim_newlines=True):
+        """Check there is an occurrence of a tag within an HTML input string
+        and check the text it encloses equals exactly the expecatation.
+        """
+
+        self.assertIsValidHtmlDocument(value)
+
+        # TODO: this is a definitively stop-gap way of finding a tag within the HTML
+        #       and is used purely to keep this initial change to a reasonable size.
+
+        actual_text = self.assertHtmlElement(value, tag_name)
+        if trim_newlines:
+            actual_text = actual_text.strip('\n')
+        self.assertEqual(actual_text, expected_text)
+
+    def assertIsValidHtmlDocument(self, value):
+        """Check that the input string contains a valid HTML document.
+        """
+
+        assert isinstance(value, type(u"")), "input string was not utf8"
+
+        error = None
+        try:
+            has_doctype = value.startswith("<!DOCTYPE html")
+            assert has_doctype, "no valid document opener"
+            end_html_tag_idx = value.rfind('</html>')
+            maybe_document_end = value[end_html_tag_idx:].rstrip()
+            assert maybe_document_end == '</html>', "no valid document closer"
+        except Exception as exc:
+            error = exc
+        if error:
+            raise AssertionError("failed to verify input string as HTML: %s", str(error))