diff --git a/.gitignore b/.gitignore index 81d3cfafe..3cb78bc1d 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,10 @@ dist/ downloads/ eggs/ .eggs/ -lib/ -lib64/ +/lib/ +usr/lib/ +/lib64/ +usr/lib64/ parts/ sdist/ var/ diff --git a/mig/lib/README b/mig/lib/README new file mode 100644 index 000000000..caa680a21 --- /dev/null +++ b/mig/lib/README @@ -0,0 +1,10 @@ += Modernization and Clean Up = +We will gradually move code here in the on-going modernization and clean up +efforts. +That also means that any code placed here MUST comply with the project style +guides, be lint clean, documented and have decent unit test coverage. + +You may want to use autopep8, pylint, ruff or any available make lint targets +to help verify. +The black code formatter and isort may also come in handy. You can see usage +hints in `.github/workflows/python-stylecheck.yml`. diff --git a/mig/lib/xgicore.py b/mig/lib/xgicore.py new file mode 100644 index 000000000..79ac5d895 --- /dev/null +++ b/mig/lib/xgicore.py @@ -0,0 +1,69 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# xgicore - Xgi wrapper functions for functionality backends +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +"""Shared helpers for CGI+WSGI interface to functionality backends.""" + + +def get_output_format(configuration, user_args, default_format="html"): + """Get output_format from user_args.""" + return user_args.get("output_format", [default_format])[0] + + +def override_output_format(configuration, user_args, out_objs, out_format): + """Override output_format if requested in start entry of output_objs.""" + if not [ + i + for i in out_objs + if i.get("object_type", None) == "start" + and i.get("override_format", False) + ]: + return out_format + return get_output_format(configuration, user_args) + + +def fill_start_headers(configuration, out_objs, out_format): + """Make sure out_objs has start entry with basic content headers.""" + start_entry = None + for entry in out_objs: + if entry["object_type"] == "start": + start_entry = entry + if not start_entry: + start_entry = {"object_type": "start", "headers": []} + out_objs.insert(0, start_entry) + elif not start_entry.get("headers", False): + start_entry["headers"] = [] + # Now fill headers to match output format + default_content = "text/html" + if "json" == out_format: + default_content = "application/json" + elif "file" == out_format: + default_content = "application/octet-stream" + elif "html" != out_format: + default_content = "text/plain" + if not start_entry["headers"]: + start_entry["headers"].append(("Content-Type", default_content)) + return start_entry diff --git a/mig/shared/cgiscriptstub.py b/mig/shared/cgiscriptstub.py index 7ed6b4176..84c231ed1 100755 --- a/mig/shared/cgiscriptstub.py +++ b/mig/shared/cgiscriptstub.py @@ -4,7 +4,7 @@ # --- BEGIN_HEADER --- # # cgiscriptstub - cgi wrapper functions for functionality backends -# Copyright (C) 2003-2024 The MiG Project lead by Brian Vinter +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. # @@ -42,6 +42,8 @@ except: pass +from mig.lib.xgicore import fill_start_headers, get_output_format, \ + override_output_format from mig.shared.bailout import crash_helper from mig.shared.base import requested_backend, allow_script, \ is_default_str_coding, force_default_str_coding_rec @@ -79,23 +81,7 @@ def finish_cgi_script(configuration, backend, output_format, ret_code, ret_msg, """Shared finalization""" logger = configuration.logger - default_content = 'text/html' - if 'json' == output_format: - default_content = 'application/json' - elif 'file' == output_format: - default_content = 'application/octet-stream' - elif 'html' != output_format: - default_content = 'text/plain' - default_headers = [('Content-Type', default_content)] - start_entry = None - for entry in output_objs: - if entry['object_type'] == 'start': - start_entry = entry - if not start_entry: - start_entry = {'object_type': 'start', 'headers': default_headers} - output_objs = [start_entry] + output_objs - elif not start_entry.get('headers', []): - start_entry['headers'] = default_headers + start_entry = fill_start_headers(configuration, output_objs, output_format) headers = start_entry['headers'] output = format_output(configuration, backend, ret_code, ret_msg, @@ -166,9 +152,7 @@ def run_cgi_script_possibly_with_cert(main, delayed_input=None, logger.debug("handling cgi request with python %s from %s (%s)" % (sys.version_info, client_id, environ)) - # default to html output - - output_format = user_arguments_dict.get('output_format', ['html'])[-1] + output_format = get_output_format(configuration, user_arguments_dict) # TODO: add environ arg support to all main backends and use here @@ -191,8 +175,14 @@ def run_cgi_script_possibly_with_cert(main, delayed_input=None, after_time = time.time() out_obj.append({'object_type': 'timing_info', 'text': "done in %.3fs" % (after_time - before_time)}) + + # TODO: drop delay_format and rely on shared override_format marker instead if delay_format: - output_format = user_arguments_dict.get('output_format', ['html'])[-1] + output_format = get_output_format(configuration, user_arguments_dict) + + # NOTE: optional output_format override if backend requests it in start + output_format = override_output_format(configuration, user_arguments_dict, + out_obj, output_format) finish_cgi_script(configuration, backend, output_format, ret_code, ret_msg, out_obj) diff --git a/mig/shared/functionality/showvgridprivatefile.py b/mig/shared/functionality/showvgridprivatefile.py index 00a2d80f2..2cd6e8200 100644 --- a/mig/shared/functionality/showvgridprivatefile.py +++ b/mig/shared/functionality/showvgridprivatefile.py @@ -116,6 +116,9 @@ def main(client_id, user_arguments_dict): if force_file: content = read_file(abs_path, logger, mode=src_mode) lines = [content] + # Force delivery of binary as file download + user_arguments_dict['output_format'] = ['file'] + start_entry['override_format'] = True else: content = lines = read_file_lines(abs_path, logger, mode=src_mode) diff --git a/mig/wsgi-bin/migwsgi.py b/mig/wsgi-bin/migwsgi.py index e0937ab07..073958880 100755 --- a/mig/wsgi-bin/migwsgi.py +++ b/mig/wsgi-bin/migwsgi.py @@ -4,7 +4,7 @@ # --- BEGIN_HEADER --- # # migwsgi.py - Provides the entire WSGI interface -# Copyright (C) 2003-2024 The MiG Project lead by Brian Vinter +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH # # This file is part of MiG. # @@ -34,6 +34,8 @@ import sys import time +from mig.lib.xgicore import fill_start_headers, get_output_format, \ + override_output_format from mig.shared import returnvalues from mig.shared.bailout import bailout_helper, crash_helper, compact_string from mig.shared.base import requested_backend, allow_script, \ @@ -271,11 +273,7 @@ def application(environ, start_response, configuration=None, from mig.shared.httpsclient import extract_client_id client_id = extract_client_id(configuration, environ) - # Default to html output - - default_content = 'text/html' - output_format = 'html' - + output_format = "UNSET" backend = "UNKNOWN" output_objs = [] fieldstorage = None @@ -316,8 +314,7 @@ def application(environ, start_response, configuration=None, fieldstorage = cgi.FieldStorage(fp=environ['wsgi.input'], environ=environ) user_arguments_dict = fieldstorage_to_dict(fieldstorage) - if 'output_format' in user_arguments_dict: - output_format = user_arguments_dict['output_format'][0] + output_format = get_output_format(configuration, user_arguments_dict) module_path = 'mig.shared.functionality.%s' % backend (allow, msg) = allow_script(configuration, script_name, client_id) @@ -346,24 +343,10 @@ def application(environ, start_response, configuration=None, (ret_code, ret_msg) = ret_val - if 'json' == output_format: - default_content = 'application/json' - elif 'file' == output_format: - default_content = 'application/octet-stream' - elif 'html' != output_format: - default_content = 'text/plain' - default_headers = [('Content-Type', default_content)] - start_entry = None - for entry in output_objs: - if entry['object_type'] == 'start': - start_entry = entry - if not start_entry: - # _logger.debug("WSGI adding explicit headers: %s" % default_headers) - start_entry = {'object_type': 'start', 'headers': default_headers} - output_objs = [start_entry] + output_objs - elif not start_entry.get('headers', []): - # _logger.debug("WSGI adding missing headers: %s" % default_headers) - start_entry['headers'] = default_headers + # NOTE: optional output_format override if backend requests it in start + output_format = override_output_format(configuration, user_arguments_dict, + output_objs, output_format) + start_entry = fill_start_headers(configuration, output_objs, output_format) response_headers = start_entry['headers'] # Pass wsgi info and helpers for optional use in output delivery diff --git a/tests/test_mig_lib_xgicore.py b/tests/test_mig_lib_xgicore.py new file mode 100644 index 000000000..e63e22b7d --- /dev/null +++ b/tests/test_mig_lib_xgicore.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# test_mig_lib_xgicore - unit test of the corresponding mig lib module +# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""Unit test xgicore functions""" + +import os +import sys + +from tests.support import MigTestCase, FakeConfiguration, testmain + +from mig.lib.xgicore import * + + +class MigLibXgicore__get_output_format(MigTestCase): + """Unit test get_output_format""" + + def test_default_when_missing(self): + """Test that default output_format is returned when not set.""" + expected = "html" + user_args = {} + actual = get_output_format(FakeConfiguration(), user_args, + default_format=expected) + self.assertEqual(actual, expected, + "mismatch in default output_format") + + def test_get_single_requested_format(self): + """Test that the requested output_format is returned.""" + expected = "file" + user_args = {'output_format': [expected]} + actual = get_output_format(FakeConfiguration(), user_args, + default_format='BOGUS') + self.assertEqual(actual, expected, + "mismatch in extracted output_format") + + def test_get_first_requested_format(self): + """Test that first requested output_format is returned.""" + expected = "file" + user_args = {'output_format': [expected, 'BOGUS']} + actual = get_output_format(FakeConfiguration(), user_args, + default_format='BOGUS') + self.assertEqual(actual, expected, + "mismatch in extracted output_format") + + +class MigLibXgicore__override_output_format(MigTestCase): + """Unit test override_output_format""" + + def test_unchanged_without_override(self): + """Test that existing output_format is returned when not overriden.""" + expected = "html" + user_args = {} + out_objs = [] + actual = override_output_format(FakeConfiguration(), user_args, + out_objs, expected) + self.assertEqual(actual, expected, + "mismatch in unchanged output_format") + + def test_get_single_requested_format(self): + """Test that the requested output_format is returned if overriden.""" + expected = "file" + user_args = {'output_format': [expected]} + out_objs = [{'object_type': 'start', 'override_format': True}] + actual = override_output_format(FakeConfiguration(), user_args, + out_objs, 'OVERRIDE') + self.assertEqual(actual, expected, + "mismatch in overriden output_format") + + def test_get_first_requested_format(self): + """Test that first requested output_format is returned if overriden.""" + expected = "file" + user_args = {'output_format': [expected, 'BOGUS']} + actual = get_output_format(FakeConfiguration(), user_args, + default_format='BOGUS') + self.assertEqual(actual, expected, + "mismatch in extracted output_format") + + +class MigLibXgicore__fill_start_headers(MigTestCase): + """Unit test fill_start_headers""" + + def test_unchanged_when_set(self): + """Test that existing valid start entry is returned as-is.""" + out_format = "file" + headers = [('Content-Type', 'application/octet-stream'), + ('Content-Size', 42)] + expected = {'object_type': 'start', 'headers': headers} + out_objs = [expected, {'object_type': 'binary', 'data': 42*b'0'}] + actual = fill_start_headers(FakeConfiguration(), out_objs, out_format) + self.assertEqual(actual, expected, + "mismatch in unchanged start entry") + + def test_headers_added_when_missing(self): + """Test that start entry headers are added if missing.""" + out_format = "file" + headers = [('Content-Type', 'application/octet-stream')] + minimal_start = {'object_type': 'start'} + expected = {'object_type': 'start', 'headers': headers} + out_objs = [minimal_start, {'object_type': 'binary', 'data': 42*b'0'}] + actual = fill_start_headers(FakeConfiguration(), out_objs, out_format) + self.assertEqual(actual, expected, + "mismatch in auto initialized start entry") + + def test_start_added_when_missing(self): + """Test that start entry is added if missing.""" + out_format = "file" + headers = [('Content-Type', 'application/octet-stream')] + expected = {'object_type': 'start', 'headers': headers} + out_objs = [{'object_type': 'binary', 'data': 42*b'0'}] + actual = fill_start_headers(FakeConfiguration(), out_objs, out_format) + self.assertEqual(actual, expected, + "mismatch in auto initialized start entry") + + +if __name__ == '__main__': + testmain()