diff --git a/.gitignore b/.gitignore index 81d3cfafe..dea8c0347 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ # Byte-compiled / optimized / DLL files +__jinja__/ __pycache__/ *.py[cod] *$py.class diff --git a/local-requirements.txt b/local-requirements.txt index 7faf3b026..9edc94117 100644 --- a/local-requirements.txt +++ b/local-requirements.txt @@ -7,4 +7,5 @@ autopep8;python_version >= "3" # NOTE: paramiko-3.0.0 dropped python2 and python3.6 support paramiko;python_version >= "3.7" paramiko<3;python_version < "3.7" +polib werkzeug diff --git a/mig/assets/templates/.gitkeep b/mig/assets/templates/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/mig/lib/__init__.py b/mig/lib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mig/lib/templates/__init__.py b/mig/lib/templates/__init__.py new file mode 100644 index 000000000..a46934048 --- /dev/null +++ b/mig/lib/templates/__init__.py @@ -0,0 +1,169 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# base - shared base helper functions +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +import errno +from jinja2 import meta as jinja2_meta, select_autoescape, Environment, \ + FileSystemLoader, FileSystemBytecodeCache +import os +import weakref + +from mig.shared.compat import PY2 +from mig.shared.defaults import MIG_BASE + + +if PY2: + from chainmap import ChainMap +else: + from collections import ChainMap + +TEMPLATES_DIR = os.path.join(MIG_BASE, 'mig/assets/templates') +TEMPLATES_CACHE_DIR = os.path.join(TEMPLATES_DIR, '__jinja__') + +_all_template_dirs = [ + TEMPLATES_DIR, +] + + +def _clear_global_store(): + global _global_store + _global_store = None + + +def cache_dir(): + return TEMPLATES_CACHE_DIR + + +def _global_template_dirs(): + return _all_template_dirs + + +class _BonundTemplate: + def __init__(self, template, template_args): + self.tmpl = template + self.args = template_args + + def render(self): + return self.tmpl.render(**self.args) + + +class _FormatContext: + def __init__(self, configuration): + self.output_format = None + self.configuration = configuration + self.script_map = {} + self.style_map = {} + + def __getitem__(self, key): + return self.__dict__[key] + + def __iter__(self): + return iter(self.__dict__) + + def extend(self, template, template_args): + return _BonundTemplate(template, ChainMap(template_args, self)) + + +class TemplateStore: + def __init__(self, template_dirs, cache_dir=None, extra_globals=None): + assert cache_dir is not None + + self._cache_dir = cache_dir + self._template_globals = extra_globals + self._template_environment = Environment( + extensions=['jinja2.ext.i18n'], + loader=FileSystemLoader(template_dirs), + bytecode_cache=FileSystemBytecodeCache(cache_dir, '%s'), + autoescape=select_autoescape() + ) + self._template_environment.install_null_translations() + + @property + def cache_dir(self): + return self._cache_dir + + @property + def context(self): + return self._template_globals + + def _get_template(self, template_fqname): + return self._template_environment.get_template(template_fqname) + + def _get_template_source(self, template_fqname): + template = self._template_environment.get_template(template_fqname) + with open(template.filename) as f: + return f.read() + + def grab_template(self, template_name, template_group, output_format, template_globals=None, **kwargs): + template_fqname = "%s_%s.%s.jinja" % ( + template_group, template_name, output_format) + return self._template_environment.get_template(template_fqname, globals=template_globals) + + def list_templates(self): + return [t for t in self._template_environment.list_templates() if t.endswith('.jinja')] + + def extract_translations(self, template_fqname): + template_source = self._get_template_source(template_fqname) + return self._template_environment.extract_translations(template_source) + + def extract_variables(self, template_fqname): + template_source = self._get_template_source(template_fqname) + ast = self._template_environment.parse(template_source) + return jinja2_meta.find_undeclared_variables(ast) + + @staticmethod + def populated(template_dirs, cache_dir=None, context=None): + assert cache_dir is not None + + try: + os.mkdir(cache_dir) + except OSError as direxc: + if direxc.errno != errno.EEXIST: # FileExistsError + raise + + store = TemplateStore( + template_dirs, cache_dir=cache_dir, extra_globals=context) + + for template_fqname in store.list_templates(): + store._get_template(template_fqname) + + return store + + +def init_global_templates(configuration, _templates_dirs=_global_template_dirs): + _context = configuration.context() + + try: + return configuration.context(namespace='templates') + except KeyError as exc: + pass + + store = TemplateStore.populated( + _templates_dirs(), + cache_dir=cache_dir(), + context=_FormatContext(configuration) + ) + return configuration.context_set(store, namespace='templates') diff --git a/mig/lib/templates/__main__.py b/mig/lib/templates/__main__.py new file mode 100644 index 000000000..3da8880b5 --- /dev/null +++ b/mig/lib/templates/__main__.py @@ -0,0 +1,120 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# base - shared base helper functions +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +from __future__ import print_function +import os +import sys + +from mig.lib.templates import init_global_templates, _global_template_dirs +from mig.shared.compat import SimpleNamespace +from mig.shared.conf import get_configuration_object + + +def warn(message): + print(message, sys.stderr, True) + + +def main(args, _print=print): + configuration = get_configuration_object( + config_file=args.config_file, skip_log=True, disable_auth_log=True) + + if args.tmpldir: + def _template_dirs(): return [args.tmpldir] + else: + _template_dirs = _global_template_dirs + + template_store = init_global_templates( + configuration, _templates_dirs=_template_dirs) + + command = args.command + if command == 'show': + print(template_store.list_templates()) + elif command == 'prime': + try: + os.mkdir(template_store.cache_dir) + except FileExistsError: + pass + + for template_fqname in template_store.list_templates(): + template_store._get_template(template_fqname) + elif command == 'translations': + try: + os.mkdir(template_store.cache_dir) + except FileExistsError: + pass + + for template_fqname in template_store.list_templates(): + extracted_tuples = list( + template_store.extract_translations(template_fqname)) + if len(extracted_tuples) == 0: + continue + _print("<%s>" % (template_fqname,)) + for _, __, string in template_store.extract_translations(template_fqname): + _print(' "%s"' % (string,)) + _print("" % (template_fqname,)) + + + elif command == 'translations-mo': + try: + os.mkdir(template_store.cache_dir) + except FileExistsError: + pass + + import polib + pofile = polib.POFile() + for template_fqname in template_store.list_templates(): + for _, __, string in template_store.extract_translations(template_fqname): + poentry = polib.POEntry(msgid=string, msgstr=string) + pofile.append(poentry) + + _print(pofile) + + elif command == 'vars': + for template_ref in template_store.list_templates(): + _print("<%s>" % (template_ref,)) + for var in template_store.extract_variables(template_ref): + _print(" {{%s}}" % (var,)) + _print("" % (template_ref,)) + else: + raise RuntimeError("unknown command: %s" % (command,)) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument('-c', dest='config_file', default=None) + parser.add_argument('--template-dir', dest='tmpldir') + parser.add_argument('command') + args = parser.parse_args() + + try: + main(args) + sys.exit(0) + except Exception as exc: + warn(str(exc)) + sys.exit(1) diff --git a/mig/shared/configuration.py b/mig/shared/configuration.py index 5dd5b145b..709f2988f 100644 --- a/mig/shared/configuration.py +++ b/mig/shared/configuration.py @@ -716,6 +716,7 @@ def __init__(self, config_file, verbose=False, skip_log=False, disable_auth_log=False): self.config_file = config_file self.mig_server_id = None + self._context = None configuration_options = copy.deepcopy(_CONFIGURATION_DEFAULTS) @@ -727,6 +728,23 @@ def __init__(self, config_file, verbose=False, skip_log=False, disable_auth_log=disable_auth_log, _config_file=config_file) + def context(self, namespace=None): + if self._context is None: + self._context = {} + if namespace is None: + return self._context + try: + return self._context[namespace] + except KeyError: + raise + + def context_set(self, value, namespace=None): + assert namespace is not None + + context = self.context() + context[namespace] = value + return value + def reload_config(self, verbose, skip_log=False, disable_auth_log=False, _config_file=None): """Re-read and parse configuration file. Optional skip_log arg diff --git a/mig/shared/objecttypes.py b/mig/shared/objecttypes.py index 64b21303f..ccea24883 100644 --- a/mig/shared/objecttypes.py +++ b/mig/shared/objecttypes.py @@ -28,9 +28,13 @@ """ Defines valid objecttypes and provides a method to verify if an object is correct """ +from mig.lib.templates import init_global_templates + + start = {'object_type': 'start', 'required': [], 'optional': ['headers' ]} end = {'object_type': 'end', 'required': [], 'optional': []} +template = {'object_type': 'template'} timing_info = {'object_type': 'timing_info', 'required': [], 'optional': []} title = {'object_type': 'title', 'required': ['text'], @@ -396,6 +400,7 @@ valid_types_list = [ start, end, + template, timing_info, title, text, @@ -499,6 +504,8 @@ image_settings_list, ] +base_template_required = set(('template_name', 'template_group', 'template_args,')) + # valid_types_dict = {"title":title, "link":link, "header":header} # autogenerate dict based on list. Dictionary access is prefered to allow @@ -539,8 +546,8 @@ def get_object_type_info(object_type_list): return out -def validate(input_object): - """ validate input_object """ +def validate(input_object, configuration=None): + """ validate presented objects against their definitions """ if not type(input_object) == type([]): return (False, 'validate object must be a list' % ()) @@ -560,6 +567,19 @@ def validate(input_object): this_object_type = obj['object_type'] valid_object_type = valid_types_dict[this_object_type] + + if this_object_type == 'template': + # the required keys stuff below is not applicable to templates + # because templates know what they need in terms of data thus + # are self-documenting - use this fact to perform validation + #template_ref = "%s_%s.html" % (obj['template_group'], ) + store = init_global_templates(configuration) + template = store.grab_template(obj['template_name'], obj['template_group'], 'html') + valid_object_type = { + 'required': store.extract_variables(template) + } + obj = obj.get('template_args', None) + if 'required' in valid_object_type: for req in valid_object_type['required']: if req not in obj: diff --git a/mig/shared/output.py b/mig/shared/output.py index f0490aa39..3de2c75da 100644 --- a/mig/shared/output.py +++ b/mig/shared/output.py @@ -43,6 +43,7 @@ import time import traceback +from mig.lib.templates import init_global_templates from mig.shared import returnvalues from mig.shared.bailout import bailout_title, crash_helper, \ filter_output_objects @@ -746,6 +747,15 @@ def html_format(configuration, ret_val, ret_msg, out_obj): for i in out_obj: if i['object_type'] == 'start': pass + elif i['object_type'] == 'template': + store = init_global_templates(configuration) + template = store.grab_template( + i['template_name'], + i['template_group'], + 'html', + ) + bound = store.context.extend(template, i['template_args']) + lines.append(bound.render()) elif i['object_type'] == 'error_text': msg = "%(text)s" % i if i.get('exc', False): @@ -2821,7 +2831,7 @@ def format_output( logger = configuration.logger #logger.debug("format output to %s" % outputformat) valid_formats = get_valid_outputformats() - (val_ret, val_msg) = validate(out_obj) + (val_ret, val_msg) = validate(out_obj, configuration) if not val_ret: logger.error("%s formatting failed: %s (%s)" % (outputformat, val_msg, val_ret)) diff --git a/mig/wsgi-bin/migwsgi.py b/mig/wsgi-bin/migwsgi.py index e0937ab07..f8df55b2f 100755 --- a/mig/wsgi-bin/migwsgi.py +++ b/mig/wsgi-bin/migwsgi.py @@ -130,7 +130,7 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict, crash_helper(configuration, backend, output_objects) return (output_objects, returnvalues.ERROR) - (val_ret, val_msg) = validate(output_objects) + (val_ret, val_msg) = validate(output_objects, configuration=configuration) if not val_ret: (ret_code, ret_msg) = returnvalues.OUTPUT_VALIDATION_ERROR bailout_helper(configuration, backend, output_objects, diff --git a/requirements.txt b/requirements.txt index 5c2b1bc8f..57ea1829f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,8 @@ # https://pip.pypa.io/en/stable/reference/requirement-specifiers/ future +chainmap;python_version < "3.3" + # cgi was removed from the standard library in Python 3.13 legacy-cgi;python_version >= "3.13" @@ -26,6 +28,10 @@ email-validator;python_version >= "3.7" email-validator<2.0;python_version >= "3" and python_version < "3.7" email-validator<1.3;python_version < "3" +jinja2<3;python_version < "3" +jinja2==3.0.*;python_version >= "3" and python_version < "3.7" +jinja2;python_version >= "3.7" + # NOTE: additional optional dependencies depending on site conf are listed # in recommended.txt and can be installed in the same manner by pointing # pip there. diff --git a/tests/data/templates/test_other.html.jinja b/tests/data/templates/test_other.html.jinja new file mode 100644 index 000000000..f22ed0101 --- /dev/null +++ b/tests/data/templates/test_other.html.jinja @@ -0,0 +1 @@ +{{ other }} diff --git a/tests/data/templates/test_something.html.jinja b/tests/data/templates/test_something.html.jinja new file mode 100644 index 000000000..4c912a491 --- /dev/null +++ b/tests/data/templates/test_something.html.jinja @@ -0,0 +1 @@ + diff --git a/tests/data/templates/test_translated.html.jinja b/tests/data/templates/test_translated.html.jinja new file mode 100644 index 000000000..d6eb4a3af --- /dev/null +++ b/tests/data/templates/test_translated.html.jinja @@ -0,0 +1 @@ +
{% trans article_title %}The title of my article{% endtrans %}
diff --git a/tests/snapshots/test_objects_with_type_template.html b/tests/snapshots/test_objects_with_type_template.html new file mode 100644 index 000000000..8eaf5bde5 --- /dev/null +++ b/tests/snapshots/test_objects_with_type_template.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/test_mig_lib_templates.py b/tests/test_mig_lib_templates.py new file mode 100644 index 000000000..d31c5b26c --- /dev/null +++ b/tests/test_mig_lib_templates.py @@ -0,0 +1,64 @@ +import os +import shutil + +from tests.support import MigTestCase, testmain, \ + MIG_BASE, TEST_DATA_DIR, TEST_OUTPUT_DIR + +from mig.lib.templates import TemplateStore, _global_template_dirs, \ + init_global_templates + + +TEST_CACHE_DIR = os.path.join(TEST_OUTPUT_DIR, '__template_cache__') +TEST_TMPL_DIR = os.path.join(TEST_DATA_DIR, 'templates') +TEST_TMPL_COUNT = len(list(x for x in os.listdir( + TEST_TMPL_DIR) if not x.startswith('.'))) + + +class TestMigSharedTemplates_instance(MigTestCase): + def after_each(self): + shutil.rmtree(TEST_CACHE_DIR, ignore_errors=True) + + def _provide_configuration(self): + return 'testconfig' + + def test_the_creation_of_a_template_store(self): + store = TemplateStore.populated( + TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + self.assertIsInstance(store, TemplateStore) + + def test_a_listing_all_templates(self): + store = TemplateStore.populated( + TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + self.assertEqual(len(store.list_templates()), TEST_TMPL_COUNT) + + def test_grab_template(self): + store = TemplateStore.populated( + TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + template = store.grab_template('other', 'test', 'html') + pass + + def test_variables_for_remplate_ref(self): + store = TemplateStore.populated( + TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + template_vars = store.extract_variables('test_something.html.jinja') + self.assertEqual(template_vars, set(['content'])) + + def test_translated_template(self): + store = TemplateStore.populated( + TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + store.extract_translations('test_translated.html.jinja') + + +class TestMigSharedTemplates_global(MigTestCase): + def _provide_configuration(self): + return 'testconfig' + + def test_cache_location(self): + store = init_global_templates(self.configuration) + + relative_cache_dir = os.path.relpath(store.cache_dir, MIG_BASE) + self.assertEqual(relative_cache_dir, 'mig/assets/templates/__jinja__') + + +if __name__ == '__main__': + testmain() diff --git a/tests/test_mig_shared_configuration.py b/tests/test_mig_shared_configuration.py index e3132756a..81658f4f4 100644 --- a/tests/test_mig_shared_configuration.py +++ b/tests/test_mig_shared_configuration.py @@ -41,7 +41,7 @@ def _is_method(value): def _to_dict(obj): return {k: v for k, v in inspect.getmembers(obj) - if not (k.startswith('__') or _is_method(v))} + if not (k.startswith('_') or _is_method(v))} class MigSharedConfiguration(MigTestCase): diff --git a/tests/test_mig_wsgibin.py b/tests/test_mig_wsgibin.py index 857d03b99..d4be4bc48 100644 --- a/tests/test_mig_wsgibin.py +++ b/tests/test_mig_wsgibin.py @@ -34,7 +34,7 @@ import stat import sys -from tests.support import PY2, MIG_BASE, MigTestCase, testmain, is_path_within +from tests.support import PY2, MIG_BASE, TEST_DATA_DIR, MigTestCase, testmain from tests.support.snapshotsupp import SnapshotAssertMixin from tests.support.wsgisupp import prepare_wsgi, WsgiAssertMixin @@ -49,6 +49,13 @@ from html.parser import HTMLParser +def _force_test_templates(configuration): + from mig.lib.templates import init_global_templates + test_tmpl_dir = os.path.join(TEST_DATA_DIR, 'templates') + # populate current context with a template store backed onto test templates + init_global_templates(configuration, _templates_dirs=lambda: [test_tmpl_dir]) + + class DocumentBasicsHtmlParser(HTMLParser): """An HTML parser using builtin machinery to check basic html structure.""" @@ -303,6 +310,33 @@ def test_objects_with_type_text(self): output, _ = self.assertWsgiResponse(wsgi_result, self.fake_wsgi, 200) self.assertSnapshotOfHtmlContent(output) + def test_objects_with_type_template(self): + output_objects = [ + # workaround invalid HTML being generated with no title object + { + 'object_type': 'title', + 'text': 'TEST' + }, + { + 'object_type': 'template', + 'template_name': 'something', + 'template_group': 'test', + 'template_args': { + 'content': 'here!!' + } + } + ] + self.fake_backend.set_response(output_objects, returnvalues.OK) + _force_test_templates(self.configuration) + + wsgi_result = migwsgi.application( + *self.application_args, + **self.application_kwargs + ) + + output, _ = self.assertWsgiResponse(wsgi_result, self.fake_wsgi, 200) + self.assertSnapshotOfHtmlContent(output) + if __name__ == '__main__': testmain()