From 0e3d381e80af05e10656849d0c947e1d7c0ae7a4 Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Mon, 20 Jan 2025 20:36:31 +0100 Subject: [PATCH 1/3] Introduce support for jinja2-based templates. Make sure there is a separation of the test templates from the mainline stuff and tweak paths. --- .gitignore | 1 + mig/assets/templates/.gitkeep | 0 mig/lib/__init__.py | 0 mig/lib/templates/__init__.py | 161 ++++++++++++++++++ mig/lib/templates/__main__.py | 78 +++++++++ mig/shared/objecttypes.py | 24 ++- mig/shared/output.py | 12 +- mig/wsgi-bin/migwsgi.py | 2 +- requirements.txt | 6 + tests/data/templates/test_other.html.jinja | 1 + .../data/templates/test_something.html.jinja | 1 + .../test_objects_with_type_template.html | 1 + tests/test_mig_lib_templates.py | 52 ++++++ tests/test_mig_wsgibin.py | 36 +++- 14 files changed, 370 insertions(+), 5 deletions(-) create mode 100644 mig/assets/templates/.gitkeep create mode 100644 mig/lib/__init__.py create mode 100644 mig/lib/templates/__init__.py create mode 100644 mig/lib/templates/__main__.py create mode 100644 tests/data/templates/test_other.html.jinja create mode 100644 tests/data/templates/test_something.html.jinja create mode 100644 tests/snapshots/test_objects_with_type_template.html create mode 100644 tests/test_mig_lib_templates.py diff --git a/.gitignore b/.gitignore index 81d3cfafe..dea8c0347 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ # Byte-compiled / optimized / DLL files +__jinja__/ __pycache__/ *.py[cod] *$py.class diff --git a/mig/assets/templates/.gitkeep b/mig/assets/templates/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/mig/lib/__init__.py b/mig/lib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mig/lib/templates/__init__.py b/mig/lib/templates/__init__.py new file mode 100644 index 000000000..9878cd4fd --- /dev/null +++ b/mig/lib/templates/__init__.py @@ -0,0 +1,161 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# base - shared base helper functions +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +import errno +from jinja2 import meta as jinja2_meta, select_autoescape, Environment, \ + FileSystemLoader, FileSystemBytecodeCache +import os +import weakref + +from mig.shared.compat import PY2 +from mig.shared.defaults import MIG_BASE + + +if PY2: + from chainmap import ChainMap +else: + from collections import ChainMap + +TEMPLATES_DIR = os.path.join(MIG_BASE, 'mig/assets/templates') +TEMPLATES_CACHE_DIR = os.path.join(TEMPLATES_DIR, '__jinja__') + +_all_template_dirs = [ + TEMPLATES_DIR, +] +_global_store = None + + +def _clear_global_store(): + global _global_store + _global_store = None + + +def cache_dir(): + return TEMPLATES_CACHE_DIR + + +def template_dirs(): + return _all_template_dirs + + +class _BonundTemplate: + def __init__(self, template, template_args): + self.tmpl = template + self.args = template_args + + + def render(self): + return self.tmpl.render(**self.args) + + +class _FormatContext: + def __init__(self, configuration): + self.output_format = None + self.configuration = configuration + self.script_map = {} + self.style_map = {} + + def __getitem__(self, key): + return self.__dict__[key] + + def __iter__(self): + return iter(self.__dict__) + + def extend(self, template, template_args): + return _BonundTemplate(template, ChainMap(template_args, self)) + + +class TemplateStore: + def __init__(self, template_dirs, cache_dir=None, extra_globals=None): + assert cache_dir is not None + + self._cache_dir = cache_dir + self._template_globals = extra_globals + self._template_environment = Environment( + loader=FileSystemLoader(template_dirs), + bytecode_cache=FileSystemBytecodeCache(cache_dir, '%s'), + autoescape=select_autoescape() + ) + + @property + def cache_dir(self): + return self._cache_dir + + @property + def context(self): + return self._template_globals + + def _get_template(self, template_fqname): + return self._template_environment.get_template(template_fqname) + + def grab_template(self, template_name, template_group, output_format, template_globals=None, **kwargs): + template_fqname = "%s_%s.%s.jinja" % ( + template_group, template_name, output_format) + return self._template_environment.get_template(template_fqname, globals=template_globals) + + def list_templates(self): + return [t for t in self._template_environment.list_templates() if t.endswith('.jinja')] + + def extract_variables(self, template_fqname): + template = self._template_environment.get_template(template_fqname) + with open(template.filename) as f: + template_source = f.read() + ast = self._template_environment.parse(template_source) + return jinja2_meta.find_undeclared_variables(ast) + + @staticmethod + def populated(template_dirs, cache_dir=None, context=None): + assert cache_dir is not None + + try: + os.mkdir(cache_dir) + except OSError as direxc: + if direxc.errno != errno.EEXIST: # FileExistsError + raise + + store = TemplateStore( + template_dirs, cache_dir=cache_dir, extra_globals=context) + + for template_fqname in store.list_templates(): + store._get_template(template_fqname) + + return store + + +def init_global_templates(configuration, _templates_dirs=template_dirs): + global _global_store + + if _global_store is not None: + return _global_store + + _global_store = TemplateStore.populated( + _templates_dirs(), + cache_dir=cache_dir(), + context=_FormatContext(configuration) + ) + + return _global_store diff --git a/mig/lib/templates/__main__.py b/mig/lib/templates/__main__.py new file mode 100644 index 000000000..32453b66d --- /dev/null +++ b/mig/lib/templates/__main__.py @@ -0,0 +1,78 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# base - shared base helper functions +# Copyright (C) 2003-2024 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +from types import SimpleNamespace +import os +import sys + +from mig.lib.templates import init_global_templates +from mig.shared.conf import get_configuration_object + + +def warn(message): + print(message, file=sys.stderr, flush=True) + + +def main(args, _print=print): + configuration = get_configuration_object(config_file=args.config_file) + template_store = init_global_templates(configuration) + + command = args.command + if command == 'show': + print(template_store.list_templates()) + elif command == 'prime': + try: + os.mkdir(template_store.cache_dir) + except FileExistsError: + pass + + for template_fqname in template_store.list_templates(): + template_store._get_template(template_fqname) + elif command == 'vars': + for template_ref in template_store.list_templates(): + _print("<%s>" % (template_ref,)) + for var in template_store.extract_variables(template_ref): + _print(" {{%s}}" % (var,)) + _print("" % (template_ref,)) + else: + raise RuntimeError("unknown command: %s" % (command,)) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument('-c', dest='config_file', default=None) + parser.add_argument('command') + args = parser.parse_args() + + try: + main(args) + sys.exit(0) + except Exception as exc: + warn(str(exc)) + sys.exit(1) diff --git a/mig/shared/objecttypes.py b/mig/shared/objecttypes.py index 64b21303f..ccea24883 100644 --- a/mig/shared/objecttypes.py +++ b/mig/shared/objecttypes.py @@ -28,9 +28,13 @@ """ Defines valid objecttypes and provides a method to verify if an object is correct """ +from mig.lib.templates import init_global_templates + + start = {'object_type': 'start', 'required': [], 'optional': ['headers' ]} end = {'object_type': 'end', 'required': [], 'optional': []} +template = {'object_type': 'template'} timing_info = {'object_type': 'timing_info', 'required': [], 'optional': []} title = {'object_type': 'title', 'required': ['text'], @@ -396,6 +400,7 @@ valid_types_list = [ start, end, + template, timing_info, title, text, @@ -499,6 +504,8 @@ image_settings_list, ] +base_template_required = set(('template_name', 'template_group', 'template_args,')) + # valid_types_dict = {"title":title, "link":link, "header":header} # autogenerate dict based on list. Dictionary access is prefered to allow @@ -539,8 +546,8 @@ def get_object_type_info(object_type_list): return out -def validate(input_object): - """ validate input_object """ +def validate(input_object, configuration=None): + """ validate presented objects against their definitions """ if not type(input_object) == type([]): return (False, 'validate object must be a list' % ()) @@ -560,6 +567,19 @@ def validate(input_object): this_object_type = obj['object_type'] valid_object_type = valid_types_dict[this_object_type] + + if this_object_type == 'template': + # the required keys stuff below is not applicable to templates + # because templates know what they need in terms of data thus + # are self-documenting - use this fact to perform validation + #template_ref = "%s_%s.html" % (obj['template_group'], ) + store = init_global_templates(configuration) + template = store.grab_template(obj['template_name'], obj['template_group'], 'html') + valid_object_type = { + 'required': store.extract_variables(template) + } + obj = obj.get('template_args', None) + if 'required' in valid_object_type: for req in valid_object_type['required']: if req not in obj: diff --git a/mig/shared/output.py b/mig/shared/output.py index f0490aa39..3de2c75da 100644 --- a/mig/shared/output.py +++ b/mig/shared/output.py @@ -43,6 +43,7 @@ import time import traceback +from mig.lib.templates import init_global_templates from mig.shared import returnvalues from mig.shared.bailout import bailout_title, crash_helper, \ filter_output_objects @@ -746,6 +747,15 @@ def html_format(configuration, ret_val, ret_msg, out_obj): for i in out_obj: if i['object_type'] == 'start': pass + elif i['object_type'] == 'template': + store = init_global_templates(configuration) + template = store.grab_template( + i['template_name'], + i['template_group'], + 'html', + ) + bound = store.context.extend(template, i['template_args']) + lines.append(bound.render()) elif i['object_type'] == 'error_text': msg = "%(text)s" % i if i.get('exc', False): @@ -2821,7 +2831,7 @@ def format_output( logger = configuration.logger #logger.debug("format output to %s" % outputformat) valid_formats = get_valid_outputformats() - (val_ret, val_msg) = validate(out_obj) + (val_ret, val_msg) = validate(out_obj, configuration) if not val_ret: logger.error("%s formatting failed: %s (%s)" % (outputformat, val_msg, val_ret)) diff --git a/mig/wsgi-bin/migwsgi.py b/mig/wsgi-bin/migwsgi.py index e0937ab07..f8df55b2f 100755 --- a/mig/wsgi-bin/migwsgi.py +++ b/mig/wsgi-bin/migwsgi.py @@ -130,7 +130,7 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict, crash_helper(configuration, backend, output_objects) return (output_objects, returnvalues.ERROR) - (val_ret, val_msg) = validate(output_objects) + (val_ret, val_msg) = validate(output_objects, configuration=configuration) if not val_ret: (ret_code, ret_msg) = returnvalues.OUTPUT_VALIDATION_ERROR bailout_helper(configuration, backend, output_objects, diff --git a/requirements.txt b/requirements.txt index 5c2b1bc8f..57ea1829f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,8 @@ # https://pip.pypa.io/en/stable/reference/requirement-specifiers/ future +chainmap;python_version < "3.3" + # cgi was removed from the standard library in Python 3.13 legacy-cgi;python_version >= "3.13" @@ -26,6 +28,10 @@ email-validator;python_version >= "3.7" email-validator<2.0;python_version >= "3" and python_version < "3.7" email-validator<1.3;python_version < "3" +jinja2<3;python_version < "3" +jinja2==3.0.*;python_version >= "3" and python_version < "3.7" +jinja2;python_version >= "3.7" + # NOTE: additional optional dependencies depending on site conf are listed # in recommended.txt and can be installed in the same manner by pointing # pip there. diff --git a/tests/data/templates/test_other.html.jinja b/tests/data/templates/test_other.html.jinja new file mode 100644 index 000000000..f22ed0101 --- /dev/null +++ b/tests/data/templates/test_other.html.jinja @@ -0,0 +1 @@ +{{ other }} diff --git a/tests/data/templates/test_something.html.jinja b/tests/data/templates/test_something.html.jinja new file mode 100644 index 000000000..4c912a491 --- /dev/null +++ b/tests/data/templates/test_something.html.jinja @@ -0,0 +1 @@ + diff --git a/tests/snapshots/test_objects_with_type_template.html b/tests/snapshots/test_objects_with_type_template.html new file mode 100644 index 000000000..8eaf5bde5 --- /dev/null +++ b/tests/snapshots/test_objects_with_type_template.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/test_mig_lib_templates.py b/tests/test_mig_lib_templates.py new file mode 100644 index 000000000..2bb3a4967 --- /dev/null +++ b/tests/test_mig_lib_templates.py @@ -0,0 +1,52 @@ +import os +import shutil + +from tests.support import MigTestCase, testmain, \ + MIG_BASE, TEST_DATA_DIR, TEST_OUTPUT_DIR + +from mig.lib.templates import TemplateStore, template_dirs, \ + init_global_templates + +TEST_CACHE_DIR = os.path.join(TEST_OUTPUT_DIR, '__template_cache__') +TEST_TMPL_DIR = os.path.join(TEST_DATA_DIR, 'templates') + + +class TestMigSharedTemplates_instance(MigTestCase): + def after_each(self): + shutil.rmtree(TEST_CACHE_DIR, ignore_errors=True) + + def _provide_configuration(self): + return 'testconfig' + + def test_the_creation_of_a_template_store(self): + store = TemplateStore.populated(TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + self.assertIsInstance(store, TemplateStore) + + def test_a_listing_all_templates(self): + store = TemplateStore.populated(TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + self.assertEqual(len(store.list_templates()), 2) + + def test_grab_template(self): + store = TemplateStore.populated(TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + template = store.grab_template('other', 'test', 'html') + pass + + def test_variables_for_remplate_ref(self): + store = TemplateStore.populated(TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + template_vars = store.extract_variables('test_something.html.jinja') + self.assertEqual(template_vars, set(['content'])) + + +class TestMigSharedTemplates_global(MigTestCase): + def _provide_configuration(self): + return 'testconfig' + + def test_cache_location(self): + store = init_global_templates(self.configuration) + + relative_cache_dir = os.path.relpath(store.cache_dir, MIG_BASE) + self.assertEqual(relative_cache_dir, 'mig/assets/templates/__jinja__') + + +if __name__ == '__main__': + testmain() diff --git a/tests/test_mig_wsgibin.py b/tests/test_mig_wsgibin.py index 857d03b99..b00969c19 100644 --- a/tests/test_mig_wsgibin.py +++ b/tests/test_mig_wsgibin.py @@ -34,7 +34,7 @@ import stat import sys -from tests.support import PY2, MIG_BASE, MigTestCase, testmain, is_path_within +from tests.support import PY2, MIG_BASE, TEST_DATA_DIR, MigTestCase, testmain from tests.support.snapshotsupp import SnapshotAssertMixin from tests.support.wsgisupp import prepare_wsgi, WsgiAssertMixin @@ -49,6 +49,13 @@ from html.parser import HTMLParser +def _force_test_templates(configuration): + from mig.lib.templates import init_global_templates, _clear_global_store + _clear_global_store() + test_tmpl_dir = os.path.join(TEST_DATA_DIR, 'templates') + init_global_templates(configuration, _templates_dirs=lambda: [test_tmpl_dir]) + + class DocumentBasicsHtmlParser(HTMLParser): """An HTML parser using builtin machinery to check basic html structure.""" @@ -303,6 +310,33 @@ def test_objects_with_type_text(self): output, _ = self.assertWsgiResponse(wsgi_result, self.fake_wsgi, 200) self.assertSnapshotOfHtmlContent(output) + def test_objects_with_type_template(self): + output_objects = [ + # workaround invalid HTML being generated with no title object + { + 'object_type': 'title', + 'text': 'TEST' + }, + { + 'object_type': 'template', + 'template_name': 'something', + 'template_group': 'test', + 'template_args': { + 'content': 'here!!' + } + } + ] + self.fake_backend.set_response(output_objects, returnvalues.OK) + _force_test_templates(self.configuration) + + wsgi_result = migwsgi.application( + *self.application_args, + **self.application_kwargs + ) + + output, _ = self.assertWsgiResponse(wsgi_result, self.fake_wsgi, 200) + self.assertSnapshotOfHtmlContent(output) + if __name__ == '__main__': testmain() From aced52bd3c2bf3cdcd1f29f7a23091811391955d Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Mon, 3 Mar 2025 14:37:12 +0100 Subject: [PATCH 2/3] Avoid a global by placing the "global" template store in request context. --- mig/lib/templates/__init__.py | 14 +++++++------- mig/shared/configuration.py | 18 ++++++++++++++++++ tests/test_mig_shared_configuration.py | 2 +- tests/test_mig_wsgibin.py | 4 ++-- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/mig/lib/templates/__init__.py b/mig/lib/templates/__init__.py index 9878cd4fd..57eacaea9 100644 --- a/mig/lib/templates/__init__.py +++ b/mig/lib/templates/__init__.py @@ -46,7 +46,6 @@ _all_template_dirs = [ TEMPLATES_DIR, ] -_global_store = None def _clear_global_store(): @@ -147,15 +146,16 @@ def populated(template_dirs, cache_dir=None, context=None): def init_global_templates(configuration, _templates_dirs=template_dirs): - global _global_store + _context = configuration.context() - if _global_store is not None: - return _global_store + try: + return configuration.context(namespace='templates') + except KeyError as exc: + pass - _global_store = TemplateStore.populated( + store = TemplateStore.populated( _templates_dirs(), cache_dir=cache_dir(), context=_FormatContext(configuration) ) - - return _global_store + return configuration.context_set(store, namespace='templates') diff --git a/mig/shared/configuration.py b/mig/shared/configuration.py index 5dd5b145b..709f2988f 100644 --- a/mig/shared/configuration.py +++ b/mig/shared/configuration.py @@ -716,6 +716,7 @@ def __init__(self, config_file, verbose=False, skip_log=False, disable_auth_log=False): self.config_file = config_file self.mig_server_id = None + self._context = None configuration_options = copy.deepcopy(_CONFIGURATION_DEFAULTS) @@ -727,6 +728,23 @@ def __init__(self, config_file, verbose=False, skip_log=False, disable_auth_log=disable_auth_log, _config_file=config_file) + def context(self, namespace=None): + if self._context is None: + self._context = {} + if namespace is None: + return self._context + try: + return self._context[namespace] + except KeyError: + raise + + def context_set(self, value, namespace=None): + assert namespace is not None + + context = self.context() + context[namespace] = value + return value + def reload_config(self, verbose, skip_log=False, disable_auth_log=False, _config_file=None): """Re-read and parse configuration file. Optional skip_log arg diff --git a/tests/test_mig_shared_configuration.py b/tests/test_mig_shared_configuration.py index e3132756a..81658f4f4 100644 --- a/tests/test_mig_shared_configuration.py +++ b/tests/test_mig_shared_configuration.py @@ -41,7 +41,7 @@ def _is_method(value): def _to_dict(obj): return {k: v for k, v in inspect.getmembers(obj) - if not (k.startswith('__') or _is_method(v))} + if not (k.startswith('_') or _is_method(v))} class MigSharedConfiguration(MigTestCase): diff --git a/tests/test_mig_wsgibin.py b/tests/test_mig_wsgibin.py index b00969c19..d4be4bc48 100644 --- a/tests/test_mig_wsgibin.py +++ b/tests/test_mig_wsgibin.py @@ -50,9 +50,9 @@ def _force_test_templates(configuration): - from mig.lib.templates import init_global_templates, _clear_global_store - _clear_global_store() + from mig.lib.templates import init_global_templates test_tmpl_dir = os.path.join(TEST_DATA_DIR, 'templates') + # populate current context with a template store backed onto test templates init_global_templates(configuration, _templates_dirs=lambda: [test_tmpl_dir]) From 6abeec60633ff4d07fb189f89b13360822f6f350 Mon Sep 17 00:00:00 2001 From: Alex Burke Date: Fri, 21 Mar 2025 11:10:45 +0100 Subject: [PATCH 3/3] Make in the necessary glue to support future translation of templates. --- local-requirements.txt | 1 + mig/lib/templates/__init__.py | 20 ++++--- mig/lib/templates/__main__.py | 52 +++++++++++++++++-- .../data/templates/test_translated.html.jinja | 1 + tests/test_mig_lib_templates.py | 24 ++++++--- 5 files changed, 81 insertions(+), 17 deletions(-) create mode 100644 tests/data/templates/test_translated.html.jinja diff --git a/local-requirements.txt b/local-requirements.txt index 7faf3b026..9edc94117 100644 --- a/local-requirements.txt +++ b/local-requirements.txt @@ -7,4 +7,5 @@ autopep8;python_version >= "3" # NOTE: paramiko-3.0.0 dropped python2 and python3.6 support paramiko;python_version >= "3.7" paramiko<3;python_version < "3.7" +polib werkzeug diff --git a/mig/lib/templates/__init__.py b/mig/lib/templates/__init__.py index 57eacaea9..a46934048 100644 --- a/mig/lib/templates/__init__.py +++ b/mig/lib/templates/__init__.py @@ -57,7 +57,7 @@ def cache_dir(): return TEMPLATES_CACHE_DIR -def template_dirs(): +def _global_template_dirs(): return _all_template_dirs @@ -66,7 +66,6 @@ def __init__(self, template, template_args): self.tmpl = template self.args = template_args - def render(self): return self.tmpl.render(**self.args) @@ -95,10 +94,12 @@ def __init__(self, template_dirs, cache_dir=None, extra_globals=None): self._cache_dir = cache_dir self._template_globals = extra_globals self._template_environment = Environment( + extensions=['jinja2.ext.i18n'], loader=FileSystemLoader(template_dirs), bytecode_cache=FileSystemBytecodeCache(cache_dir, '%s'), autoescape=select_autoescape() ) + self._template_environment.install_null_translations() @property def cache_dir(self): @@ -111,6 +112,11 @@ def context(self): def _get_template(self, template_fqname): return self._template_environment.get_template(template_fqname) + def _get_template_source(self, template_fqname): + template = self._template_environment.get_template(template_fqname) + with open(template.filename) as f: + return f.read() + def grab_template(self, template_name, template_group, output_format, template_globals=None, **kwargs): template_fqname = "%s_%s.%s.jinja" % ( template_group, template_name, output_format) @@ -119,10 +125,12 @@ def grab_template(self, template_name, template_group, output_format, template_g def list_templates(self): return [t for t in self._template_environment.list_templates() if t.endswith('.jinja')] + def extract_translations(self, template_fqname): + template_source = self._get_template_source(template_fqname) + return self._template_environment.extract_translations(template_source) + def extract_variables(self, template_fqname): - template = self._template_environment.get_template(template_fqname) - with open(template.filename) as f: - template_source = f.read() + template_source = self._get_template_source(template_fqname) ast = self._template_environment.parse(template_source) return jinja2_meta.find_undeclared_variables(ast) @@ -145,7 +153,7 @@ def populated(template_dirs, cache_dir=None, context=None): return store -def init_global_templates(configuration, _templates_dirs=template_dirs): +def init_global_templates(configuration, _templates_dirs=_global_template_dirs): _context = configuration.context() try: diff --git a/mig/lib/templates/__main__.py b/mig/lib/templates/__main__.py index 32453b66d..3da8880b5 100644 --- a/mig/lib/templates/__main__.py +++ b/mig/lib/templates/__main__.py @@ -25,21 +25,30 @@ # -- END_HEADER --- # -from types import SimpleNamespace +from __future__ import print_function import os import sys -from mig.lib.templates import init_global_templates +from mig.lib.templates import init_global_templates, _global_template_dirs +from mig.shared.compat import SimpleNamespace from mig.shared.conf import get_configuration_object def warn(message): - print(message, file=sys.stderr, flush=True) + print(message, sys.stderr, True) def main(args, _print=print): - configuration = get_configuration_object(config_file=args.config_file) - template_store = init_global_templates(configuration) + configuration = get_configuration_object( + config_file=args.config_file, skip_log=True, disable_auth_log=True) + + if args.tmpldir: + def _template_dirs(): return [args.tmpldir] + else: + _template_dirs = _global_template_dirs + + template_store = init_global_templates( + configuration, _templates_dirs=_template_dirs) command = args.command if command == 'show': @@ -52,6 +61,38 @@ def main(args, _print=print): for template_fqname in template_store.list_templates(): template_store._get_template(template_fqname) + elif command == 'translations': + try: + os.mkdir(template_store.cache_dir) + except FileExistsError: + pass + + for template_fqname in template_store.list_templates(): + extracted_tuples = list( + template_store.extract_translations(template_fqname)) + if len(extracted_tuples) == 0: + continue + _print("<%s>" % (template_fqname,)) + for _, __, string in template_store.extract_translations(template_fqname): + _print(' "%s"' % (string,)) + _print("" % (template_fqname,)) + + + elif command == 'translations-mo': + try: + os.mkdir(template_store.cache_dir) + except FileExistsError: + pass + + import polib + pofile = polib.POFile() + for template_fqname in template_store.list_templates(): + for _, __, string in template_store.extract_translations(template_fqname): + poentry = polib.POEntry(msgid=string, msgstr=string) + pofile.append(poentry) + + _print(pofile) + elif command == 'vars': for template_ref in template_store.list_templates(): _print("<%s>" % (template_ref,)) @@ -67,6 +108,7 @@ def main(args, _print=print): parser = argparse.ArgumentParser() parser.add_argument('-c', dest='config_file', default=None) + parser.add_argument('--template-dir', dest='tmpldir') parser.add_argument('command') args = parser.parse_args() diff --git a/tests/data/templates/test_translated.html.jinja b/tests/data/templates/test_translated.html.jinja new file mode 100644 index 000000000..d6eb4a3af --- /dev/null +++ b/tests/data/templates/test_translated.html.jinja @@ -0,0 +1 @@ +
{% trans article_title %}The title of my article{% endtrans %}
diff --git a/tests/test_mig_lib_templates.py b/tests/test_mig_lib_templates.py index 2bb3a4967..d31c5b26c 100644 --- a/tests/test_mig_lib_templates.py +++ b/tests/test_mig_lib_templates.py @@ -4,11 +4,14 @@ from tests.support import MigTestCase, testmain, \ MIG_BASE, TEST_DATA_DIR, TEST_OUTPUT_DIR -from mig.lib.templates import TemplateStore, template_dirs, \ +from mig.lib.templates import TemplateStore, _global_template_dirs, \ init_global_templates + TEST_CACHE_DIR = os.path.join(TEST_OUTPUT_DIR, '__template_cache__') TEST_TMPL_DIR = os.path.join(TEST_DATA_DIR, 'templates') +TEST_TMPL_COUNT = len(list(x for x in os.listdir( + TEST_TMPL_DIR) if not x.startswith('.'))) class TestMigSharedTemplates_instance(MigTestCase): @@ -19,23 +22,32 @@ def _provide_configuration(self): return 'testconfig' def test_the_creation_of_a_template_store(self): - store = TemplateStore.populated(TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + store = TemplateStore.populated( + TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) self.assertIsInstance(store, TemplateStore) def test_a_listing_all_templates(self): - store = TemplateStore.populated(TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) - self.assertEqual(len(store.list_templates()), 2) + store = TemplateStore.populated( + TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + self.assertEqual(len(store.list_templates()), TEST_TMPL_COUNT) def test_grab_template(self): - store = TemplateStore.populated(TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + store = TemplateStore.populated( + TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) template = store.grab_template('other', 'test', 'html') pass def test_variables_for_remplate_ref(self): - store = TemplateStore.populated(TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + store = TemplateStore.populated( + TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) template_vars = store.extract_variables('test_something.html.jinja') self.assertEqual(template_vars, set(['content'])) + def test_translated_template(self): + store = TemplateStore.populated( + TEST_TMPL_DIR, cache_dir=TEST_CACHE_DIR) + store.extract_translations('test_translated.html.jinja') + class TestMigSharedTemplates_global(MigTestCase): def _provide_configuration(self):