Introduce support for jinja2-based templates.

Make sure there is a separation of the test templates from the mainline stuff
and tweak paths.

Author: albu-diku

.gitignore

@@ -1,4 +1,5 @@
 # Byte-compiled / optimized / DLL files
+__jinja__/
 __pycache__/
 *.py[cod]
 *$py.class

mig/assets/templates/.gitkeep

@@ -0,0 +1,161 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# base - shared base helper functions
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# -- END_HEADER ---
+#
+
+import errno
+from jinja2 import meta as jinja2_meta, select_autoescape, Environment, \
+    FileSystemLoader, FileSystemBytecodeCache
+import os
+import weakref
+
+from mig.shared.compat import PY2
+from mig.shared.defaults import MIG_BASE
+
+
+if PY2:
+    from chainmap import ChainMap
+else:
+    from collections import ChainMap
+
+TEMPLATES_DIR = os.path.join(MIG_BASE, 'mig/assets/templates')
+TEMPLATES_CACHE_DIR = os.path.join(TEMPLATES_DIR, '__jinja__')
+
+_all_template_dirs = [
+    TEMPLATES_DIR,
+]
+_global_store = None
+
+
+def _clear_global_store():
+    global _global_store
+    _global_store = None
+
+
+def cache_dir():
+    return TEMPLATES_CACHE_DIR
+
+
+def template_dirs():
+    return _all_template_dirs
+
+
+class _BonundTemplate:
+    def __init__(self, template, template_args):
+        self.tmpl = template
+        self.args = template_args
+
+
+    def render(self):
+        return self.tmpl.render(**self.args)
+
+
+class _FormatContext:
+    def __init__(self, configuration):
+        self.output_format = None
+        self.configuration = configuration
+        self.script_map = {}
+        self.style_map = {}
+
+    def __getitem__(self, key):
+        return self.__dict__[key]
+
+    def __iter__(self):
+        return iter(self.__dict__)
+
+    def extend(self, template, template_args):
+        return _BonundTemplate(template, ChainMap(template_args, self))
+
+
+class TemplateStore:
+    def __init__(self, template_dirs, cache_dir=None, extra_globals=None):
+        assert cache_dir is not None
+
+        self._cache_dir = cache_dir
+        self._template_globals = extra_globals
+        self._template_environment = Environment(
+            loader=FileSystemLoader(template_dirs),
+            bytecode_cache=FileSystemBytecodeCache(cache_dir, '%s'),
+            autoescape=select_autoescape()
+        )
+
+    @property
+    def cache_dir(self):
+        return self._cache_dir
+
+    @property
+    def context(self):
+        return self._template_globals
+
+    def _get_template(self, template_fqname):
+        return self._template_environment.get_template(template_fqname)
+
+    def grab_template(self, template_name, template_group, output_format, template_globals=None, **kwargs):
+        template_fqname = "%s_%s.%s.jinja" % (
+            template_group, template_name, output_format)
+        return self._template_environment.get_template(template_fqname, globals=template_globals)
+
+    def list_templates(self):
+        return [t for t in self._template_environment.list_templates() if t.endswith('.jinja')]
+
+    def extract_variables(self, template_fqname):
+        template = self._template_environment.get_template(template_fqname)
+        with open(template.filename) as f:
+            template_source = f.read()
+        ast = self._template_environment.parse(template_source)
+        return jinja2_meta.find_undeclared_variables(ast)
+
+    @staticmethod
+    def populated(template_dirs, cache_dir=None, context=None):
+        assert cache_dir is not None
+
+        try:
+            os.mkdir(cache_dir)
+        except OSError as direxc:
+            if direxc.errno != errno.EEXIST:  # FileExistsError
+                raise
+
+        store = TemplateStore(
+            template_dirs, cache_dir=cache_dir, extra_globals=context)
+
+        for template_fqname in store.list_templates():
+            store._get_template(template_fqname)
+
+        return store
+
+
+def init_global_templates(configuration, _templates_dirs=template_dirs):
+    global _global_store
+
+    if _global_store is not None:
+        return _global_store
+
+    _global_store = TemplateStore.populated(
+        _templates_dirs(),
+        cache_dir=cache_dir(),
+        context=_FormatContext(configuration)
+    )
+
+    return _global_store

mig/lib/__init__.py

@@ -0,0 +1,78 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# base - shared base helper functions
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# -- END_HEADER ---
+#
+
+from types import SimpleNamespace
+import os
+import sys
+
+from mig.lib.templates import init_global_templates
+from mig.shared.conf import get_configuration_object
+
+
+def warn(message):
+    print(message, file=sys.stderr, flush=True)
+
+
+def main(args, _print=print):
+    configuration = get_configuration_object(config_file=args.config_file)
+    template_store = init_global_templates(configuration)
+
+    command = args.command
+    if command == 'show':
+        print(template_store.list_templates())
+    elif command == 'prime':
+        try:
+            os.mkdir(template_store.cache_dir)
+        except FileExistsError:
+            pass
+
+        for template_fqname in template_store.list_templates():
+            template_store._get_template(template_fqname)
+    elif command == 'vars':
+        for template_ref in template_store.list_templates():
+            _print("<%s>" % (template_ref,))
+            for var in template_store.extract_variables(template_ref):
+                _print("  {{%s}}" % (var,))
+            _print("</%s>" % (template_ref,))
+    else:
+        raise RuntimeError("unknown command: %s" % (command,))
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', dest='config_file', default=None)
+    parser.add_argument('command')
+    args = parser.parse_args()
+
+    try:
+        main(args)
+        sys.exit(0)
+    except Exception as exc:
+        warn(str(exc))
+        sys.exit(1)

mig/lib/templates/__init__.py

@@ -28,9 +28,13 @@
 
 """ Defines valid objecttypes and provides a method to verify if an object is correct """
 
+from mig.lib.templates import init_global_templates
+
+
 start = {'object_type': 'start', 'required': [], 'optional': ['headers'
                                                               ]}
 end = {'object_type': 'end', 'required': [], 'optional': []}
+template = {'object_type': 'template'}
 timing_info = {'object_type': 'timing_info', 'required': [],
                'optional': []}
 title = {'object_type': 'title', 'required': ['text'],
@@ -396,6 +400,7 @@
 valid_types_list = [
     start,
     end,
+    template,
     timing_info,
     title,
     text,
@@ -499,6 +504,8 @@
     image_settings_list,
 ]
 
+base_template_required = set(('template_name', 'template_group', 'template_args,'))
+
 # valid_types_dict = {"title":title, "link":link, "header":header}
 
 # autogenerate dict based on list. Dictionary access is prefered to allow
@@ -539,8 +546,8 @@ def get_object_type_info(object_type_list):
     return out
 
 
-def validate(input_object):
-    """ validate input_object """
+def validate(input_object, configuration=None):
+    """ validate presented objects against their definitions """
 
     if not type(input_object) == type([]):
         return (False, 'validate object must be a list' % ())
@@ -560,6 +567,19 @@ def validate(input_object):
 
             this_object_type = obj['object_type']
             valid_object_type = valid_types_dict[this_object_type]
+
+            if this_object_type == 'template':
+                # the required keys stuff below is not applicable to templates
+                # because templates know what they need in terms of data thus
+                # are self-documenting - use this fact to perform validation
+                #template_ref = "%s_%s.html" % (obj['template_group'], )
+                store = init_global_templates(configuration)
+                template = store.grab_template(obj['template_name'], obj['template_group'], 'html')
+                valid_object_type = {
+                    'required': store.extract_variables(template)
+                }
+                obj = obj.get('template_args', None)
+
             if 'required' in valid_object_type:
                 for req in valid_object_type['required']:
                     if req not in obj:

mig/lib/templates/__main__.py

@@ -43,6 +43,7 @@
 import time
 import traceback
 
+from mig.lib.templates import init_global_templates
 from mig.shared import returnvalues
 from mig.shared.bailout import bailout_title, crash_helper, \
     filter_output_objects
@@ -746,6 +747,15 @@ def html_format(configuration, ret_val, ret_msg, out_obj):
     for i in out_obj:
         if i['object_type'] == 'start':
             pass
+        elif i['object_type'] == 'template':
+            store = init_global_templates(configuration)
+            template = store.grab_template(
+                i['template_name'],
+                i['template_group'],
+                'html',
+            )
+            bound = store.context.extend(template, i['template_args'])
+            lines.append(bound.render())
         elif i['object_type'] == 'error_text':
             msg = "%(text)s" % i
             if i.get('exc', False):
@@ -2821,7 +2831,7 @@ def format_output(
     logger = configuration.logger
     #logger.debug("format output to %s" % outputformat)
     valid_formats = get_valid_outputformats()
-    (val_ret, val_msg) = validate(out_obj)
+    (val_ret, val_msg) = validate(out_obj, configuration)
     if not val_ret:
         logger.error("%s formatting failed: %s (%s)" %
                      (outputformat, val_msg, val_ret))

mig/shared/objecttypes.py

@@ -130,7 +130,7 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict,
         crash_helper(configuration, backend, output_objects)
         return (output_objects, returnvalues.ERROR)
 
-    (val_ret, val_msg) = validate(output_objects)
+    (val_ret, val_msg) = validate(output_objects, configuration=configuration)
     if not val_ret:
         (ret_code, ret_msg) = returnvalues.OUTPUT_VALIDATION_ERROR
         bailout_helper(configuration, backend, output_objects,

mig/shared/output.py

@@ -2,6 +2,8 @@
 # https://pip.pypa.io/en/stable/reference/requirement-specifiers/
 future
 
+chainmap;python_version < "3.3"
+
 # cgi was removed from the standard library in Python 3.13
 legacy-cgi;python_version >= "3.13"
 
@@ -26,6 +28,10 @@ email-validator;python_version >= "3.7"
 email-validator<2.0;python_version >= "3" and python_version < "3.7"
 email-validator<1.3;python_version < "3"
 
+jinja2<3;python_version < "3"
+jinja2==3.0.*;python_version >= "3" and python_version < "3.7"
+jinja2;python_version >= "3.7"
+
 # NOTE: additional optional dependencies depending on site conf are listed
 #       in recommended.txt and can be installed in the same manner by pointing
 #       pip there.

mig/wsgi-bin/migwsgi.py

@@ -0,0 +1 @@
+<q>{{ other }}</q>

requirements.txt

@@ -0,0 +1 @@
+<aside>{{ content }}</aside>

tests/data/templates/test_other.html.jinja

@@ -0,0 +1 @@
+<aside>here!!</aside>