Make the force recursive functions work on Python 3.

* support tuples
* string conversion
* define the walking internals only once

Author: albu-diku

mig/shared/base.py

@@ -32,10 +32,12 @@
 from past.builtins import basestring
 
 import base64
+import codecs
 import os
 import re
 
 # IMPORTANT: do not import any other MiG modules here - to avoid import loops
+from mig.shared.compat import PY2
 from mig.shared.defaults import default_str_coding, default_fs_coding, \
     keyword_all, keyword_auto, sandbox_names, _user_invisible_files, \
     _user_invisible_dirs, _vgrid_xgi_scripts, cert_field_order, csrf_field, \
@@ -505,23 +507,30 @@ def force_utf8(val, highlight=''):
         val = "%s" % val
     if not is_unicode(val):
         return val
-    return "%s%s%s" % (highlight, val.encode("utf8"), highlight)
+    return codecs.encode("%s%s%s" % (highlight, val, highlight), 'utf8')
+
+
+def _walk_and_covert_recursive(input_obj, highlight='', _as_bytes=False, _force_primitive=None, _force_recursive=None):
+    thetype = type(input_obj)
+    if issubclass(thetype, dict):
+        return {_force_recursive(i, highlight): _force_recursive(j, highlight) for (i, j) in
+                input_obj.items()}
+    elif issubclass(thetype, (list, tuple)):
+        return thetype((_force_recursive(i, highlight) for i in input_obj))
+    elif not is_unicode(input_obj):
+        return _force_primitive(input_obj, highlight)
+    elif not PY2 and _as_bytes:
+        return _force_primitive(input_obj, highlight)
+    else:
+        return input_obj
 
 
 def force_utf8_rec(input_obj, highlight=''):
     """Recursive object conversion from unicode to utf8: useful to convert e.g.
     dictionaries with nested unicode strings to a pure utf8 version. Actual
     changes are marked out with the highlight string if given.
     """
-    if isinstance(input_obj, dict):
-        return {force_utf8_rec(i, highlight): force_utf8_rec(j, highlight) for (i, j) in
-                input_obj.items()}
-    elif isinstance(input_obj, list):
-        return [force_utf8_rec(i, highlight) for i in input_obj]
-    elif is_unicode(input_obj):
-        return force_utf8(input_obj, highlight)
-    else:
-        return input_obj
+    return _walk_and_covert_recursive(input_obj, highlight, _as_bytes=True, _force_primitive=force_utf8, _force_recursive=force_utf8_rec)
 
 
 def force_unicode(val, highlight=''):
@@ -541,15 +550,7 @@ def force_unicode_rec(input_obj, highlight=''):
     dictionaries with nested utf8 strings to a pure unicode version. Actual
     changes are marked out with the highlight string if given.
     """
-    if isinstance(input_obj, dict):
-        return {force_unicode_rec(i, highlight): force_unicode_rec(j, highlight) for (i, j) in
-                input_obj.items()}
-    elif isinstance(input_obj, list):
-        return [force_unicode_rec(i, highlight) for i in input_obj]
-    elif not is_unicode(input_obj):
-        return force_unicode(input_obj, highlight)
-    else:
-        return input_obj
+    return _walk_and_covert_recursive(input_obj, highlight, _force_primitive=force_unicode, _force_recursive=force_unicode_rec)
 
 
 def _force_default_coding(input_obj, kind, highlight=''):

mig/shared/safeinput.py

@@ -58,7 +58,8 @@
     from html import escape as escape_html
 assert escape_html is not None
 
-from mig.shared.base import force_unicode, force_utf8
+from mig.shared.base import force_unicode
+from mig.shared.compat import ensure_native_string
 from mig.shared.defaults import src_dst_sep, username_charset, \
     username_max_length, session_id_charset, session_id_length, \
     subject_id_charset, subject_id_min_length, subject_id_max_length, \
@@ -2284,7 +2285,7 @@ def __init__(self, value):
     def __str__(self):
         """Return string representation"""
 
-        return force_utf8(force_unicode(self.value))
+        return ensure_native_string(self.value)
 
 
 def main(_exit=sys.exit, _print=print):

tests/test_mig_shared_base.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# test_mig_shared_compat - unit test of the corresponding mig shared module
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+"""Unit tests for the migrid module pointed to in the filename"""
+
+import binascii
+import os
+import sys
+
+from tests.support import PY2, MigTestCase, testmain
+
+from mig.shared.base import force_default_fs_coding_rec, force_default_str_coding_rec
+
+DUMMY_BYTECHARS = b'DEADBEEF'
+DUMMY_BYTESRAW = binascii.unhexlify('DEADBEEF') # 4 bytes
+DUMMY_UNICODE = u'UniCode123½¾µßðþđŋħĸþł@ª€£$¥©®'
+
+
+if PY2:
+    def _strbytes(value):
+        assert isinstance(value, str)
+        return value
+else:
+    def _strbytes(value):
+        assert isinstance(value, str)
+        return bytes(value, 'utf8')
+
+
+class MigSharedBase__force_default_fs_coding_rec(MigTestCase):
+    """Unit tests of mig.shared.base force_default_fs_coding_rec()"""
+
+    def test_encode_a_string(self):
+        output = force_default_fs_coding_rec('foobar')
+
+        self.assertEqual(output, _strbytes('foobar'))
+
+    def test_encode_within_a_dict(self):
+        output = force_default_fs_coding_rec({ 'key': 'value' })
+
+        self.assertEqual(output, { _strbytes('key'): _strbytes('value') })
+
+    def test_encode_within_a_list(self):
+        output = force_default_fs_coding_rec(['foo', 'bar', 'baz'])
+
+        self.assertEqual(output, [_strbytes('foo'), _strbytes('bar'), _strbytes('baz')])
+
+    def test_encode_within_a_tuple(self):
+        output = force_default_fs_coding_rec(('foo', 'bar', 'baz'))
+
+        self.assertEqual(output, (_strbytes('foo'), _strbytes('bar'), _strbytes('baz')))
+
+
+class MigSharedBase__force_default_str_coding_rec(MigTestCase):
+    """Unit tests of mig.shared.base force_default_str_coding_rec()"""
+
+    def test_encode_a_string(self):
+        output = force_default_str_coding_rec('foobar')
+
+        self.assertEqual(output, 'foobar')
+
+    def test_encode_within_a_dict(self):
+        output = force_default_str_coding_rec({ 'key': 'value' })
+
+        self.assertEqual(output, { 'key': 'value' })
+
+    def test_encode_within_a_list(self):
+        output = force_default_str_coding_rec(['foo', 'bar', 'baz'])
+
+        self.assertEqual(output, ['foo', 'bar', 'baz'])
+
+    def test_encode_within_a_tuple(self):
+        output = force_default_str_coding_rec(('foo', 'bar', 'baz'))
+
+        self.assertEqual(output, ('foo', 'bar', 'baz'))
+
+
+if __name__ == '__main__':
+    testmain()