Pretty printing unicode data

Author: gershnik

lib/inc/sys_string/impl/unicode/mappings.h

@@ -20,29 +20,22 @@
 
 namespace sysstr::util::unicode 
 {
-    template<class Derived>
-    class lookup
+    class is_whitespace 
     {
+    private:
+        static const char16_t chars[26];
     public:
+        static constexpr char32_t max_char = U'\u3000';
+    
         static bool test(char32_t c) noexcept
         {
-            if (c > Derived::max_char)
+            if (c > max_char)
                 return false;
-            for(auto p = Derived::chars; *p; ++p)
+            for(auto p = chars; *p; ++p)
                 if (*p == c)
                     return true;
             return false;
         }
-    };
-
-
-    class is_whitespace : public lookup<is_whitespace>
-    {
-    friend lookup<is_whitespace>;
-    private:
-        static const char16_t chars[26];
-    public:
-        static constexpr char32_t max_char = U'\u3000';
 
         static constexpr size_t data_size = sizeof(chars);
     };

lib/src/unicode_mappings.cpp

@@ -3,7 +3,7 @@
 from textwrap import dedent
 from typing import Optional
 
-from common import char_name, chars_len_in_utf16, indent_insert
+from common import chars_len_in_utf16, format_utf16_string, indent_insert
 from table_builder import table_builder
 
 
@@ -186,21 +186,10 @@ class props
         '''
         return dedent(ret)
 
-    def __print_cased_data(self):
-        ret = '\nu"'
-        char_count = 0
-        for char in self.__mapped_data:
-            ret += char_name(char)
-            char_count += 1
-            if char_count > 0 and char_count % 16 == 0:
-                ret += '"\nu"'
-        ret += '"'
-        return ret
-
     def print_impl(self):
         ret = f'''
         const char16_t case_mapper::cased_data[{self.__cased_data_len}] = 
-            {indent_insert(self.__print_cased_data(), 12)};
+            {indent_insert(format_utf16_string(self.__mapped_data), 12)};
 
         {indent_insert(self.__builder.print_impl("case_mapper::lookup"), 8)}
         '''

unicode/scripts/case_builder.py

@@ -8,12 +8,20 @@
 # 
 
 import textwrap
+from math import log10
 
 def char_name(code):
     if code < 0x10000:
         return f"\\u{code:04X}"
     else:
         return f"\\U{code:08X}"
+    
+def split_utf16(code):
+    if code < 0x10000:
+        return (code, )
+    first = ((code - 0x010000) >> 10)     + 0x00D800
+    second = ((code - 0x010000) & 0x03FF) + 0x00DC00
+    return (first, second)
 
 def parse_char_range(text):
     char_range = [int(x, 16) for x in text.split('..')]
@@ -58,6 +66,39 @@ def chars_len_in_utf16(chars):
         ret += char_len_in_utf16(char)
     return ret
 
+def format_utf16_string(chars):
+    ret = '\nu"'
+    line_len = 0
+    for char in chars:
+        cur = char_name(char)
+        ret += cur
+        line_len += len(cur)
+        if line_len >= 100:
+            ret += '"\nu"'
+            line_len = 0
+    ret += '"'
+    return ret
+
+def format_array(values, ishex=False, bits=0, max_width=120):
+    if ishex:
+        hf = hex_format_for_bits(bits)
+        fmt = '0x{:' + hf + '}'
+    else:
+        digits = int(log10(1 << bits)) + 1
+        fmt = '{:>' + str(digits) + '}'
+    
+    entry_len = len(fmt.format(0) + ', ')
+    max_count_per_line = max_width // entry_len
+        
+    ret = ''
+    for idx, value in enumerate(values):
+        if idx > 0:
+            ret += ', '
+            if idx % max_count_per_line == 0:
+                ret += '\n'
+        ret += fmt.format(value)
+    return ret
+
 def indent_insert(text: str, count: int):
     ret = textwrap.indent(text, ' ' * count, lambda line: True).lstrip()
     return ret

unicode/scripts/common.py

@@ -318,8 +318,6 @@ def main():
 
 namespace sysstr::util::unicode 
 {{
-    {indent_insert(lookup_builder.print_common_header(), 4)}
-
     {indent_insert(whitespaces.print_header('is_whitespace'), 4)}
 
     {indent_insert(case_info_builder.print_header(), 4)}

unicode/scripts/genmappings.py

@@ -7,7 +7,7 @@
 # 
 
 from textwrap import dedent
-from common import char_name, indent_insert
+from common import char_name, format_utf16_string, indent_insert
 
 class lookup_builder:
 
@@ -27,46 +27,24 @@ def generate(self):
         self.length += 1
         return self.length * 2
 
-    def __make_content(self):
-        ret = 'u"'
-        char_count = 0
-        for char in self.chars:
-            ret += char_name(char)
-            char_count += 1
-            if char_count > 0 and char_count % 16 == 0:
-                ret += '"\nu"'
-        ret += '"'
-        return ret
-    
-    @staticmethod
-    def print_common_header():
-        ret = '''
-        template<class Derived>
-        class lookup
-        {
-        public:
-            static bool test(char32_t c) noexcept
-            {
-                if (c > Derived::max_char)
-                    return false;
-                for(auto p = Derived::chars; *p; ++p)
-                    if (*p == c)
-                        return true;
-                return false;
-            }
-        };
-        '''
-        return dedent(ret)
-    
     def print_header(self, name):
         ret = f'''
-        class {name} : public lookup<{name}>
+        class {name} 
         {{
-        friend lookup<{name}>;
         private:
             static const char16_t chars[{self.length}];
         public:
             static constexpr char32_t max_char = U'{char_name(self.max_char)}';
+
+            static bool test(char32_t c) noexcept
+            {{
+                if (c > max_char)
+                    return false;
+                for(auto p = chars; *p; ++p)
+                    if (*p == c)
+                        return true;
+                return false;
+            }}
             
             static constexpr size_t data_size = sizeof(chars);
         }};
@@ -76,6 +54,6 @@ class {name} : public lookup<{name}>
     def print_impl(self, name):
         ret = f'''
         const char16_t {name}::chars[] = 
-            {indent_insert(self.__make_content(), 12)};
+            {indent_insert(format_utf16_string(self.chars), 12)};
         '''
         return dedent(ret)

unicode/scripts/lookup_builder.py

@@ -3,8 +3,7 @@
 from dataclasses import dataclass
 from textwrap import dedent
 
-from common import indent_insert
-#from trie_builder import trie_builder
+from common import format_array, indent_insert
 from table_builder import table_builder
 
 
@@ -356,34 +355,14 @@ class normalizer
         return dedent(ret)
 
 
-    def __print_compositions(self):
-        ret = '\n'
-        val_count = 0
-        for val in self.__compositions:
-            ret += f'0x{val:08X}, '
-            val_count += 1
-            if val_count > 0 and val_count % 16 == 0:
-                ret += '\n'
-        return ret
-    
-    def __print_values(self):
-        ret = '\n'
-        val_count = 0
-        for val in self.__values:
-            ret += f'0x{val:08X}, '
-            val_count += 1
-            if val_count > 0 and val_count % 16 == 0:
-                ret += '\n'
-        return ret
-    
     def print_impl(self):
         ret = f'''
         const uint32_t normalizer::compositions[] = {{
-            {indent_insert(self.__print_compositions(), 12)}
+            {indent_insert(format_array(self.__compositions, ishex=True, bits=32), 12)}
         }};
 
         const uint32_t normalizer::values[] = {{
-            {indent_insert(self.__print_values(), 12)}
+            {indent_insert(format_array(self.__values, ishex=True, bits=32), 12)}
         }};
 
         {indent_insert(self.__decomp_builder.print_impl("normalizer::lookup"), 8)}

unicode/scripts/norm_builder.py

@@ -7,7 +7,8 @@
 #
 
 from textwrap import dedent
-from common import bytes_for_bits, type_for_bits, indent_insert
+import textwrap
+from common import bytes_for_bits, format_array, type_for_bits, indent_insert
 
 
 class table_builder:
@@ -133,30 +134,17 @@ def generate(self):
 
         return total_size
 
-    def __make_nested(self, values):
+    def __make_nested(self, values, bits=0):
         ret = ''
         for block_idx, block in enumerate(values):
             if block_idx > 0:
                 ret += ','
-            ret += '\n'
-            ret += '{'
-            for idx, val in enumerate(block):
-                if idx > 0:
-                    ret += ', '
-                #ret += f'0x{val:01X}'
-                ret += f'{val}'
-            ret += '}'
-        return ret
-    
-    def __make_simple(self, values):
-        ret = '{'
-        for idx, value in enumerate(values):
-            if idx > 0:
-                ret += ', '
-                if idx % 32 == 0:
-                    ret += '\n'
-            ret += f'{value}'
-        ret += '}'
+            ret += '\n{{'
+            arr = format_array(block, bits=bits, max_width=200)
+            if arr.find('\n') != -1:
+                arr = '\n' + textwrap.indent(arr, '  ') + '\n'
+            ret += arr
+            ret += '}}'
         return ret
 
     def print_header(self, name, values_enum_content):
@@ -227,22 +215,22 @@ class {name}
 
     def print_impl(self, name):
         ret = f'''
-        const std::array<{type_for_bits(self.__bits_for_ascii)}, {len(self.__ascii)}> {name}::ascii({{
-            {indent_insert(self.__make_simple(self.__ascii), 12)}
-        }});
+        const std::array<{type_for_bits(self.__bits_for_ascii)}, {len(self.__ascii)}> {name}::ascii({{{{
+            {indent_insert(format_array(self.__ascii, ishex=True, bits=self.__bits_for_ascii), 12)}
+        }}}});
         '''
         for idx, values in enumerate(self.__values):
             if isinstance(values[0], int):
                 ret += f'''
-        const std::array<{type_for_bits(self.__bits_for_values[idx])}, {len(values)}> {name}::stage{idx + 1}({{
-            {indent_insert(self.__make_simple(values), 12)}
-        }});
+        const std::array<{type_for_bits(self.__bits_for_values[idx])}, {len(values)}> {name}::stage{idx + 1}({{{{
+            {indent_insert(format_array(values, ishex=idx!=0, bits=self.__bits_for_values[idx]), 12)}
+        }}}});
         '''
             else:
                 ret += f'''
-        const std::array<std::array<{type_for_bits(self.__bits_for_values[idx])}, {len(values[0])}>, {len(values)}> {name}::stage{idx + 1}({{
-            {indent_insert(self.__make_nested(values), 12)}
-        }});
+        const std::array<std::array<{type_for_bits(self.__bits_for_values[idx])}, {len(values[0])}>, {len(values)}> {name}::stage{idx + 1}({{{{
+            {indent_insert(self.__make_nested(values, bits=self.__bits_for_values[idx]), 12)}
+        }}}});
         '''