Backporting Unicode data improvements and normalization

Author: gershnik

CMakeLists.txt

@@ -16,9 +16,43 @@ string(STRIP ${SYSSTR_VERSION} SYSSTR_VERSION)
 
 project(sys_string VERSION ${SYSSTR_VERSION} LANGUAGES CXX)
 
+find_package (Python3 COMPONENTS Interpreter Development)
+
 
 add_subdirectory(lib)
 
+if(${Python3_Interpreter_FOUND})
+
+    file(GLOB UNICODE_DATA ${CMAKE_CURRENT_LIST_DIR}/unicode/data/*.txt)
+    file(GLOB UNICODE_SCRIPTS ${CMAKE_CURRENT_LIST_DIR}/unicode/scripts/*.py)
+
+    set(UNICODE_GENERATED_FILES
+        ${CMAKE_CURRENT_LIST_DIR}/lib/src/unicode_mappings.cpp 
+        ${CMAKE_CURRENT_LIST_DIR}/lib/inc/sys_string/impl/unicode/mappings.h
+        ${CMAKE_CURRENT_LIST_DIR}/test/test_normalization_data.h
+    )
+
+    add_custom_command(
+        COMMENT "Generating Unicoode mappings"
+        OUTPUT ${UNICODE_GENERATED_FILES}
+        COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/lib/scripts/genmappings.py 
+                        ${CMAKE_CURRENT_LIST_DIR}/lib/res 
+                        ${UNICODE_GENERATED_FILES}
+        DEPENDS 
+            ${UNICODE_DATA} 
+            ${UNICODE_SCRIPTS}
+    )
+
+    add_custom_target(generate_unicode_mappings
+        DEPENDS 
+            ${UNICODE_GENERATED_FILES}
+    )
+
+    add_dependencies(sys_string generate_unicode_mappings)
+
+endif()
+
+
 if (PROJECT_IS_TOP_LEVEL)
 
     include(lib/cmake/install.cmake)

lib/CMakeLists.txt

@@ -17,8 +17,6 @@ if (NOT SYSSTR_VERSION)
 endif()
 project(sys_string VERSION ${SYSSTR_VERSION})
 
-find_package (Python3 COMPONENTS Interpreter)
-
 set(SRCDIR ${CMAKE_CURRENT_LIST_DIR})
 set(LIBNAME sys_string)
 
@@ -94,6 +92,7 @@ set(UNICODE_FILES
     ${SRCDIR}/inc/sys_string/impl/unicode/utf_util.h
     ${SRCDIR}/inc/sys_string/impl/unicode/algorithms.h
     ${SRCDIR}/inc/sys_string/impl/unicode/mappings.h
+    ${SRCDIR}/inc/sys_string/impl/unicode/mappings_common.h
 )
 set(SOURCES
     ${SRCDIR}/src/unicode_mappings.cpp
@@ -129,22 +128,6 @@ PRIVATE
 add_library(${LIBNAME}::${LIBNAME} ALIAS ${LIBNAME})
 
 
-if(${Python3_Interpreter_FOUND})
-
-    file(GLOB UNICODE_DATA ${SRCDIR}/res/*.txt)
-    file(GLOB UNICODE_SCRIPTS ${SRCDIR}/scripts/*.py)
-
-    add_custom_command(
-        COMMENT "Generating Unicoode mappings"
-        OUTPUT ${SRCDIR}/src/unicode_mappings.cpp
-        COMMAND ${Python3_EXECUTABLE} ${SRCDIR}/scripts/genmappings.py ${SRCDIR}/res ${SRCDIR}/src/unicode_mappings.cpp 
-        DEPENDS 
-            ${UNICODE_DATA} 
-            ${UNICODE_SCRIPTS}
-    )
-
-endif()
-
 if (PROJECT_IS_TOP_LEVEL)
 
     include(cmake/install.cmake)

lib/inc/sys_string/impl/misc.h

@@ -78,6 +78,43 @@ namespace sysstr::util
 
         return std::equal(test.begin(), test.end(), access.begin());
     }
+
+    template<class Storage>
+    struct transform_sink
+    {
+        const typename sys_string_t<Storage>::utf32_view & access;
+        decltype(std::declval<sys_string_builder_t<Storage>>().chars()) chars;
+        
+        
+        SYS_STRING_FORCE_INLINE
+        void copy(typename sys_string_t<Storage>::utf32_view::iterator it)
+        {
+            auto cur = it.storage_cursor();
+            auto size = it.storage_size();
+            
+            for (decltype(size) i = 0; i < size; ++i, ++cur)
+                this->chars.push_back(*cur);
+        }
+        
+        SYS_STRING_FORCE_INLINE
+        void copy(typename sys_string_t<Storage>::utf32_view::iterator first, typename sys_string_t<Storage>::utf32_view::iterator last)
+        {
+            if (chars.empty())
+            {
+                if (first == access.begin() && last == access.end())
+                    return;
+            }
+            auto fcur = first.storage_cursor();
+            auto lcur = last.storage_cursor();
+            
+            for ( ; fcur != lcur; ++fcur)
+                this->chars.push_back(*fcur);
+        }
+        SYS_STRING_FORCE_INLINE
+        void write(char32_t c)
+            { write_unsafe<utf_encoding_of<typename sys_string_t<Storage>::storage_type>>(c, std::back_inserter(chars)); }
+        
+    };
 }
 
 namespace sysstr
@@ -140,6 +177,28 @@ auto sysstr::sys_string_t<Storage>::to_upper() const -> sys_string_t<Storage>
     return builder.build();
 }
 
+template<class Storage>
+inline
+auto sysstr::sys_string_t<Storage>::normalize(normalization norm) const -> sys_string_t
+{
+    
+    sys_string_builder_t<Storage> builder;
+    sys_string_t<Storage>::utf32_view access(*this);
+    switch (norm)
+    {
+        break; case normalization::nfd:
+            normalize::nfd<utf_encoding_of<storage_type>>()(access.begin(), access.end(), std::back_inserter(builder.chars()));
+        break; case normalization::nfc:
+        {
+            util::transform_sink<Storage> sink{access, builder.chars()};
+            normalize::nfc<utf_encoding_of<storage_type>>().call_with_sink(access.begin(), access.end(), sink);
+            if (sink.chars.empty())
+                return *this;
+        }
+        break; default: return *this;
+    }
+    return builder.build();
+}
 
 template<class Storage>
 template<class Pred>