Merge branch 'master' into smallvec

Author: sujayakar

Cargo.toml

@@ -18,7 +18,7 @@ Decomposition and Recomposition, as described in
 Unicode Standard Annex #15.
 """
 
-exclude = [ "target/*", "Cargo.lock", "scripts/tmp", "*.txt" ]
+exclude = [ "target/*", "Cargo.lock", "scripts/tmp", "*.txt", "src/normalization_tests.rs", "src/test.rs" ]
 
 [dependencies]
-smallvec = "0.6"
+smallvec = "0.6"

scripts/unicode.py

@@ -315,14 +315,19 @@ def gen_decomposition_tables(canon_decomp, compat_decomp, out):
     for table, name in tables:
         out.write("#[inline]\n")
         out.write("pub fn %s_fully_decomposed(c: char) -> Option<&'static [char]> {\n" % name)
-        out.write("    match c {\n")
+        # The "Some" constructor is around the match statement here, because
+        # putting it into the individual arms would make the item_bodies
+        # checking of rustc takes almost twice as long, and it's already pretty
+        # slow because of the huge number of match arms and the fact that there
+        # is a borrow inside each arm
+        out.write("    Some(match c {\n")
 
         for char, chars in sorted(table.items()):
             d = ", ".join("'\u{%s}'" % hexify(c) for c in chars)
-            out.write("        '\u{%s}' => Some(&[%s]),\n" % (hexify(char), d))
+            out.write("        '\u{%s}' => &[%s],\n" % (hexify(char), d))
 
-        out.write("        _ => None,\n")
-        out.write("    }\n")
+        out.write("        _ => return None,\n")
+        out.write("    })\n")
         out.write("}\n")
         out.write("\n")
 
@@ -347,12 +352,24 @@ def gen_nfc_qc(prop_tables, out):
     gen_qc_match(prop_tables['NFC_QC'], out)
     out.write("}\n")
 
+def gen_nfkc_qc(prop_tables, out):
+    out.write("#[inline]\n")
+    out.write("pub fn qc_nfkc(c: char) -> IsNormalized {\n")
+    gen_qc_match(prop_tables['NFKC_QC'], out)
+    out.write("}\n")
+
 def gen_nfd_qc(prop_tables, out):
     out.write("#[inline]\n")
     out.write("pub fn qc_nfd(c: char) -> IsNormalized {\n")
     gen_qc_match(prop_tables['NFD_QC'], out)
     out.write("}\n")
 
+def gen_nfkd_qc(prop_tables, out):
+    out.write("#[inline]\n")
+    out.write("pub fn qc_nfkd(c: char) -> IsNormalized {\n")
+    gen_qc_match(prop_tables['NFKD_QC'], out)
+    out.write("}\n")
+
 def gen_combining_mark(general_category_mark, out):
     out.write("#[inline]\n")
     out.write("pub fn is_combining_mark(c: char) -> bool {\n")
@@ -441,9 +458,15 @@ def gen_tests(tests, out):
         gen_nfc_qc(data.norm_props, out)
         out.write("\n")
 
+        gen_nfkc_qc(data.norm_props, out)
+        out.write("\n")
+
         gen_nfd_qc(data.norm_props, out)
         out.write("\n")
 
+        gen_nfkd_qc(data.norm_props, out)
+        out.write("\n")
+
         gen_stream_safe(data.ss_leading, data.ss_trailing, out)
         out.write("\n")
 

src/lib.rs

@@ -49,10 +49,14 @@ pub use quick_check::{
     IsNormalized,
     is_nfc,
     is_nfc_quick,
+    is_nfkc,
+    is_nfkc_quick,
     is_nfc_stream_safe,
     is_nfc_stream_safe_quick,
     is_nfd,
     is_nfd_quick,
+    is_nfkd,
+    is_nfkd_quick,
     is_nfd_stream_safe,
     is_nfd_stream_safe_quick,
 };

src/quick_check.rs

@@ -70,12 +70,25 @@ pub fn is_nfc_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
     quick_check(s, tables::qc_nfc, false)
 }
 
+
+/// Quickly check if a string is in NFKC.
+#[inline]
+pub fn is_nfkc_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
+    quick_check(s, tables::qc_nfkc, false)
+}
+
 /// Quickly check if a string is in NFD.
 #[inline]
 pub fn is_nfd_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
     quick_check(s, tables::qc_nfd, false)
 }
 
+/// Quickly check if a string is in NFKD.
+#[inline]
+pub fn is_nfkd_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
+    quick_check(s, tables::qc_nfkd, false)
+}
+
 /// Quickly check if a string is Stream-Safe NFC.
 #[inline]
 pub fn is_nfc_stream_safe_quick<I: Iterator<Item=char>>(s: I) -> IsNormalized {
@@ -98,6 +111,16 @@ pub fn is_nfc(s: &str) -> bool {
     }
 }
 
+/// Authoritatively check if a string is in NFKC.
+#[inline]
+pub fn is_nfkc(s: &str) -> bool {
+    match is_nfkc_quick(s.chars()) {
+        IsNormalized::Yes => true,
+        IsNormalized::No => false,
+        IsNormalized::Maybe => s.chars().eq(s.chars().nfkc()),
+    }
+}
+
 /// Authoritatively check if a string is in NFD.
 #[inline]
 pub fn is_nfd(s: &str) -> bool {
@@ -108,6 +131,16 @@ pub fn is_nfd(s: &str) -> bool {
     }
 }
 
+/// Authoritatively check if a string is in NFKD.
+#[inline]
+pub fn is_nfkd(s: &str) -> bool {
+    match is_nfkd_quick(s.chars()) {
+        IsNormalized::Yes => true,
+        IsNormalized::No => false,
+        IsNormalized::Maybe => s.chars().eq(s.chars().nfkd()),
+    }
+}
+
 /// Authoritatively check if a string is Stream-Safe NFC.
 #[inline]
 pub fn is_nfc_stream_safe(s: &str) -> bool {