store smaller slices in unicode data tables

Rust's default slices are convenient, but for tables like:

  const f: &'static [(char, &'static [char])]

they take up far too much space.  An element of the above array consumes
24 bytes on 64-bit platforms, and unicode-normalization contains about
6000 such array elements.

A better approach is to manually store a smaller slice type:

  struct Slice {
    offset: u16,
    length: u16,
  }

  const f: &'static [(char, Slice)]

and store the actual character data in a separate array on the side.
The `Slice` structures then point in to this separate array, but at a
much smaller space cost: elements of the modified `f` take up only 8
bytes on 64-bit platforms, which implies a space savings of ~96K on
64-bit platforms.  On some systems, this strategy also eliminates the
necessity of run-time relocations, which can be a further, significant
savings in binary size and runtime cost.

This change is strictly local to the library; it does not affect the
public API.

Author: froydnj

scripts/unicode.py

@@ -18,7 +18,7 @@
 # Since this should not require frequent updates, we just store this
 # out-of-line and check the unicode.rs file into git.
 
-import fileinput, re, os, sys
+import fileinput, re, os, sys, collections
 
 preamble = '''// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
 // file at the top-level directory of this distribution and at
@@ -213,6 +213,43 @@ def emit_table(f, name, t_data, t_type = "&'static [(char, char)]", is_pub=True,
     format_table_content(f, [pfun(d) for d in t_data], 8)
     f.write("\n    ];\n\n")
 
+def emit_strtab_table(f, name, keys, vfun, is_pub=True,
+                      tab_entry_type='char', slice_element_sfun=escape_char):
+    pub_string = ""
+    if is_pub:
+        pub_string = "pub "
+    f.write("    %s const %s: &'static [(char, Slice)] = &[\n"
+            % (pub_string, name))
+
+    strtab = collections.OrderedDict()
+    strtab_offset = 0
+
+    # TODO: a more sophisticated algorithm here would not only check for the
+    # existence of v in the strtab, but also v in contiguous substrings of
+    # strtab, if that's possible.
+    for k in keys:
+        v = tuple(vfun(k))
+        if v in strtab:
+            item_slice = strtab[v]
+        else:
+            value_len = len(v)
+            item_slice = (strtab_offset, value_len)
+            strtab[v] = item_slice
+            strtab_offset += value_len
+
+        f.write("%s(%s, Slice { offset: %d, length: %d }),\n"
+                % (" "*8, escape_char(k), item_slice[0], item_slice[1]))
+
+    f.write("\n    ];\n\n")
+
+    f.write("    %s const %s_STRTAB: &'static [%s] = &[\n"
+            % (pub_string, name, tab_entry_type))
+
+    for (v, _) in strtab.iteritems():
+        f.write("%s%s,\n" % (" "*8, ', '.join(slice_element_sfun(c) for c in v)))
+
+    f.write("\n    ];\n\n")
+
 def emit_norm_module(f, canon, compat, combine, norm_props, general_category_mark):
     canon_keys = canon.keys()
     canon_keys.sort()
@@ -234,35 +271,38 @@ def emit_norm_module(f, canon, compat, combine, norm_props, general_category_mar
     canon_comp_keys.sort()
 
     f.write("pub mod normalization {\n")
+    f.write("""
+pub struct Slice {
+  pub offset: u16,
+  pub length: u16,
+}
+""")
 
     def mkdata_fun(table):
         def f(char):
-            data = "(%s,&[" % escape_char(char)
-            chars = [escape_char(d) for d in table[char]]
-            data += ','.join(chars)
-            data += "])"
-            return data
+            return table[char]
         return f
 
+    # TODO: should the strtab of these two tables be of type &'static str, for
+    # smaller data?
     f.write("    // Canonical decompositions\n")
-    emit_table(f, "canonical_table", canon_keys, "&'static [(char, &'static [char])]",
-        pfun=mkdata_fun(canon))
+    emit_strtab_table(f, "canonical_table", canon_keys,
+                      vfun=mkdata_fun(canon))
 
     f.write("    // Compatibility decompositions\n")
-    emit_table(f, "compatibility_table", compat_keys, "&'static [(char, &'static [char])]",
-        pfun=mkdata_fun(compat))
+    emit_strtab_table(f, "compatibility_table", compat_keys,
+                      vfun=mkdata_fun(compat))
 
-    def comp_pfun(char):
-        data = "(%s,&[" % escape_char(char)
-        canon_comp[char].sort(lambda x, y: x[0] - y[0])
-        data += ','.join("(%s,%s)" % (escape_char(pair[0]), escape_char(pair[1]))
-                         for pair in canon_comp[char])
-        data += "])"
-        return data
+    def comp_vfun(char):
+        return sorted(canon_comp[char], lambda x, y: x[0] - y[0])
 
     f.write("    // Canonical compositions\n")
-    emit_table(f, "composition_table", canon_comp_keys,
-        "&'static [(char, &'static [(char, char)])]", pfun=comp_pfun)
+    # "&'static [(char, &'static [(char, char)])]", pfun=comp_pfun)
+    emit_strtab_table(f, "composition_table", canon_comp_keys,
+                      vfun=comp_vfun,
+                      tab_entry_type="(char, char)",
+                      slice_element_sfun=lambda pair: "(%s,%s)" % (escape_char(pair[0]),
+                                                                   escape_char(pair[1])))
 
     f.write("""
     fn bsearch_range_value_table(c: char, r: &'static [(char, char, u8)]) -> u8 {

src/normalize.rs

@@ -12,17 +12,22 @@
 
 use std::cmp::Ordering::{Equal, Less, Greater};
 use std::ops::FnMut;
-use tables::normalization::{canonical_table, compatibility_table, composition_table};
+use tables::normalization::{canonical_table, canonical_table_STRTAB};
+use tables::normalization::{compatibility_table, compatibility_table_STRTAB};
+use tables::normalization::{composition_table, composition_table_STRTAB};
+use tables::normalization::Slice;
 
-fn bsearch_table<T>(c: char, r: &'static [(char, &'static [T])]) -> Option<&'static [T]> {
+fn bsearch_table<T>(c: char, r: &'static [(char, Slice)], strtab: &'static [T]) -> Option<&'static [T]> {
     match r.binary_search_by(|&(val, _)| {
         if c == val { Equal }
         else if val < c { Less }
         else { Greater }
     }) {
         Ok(idx) => {
-            let (_, result) = r[idx];
-            Some(result)
+            let ref slice = r[idx].1;
+            let offset = slice.offset as usize;
+            let length = slice.length as usize;
+            Some(&strtab[offset..(offset + length)])
         }
         Err(_) => None
     }
@@ -50,7 +55,7 @@ fn d<F>(c: char, i: &mut F, k: bool) where F: FnMut(char) {
     }
 
     // First check the canonical decompositions
-    match bsearch_table(c, canonical_table) {
+    match bsearch_table(c, canonical_table, canonical_table_STRTAB) {
         Some(canon) => {
             for x in canon {
                 d(*x, i, k);
@@ -64,7 +69,7 @@ fn d<F>(c: char, i: &mut F, k: bool) where F: FnMut(char) {
     if !k { (*i)(c); return; }
 
     // Then check the compatibility decompositions
-    match bsearch_table(c, compatibility_table) {
+    match bsearch_table(c, compatibility_table, compatibility_table_STRTAB) {
         Some(compat) => {
             for x in compat {
                 d(*x, i, k);
@@ -83,7 +88,7 @@ fn d<F>(c: char, i: &mut F, k: bool) where F: FnMut(char) {
 /// for more information.
 pub fn compose(a: char, b: char) -> Option<char> {
     compose_hangul(a, b).or_else(|| {
-        match bsearch_table(a, composition_table) {
+        match bsearch_table(a, composition_table, composition_table_STRTAB) {
             None => None,
             Some(candidates) => {
                 match candidates.binary_search_by(|&(val, _)| {