Prevent text extractors from adding unnecessary spaces

Author: mischov

lib/meeseeks/extractor/data.ex

@@ -17,9 +17,52 @@ defmodule Meeseeks.Extractor.Data do
   def from_node(%Element{id: id}, document) do
     Helpers.child_nodes(document, id)
     |> Enum.filter(&Helpers.data_node?/1)
-    |> Enum.map(&from_node(&1, document))
-    |> Enum.intersperse(" ")
+    |> join_nodes(document)
   end
 
   def from_node(%{__struct__: struct}, _) when struct in @other_nodes, do: []
+
+  # join_nodes
+
+  defp join_nodes([], _document), do: []
+
+  defp join_nodes([node], document), do: from_node(node, document)
+
+  defp join_nodes(nodes, document), do: join_nodes(nodes, [], document)
+
+  defp join_nodes([], acc, _document), do: :lists.reverse(acc)
+
+  defp join_nodes([node | nodes], acc, document) do
+    acc = join_node(node, acc, document)
+    join_nodes(nodes, acc, document)
+  end
+
+  # Head
+  defp join_node(node, [], document) do
+    case from_node(node, document) do
+      [] -> []
+      "" -> []
+      iodata -> [iodata]
+    end
+  end
+
+  # Tail
+  defp join_node(node, acc, document) do
+    case from_node(node, document) do
+      [] ->
+        acc
+
+      "" ->
+        acc
+
+      iodata ->
+        [previous | _] = acc
+
+        if Helpers.ends_in_whitespace?(previous) do
+          [iodata | acc]
+        else
+          [iodata, " " | acc]
+        end
+    end
+  end
 end

lib/meeseeks/extractor/helpers.ex

@@ -56,4 +56,72 @@ defmodule Meeseeks.Extractor.Helpers do
   defp html_escape_char("&"), do: "&"
   defp html_escape_char("\""), do: """
   defp html_escape_char("'"), do: "'"
+
+  # ends_in_whitespace?
+
+  # Adapted from trim_trailing in String.Break, which is found in
+  # elixir/unicode/properties.ex
+
+  @whitespace_max_size 3
+
+  whitespace = List.flatten([
+    Enum.map(String.to_integer("0009", 16)..String.to_integer("000D", 16), fn int -> <<int::utf8>> end),
+    <<String.to_integer("0020", 16)::utf8>>,
+    <<String.to_integer("0085", 16)::utf8>>,
+    <<String.to_integer("00A0", 16)::utf8>>,
+    <<String.to_integer("1680", 16)::utf8>>,
+    Enum.map(String.to_integer("2000", 16)..String.to_integer("200A", 16), fn int -> <<int::utf8>> end),
+    <<String.to_integer("2028", 16)::utf8>>,
+    <<String.to_integer("2029", 16)::utf8>>,
+    <<String.to_integer("202F", 16)::utf8>>,
+    <<String.to_integer("205F", 16)::utf8>>,
+    <<String.to_integer("3000", 16)::utf8>>
+  ])
+
+  def ends_in_whitespace?(iodata)
+  def ends_in_whitespace?(l) when is_list(l), do: list_ends_in_whitespace?(l)
+  def ends_in_whitespace?(b) when is_binary(b), do: bin_ends_in_whitespace?(b)
+
+  defp list_ends_in_whitespace?([]), do: false
+  defp list_ends_in_whitespace?([x]), do: ends_in_whitespace?(x)
+  defp list_ends_in_whitespace?([_, x]), do: ends_in_whitespace?(x)
+  defp list_ends_in_whitespace?([_, _, x]), do: ends_in_whitespace?(x)
+
+  defp list_ends_in_whitespace?(l) do
+    [x | _] = :lists.reverse(l)
+    ends_in_whitespace?(x)
+  end
+
+  defp bin_ends_in_whitespace?(""), do: false
+  defp bin_ends_in_whitespace?(b) do
+    bin_ends_in_whitespace?(b, byte_size(b))
+  end
+
+  defp bin_ends_in_whitespace?(b, size) when size < @whitespace_max_size do
+    s_bin_ends_in_whitespace?(b)
+  end
+
+  defp bin_ends_in_whitespace?(b, size) do
+    b_end = binary_part(b, size, -@whitespace_max_size)
+    l_bin_ends_in_whitespace?(b_end)
+  end
+
+  for cp <- whitespace do
+    case byte_size(cp) do
+      3 ->
+        defp l_bin_ends_in_whitespace?(unquote(cp)), do: true
+
+      2 ->
+        defp l_bin_ends_in_whitespace?(<<_, unquote(cp)>>), do: true
+        defp s_bin_ends_in_whitespace?(unquote(cp)), do: true
+
+      1 ->
+        defp l_bin_ends_in_whitespace?(<<_, _, unquote(cp)>>), do: true
+        defp s_bin_ends_in_whitespace?(<<_, unquote(cp)>>), do: true
+        defp s_bin_ends_in_whitespace?(unquote(cp)), do: true
+    end
+  end
+
+  defp l_bin_ends_in_whitespace?(_), do: false
+  defp s_bin_ends_in_whitespace?(_), do: false
 end

lib/meeseeks/extractor/own_text.ex

@@ -15,11 +15,54 @@ defmodule Meeseeks.Extractor.OwnText do
   def from_node(%Element{id: id}, document) do
     Helpers.child_nodes(document, id)
     |> Enum.filter(&Helpers.text_node?/1)
-    |> Enum.map(&from_node(&1, document))
-    |> Enum.intersperse(" ")
+    |> join_nodes(document)
   end
 
   def from_node(%Text{content: content}, _), do: content
 
   def from_node(%{__struct__: struct}, _) when struct in @other_nodes, do: []
+
+  # join_nodes
+
+  defp join_nodes([], _document), do: []
+
+  defp join_nodes([node], document), do: from_node(node, document)
+
+  defp join_nodes(nodes, document), do: join_nodes(nodes, [], document)
+
+  defp join_nodes([], acc, _document), do: :lists.reverse(acc)
+
+  defp join_nodes([node | nodes], acc, document) do
+    acc = join_node(node, acc, document)
+    join_nodes(nodes, acc, document)
+  end
+
+  # Head
+  defp join_node(node, [], document) do
+    case from_node(node, document) do
+      [] -> []
+      "" -> []
+      iodata -> [iodata]
+    end
+  end
+
+  # Tail
+  defp join_node(node, acc, document) do
+    case from_node(node, document) do
+      [] ->
+        acc
+
+      "" ->
+        acc
+
+      iodata ->
+        [previous | _] = acc
+
+        if Helpers.ends_in_whitespace?(previous) do
+          [iodata | acc]
+        else
+          [iodata, " " | acc]
+        end
+    end
+  end
 end

lib/meeseeks/extractor/text.ex

@@ -14,11 +14,54 @@ defmodule Meeseeks.Extractor.Text do
 
   def from_node(%Element{id: id}, document) do
     Helpers.child_nodes(document, id)
-    |> Enum.map(&from_node(&1, document))
-    |> Enum.intersperse(" ")
+    |> join_nodes(document)
   end
 
   def from_node(%Text{content: content}, _), do: content
 
   def from_node(%{__struct__: struct}, _) when struct in @other_nodes, do: []
+
+  # join_nodes
+
+  defp join_nodes([], _document), do: []
+
+  defp join_nodes([node], document), do: from_node(node, document)
+
+  defp join_nodes(nodes, document), do: join_nodes(nodes, [], document)
+
+  defp join_nodes([], acc, _document), do: :lists.reverse(acc)
+
+  defp join_nodes([node | nodes], acc, document) do
+    acc = join_node(node, acc, document)
+    join_nodes(nodes, acc, document)
+  end
+
+  # Head
+  defp join_node(node, [], document) do
+    case from_node(node, document) do
+      [] -> []
+      "" -> []
+      iodata -> [iodata]
+    end
+  end
+
+  # Tail
+  defp join_node(node, acc, document) do
+    case from_node(node, document) do
+      [] ->
+        acc
+
+      "" ->
+        acc
+
+      iodata ->
+        [previous | _] = acc
+
+        if Helpers.ends_in_whitespace?(previous) do
+          [iodata | acc]
+        else
+          [iodata, " " | acc]
+        end
+    end
+  end
 end

test/meeseeks/document/node_test.exs

@@ -65,7 +65,7 @@ defmodule Meeseeks.Document.NodeTest do
 
   test "get data without collapsing whitespace when node has data" do
     node = Document.get_node(@document, 7)
-    expected = "3  4   5"
+    expected = "3 4   5"
     assert Node.data(node, @document, collapse_whitespace: false) == expected
   end
 
@@ -142,7 +142,7 @@ defmodule Meeseeks.Document.NodeTest do
 
   test "get text without collapsing whitespace when node has text" do
     node = Document.get_node(@document, 7)
-    expected = "0   0.5\n        1"
+    expected = "0   0.5\n       1"
     assert Node.text(node, @document, collapse_whitespace: false) == expected
   end