Add textDocument/foldingRange Provider (#492)

* initial commit

* can fold heredoc w/ closing paren

* fix indentation

* save progress (this is a mess...)

* (potentially?) working version

* the code caught a problem with a test!

* clean up a little

* add some explanatory comments

* round out functionality

* fix some off-by-1 nonsense

* grammar

* remember to add cur to the stack in all cases!

* adjust test

* whoo! case reduction!

* shorten comment

* tweaks

* better formatting

* add token-pairs module; start adding typespecs

* refactor token-pairs

* fix example

* refactor indentation tests

* refactor indentation tests

* remove unused function

* passing tests!

* fix return types

* add unusual indentation and end-to-end tests

* fix merging logic

* update test names

* remove todo

* add heredoc support

* create a unified input

* move function to helpers

* add support for comment blocks

* don't allow single line comment blocks

* try not to sort multiple times; fix test

* note issue with implementation

* include carriage returns in line-splitting logic

Co-authored-by: Łukasz Samson <lukaszsamson@gmail.com>

* use get_source_file function

* combine Enum.maps; add comment

* add `for` and `case`; add comments for clarity

* attempt to deal with utf8/16 length calculations

* fix misunderstanding and use :block_identifier

* speling is hardd

Co-authored-by: Étienne Lévesque <etienne.levesque16@gmail.com>

* make group_comments/1 a defp; add @specs

* replace a nested-reduce with a flat_map + group_by

* drop kind_map, use @token_pairs directly

* refactor, change name, add @specs

* use pipes

* fix warning

* add support for charlist heredocs

* add binary support

* tweak test approach to allow specifying range kind

* change heredoc pass to "special token" pass; add/modify tests

* change filename

* change to singular module name for consistency

* remove outdated comments

* remove debug function

* (hopefully) cover older versions of tokenize

* documentation; harmless refactor

* Switch to doctests

Also add a means to visualize folding range changes in the tests (only
shows when there is a test failure)

* Update apps/language_server/lib/language_server/providers/folding_range/token_pairs.ex

Co-authored-by: Łukasz Samson <lukaszsamson@gmail.com>
Co-authored-by: Étienne Lévesque <etienne.levesque16@gmail.com>
Co-authored-by: Jason Axelson <jason.axelson@gmail.com>
Co-authored-by: Jason Axelson <axelson@users.noreply.github.com>

Author: 4 people

apps/language_server/lib/language_server/protocol.ex

@@ -190,6 +190,16 @@ defmodule ElixirLS.LanguageServer.Protocol do
     end
   end
 
+  defmacro folding_range_req(id, uri) do
+    quote do
+      request(unquote(id), "textDocument/foldingRange", %{
+        "textDocument" => %{
+          "uri" => unquote(uri)
+        }
+      })
+    end
+  end
+
   # TODO remove in ElixirLS 0.8
   defmacro macro_expansion(id, whole_buffer, selected_macro, macro_line) do
     quote do

apps/language_server/lib/language_server/providers/folding_range.ex

@@ -0,0 +1,145 @@
+defmodule ElixirLS.LanguageServer.Providers.FoldingRange do
+  @moduledoc """
+  A textDocument/foldingRange provider implementation.
+
+  ## Background
+
+  See specification here:
+
+  https://microsoft.github.io/language-server-protocol/specifications/specification-3-15/#textDocument_foldingRange
+
+  ## Methodology
+
+  ### High level
+
+  We make multiple passes (currently 4) through the source text and create
+  folding ranges from each pass.
+  Then we merge the ranges from each pass to provide the final ranges.
+  Each pass gets a priority to help break ties (the priority is an integer,
+  higher integers win).
+
+  ### Indentation pass (priority: 1)
+
+  We use the indentation level -- determined by the column of the first
+  non-whitespace character on each line -- to provide baseline ranges.
+  All ranges from this pass are `kind?: :region` ranges.
+
+  ### Comment block pass (priority: 2)
+
+  We let "comment blocks", consecutive lines starting with `#`, from regions.
+  All ranges from this pass are `kind?: :comment` ranges.
+
+  ### Token-pairs pass (priority: 3)
+
+  We use pairs of tokens, e.g. `do` and `end`, to provide another pass of
+  ranges.
+  All ranges from this pass are `kind?: :region` ranges.
+
+  ### Special tokens pass (priority: 3)
+
+  We find strings (regular/charlist strings/heredocs) and sigils in a pass as
+  they're delimited by a few special tokens.
+  Ranges from this pass are either
+  - `kind?: :comment` if the token is paired with `@doc` or `@moduledoc`, or
+  - `kind?: :region` otherwise.
+
+  ## Notes
+
+  Each pass may return ranges in any order.
+  But all ranges are valid, i.e. endLine > startLine.
+  """
+
+  alias __MODULE__
+
+  @type input :: %{
+          tokens: [FoldingRange.Token.t()],
+          lines: [FoldingRange.Line.t()]
+        }
+
+  @type t :: %{
+          required(:startLine) => non_neg_integer(),
+          required(:endLine) => non_neg_integer(),
+          optional(:startCharacter?) => non_neg_integer(),
+          optional(:endCharacter?) => non_neg_integer(),
+          optional(:kind?) => :comment | :imports | :region
+        }
+
+  @doc """
+  Provides folding ranges for a source file
+
+  ## Example
+
+      iex> alias ElixirLS.LanguageServer.Providers.FoldingRange
+      iex> text = \"""
+      ...> defmodule A do    # 0
+      ...>   def hello() do  # 1
+      ...>     :world        # 2
+      ...>   end             # 3
+      ...> end               # 4
+      ...> \"""
+      iex> FoldingRange.provide(%{text: text})
+      {:ok, [
+        %{startLine: 0, endLine: 3, kind?: :region},
+        %{startLine: 1, endLine: 2, kind?: :region}
+      ]}
+
+  """
+  @spec provide(%{text: String.t()}) :: {:ok, [t()]} | {:error, String.t()}
+  def provide(%{text: text}) do
+    do_provide(text)
+  end
+
+  def provide(not_a_source_file) do
+    {:error, "Expected a source file, found: #{inspect(not_a_source_file)}"}
+  end
+
+  defp do_provide(text) do
+    input = convert_text_to_input(text)
+
+    passes_with_priority = [
+      {1, FoldingRange.Indentation},
+      {2, FoldingRange.CommentBlock},
+      {3, FoldingRange.TokenPair},
+      {3, FoldingRange.SpecialToken}
+    ]
+
+    ranges =
+      passes_with_priority
+      |> Enum.map(fn {priority, pass} ->
+        ranges = ranges_from_pass(pass, input)
+        {priority, ranges}
+      end)
+      |> merge_ranges_with_priorities()
+
+    {:ok, ranges}
+  end
+
+  def convert_text_to_input(text) do
+    %{
+      tokens: FoldingRange.Token.format_string(text),
+      lines: FoldingRange.Line.format_string(text)
+    }
+  end
+
+  defp ranges_from_pass(pass, input) do
+    with {:ok, ranges} <- pass.provide_ranges(input) do
+      ranges
+    else
+      _ -> []
+    end
+  end
+
+  defp merge_ranges_with_priorities(range_lists_with_priorities) do
+    range_lists_with_priorities
+    |> Enum.flat_map(fn {priority, ranges} -> Enum.zip(Stream.cycle([priority]), ranges) end)
+    |> Enum.group_by(fn {_priority, range} -> range.startLine end)
+    |> Enum.map(fn {_start, ranges_with_priority} ->
+      {_priority, range} =
+        ranges_with_priority
+        |> Enum.max_by(fn {priority, range} -> {priority, range.endLine} end)
+
+      range
+    end)
+    |> Enum.sort_by(& &1.startLine)
+  end
+end

apps/language_server/lib/language_server/providers/folding_range/comment_block.ex

@@ -0,0 +1,78 @@
+defmodule ElixirLS.LanguageServer.Providers.FoldingRange.CommentBlock do
+  @moduledoc """
+  Code folding based on comment blocks
+
+  Note that this implementation can create comment ranges inside heredocs.
+  It's a little sloppy, but it shouldn't be very impactful.
+  We'd have to merge the token and line representations of the source text to
+  mitigate this issue, so we've left it as is for now.
+  """
+
+  alias ElixirLS.LanguageServer.Providers.FoldingRange
+  alias ElixirLS.LanguageServer.Providers.FoldingRange.Line
+
+  @doc """
+  Provides ranges for the source text based on comment blocks.
+
+  ## Example
+
+      iex> alias ElixirLS.LanguageServer.Providers.FoldingRange
+      iex> text = \"""
+      ...>   defmodule SomeModule do   # 0
+      ...>     def some_function() do  # 1
+      ...>       # I'm                 # 2
+      ...>       # a                   # 3
+      ...>       # comment block       # 4
+      ...>       nil                   # 5
+      ...>     end                     # 6
+      ...>   end                       # 7
+      ...>   \"""
+      iex> FoldingRange.convert_text_to_input(text)
+      iex> |> CommentBlock.provide_ranges()
+      {:ok, [
+        %{startLine: 2, endLine: 4, kind?: :comment}
+      ]}
+  """
+  @spec provide_ranges(FoldingRange.input()) :: {:ok, [FoldingRange.t()]}
+  def provide_ranges(%{lines: lines}) do
+    ranges =
+      lines
+      |> group_comments()
+      |> Enum.map(&convert_comment_group_to_range/1)
+
+    {:ok, ranges}
+  end
+
+  @spec group_comments([Line.t()]) :: [{Line.cell(), String.t()}]
+  defp group_comments(lines) do
+    lines
+    |> Enum.reduce([[]], fn
+      {_, cell, "#"}, [[{_, "#"} | _] = head | tail] ->
+        [[{cell, "#"} | head] | tail]
+
+      {_, cell, "#"}, [[] | tail] ->
+        [[{cell, "#"}] | tail]
+
+      _, [[{_, "#"} | _] | _] = acc ->
+        [[] | acc]
+
+      _, acc ->
+        acc
+    end)
+    |> Enum.filter(fn group -> length(group) > 1 end)
+  end
+
+  @spec group_comments([{Line.cell(), String.t()}]) :: [FoldingRange.t()]
+  defp convert_comment_group_to_range(group) do
+    {{{end_line, _}, _}, {{start_line, _}, _}} =
+      group |> FoldingRange.Helpers.first_and_last_of_list()
+
+    %{
+      startLine: start_line,
+      # We're not doing end_line - 1 on purpose.
+      # It seems weird to show the first _and_ last line of a comment block.
+      endLine: end_line,
+      kind?: :comment
+    }
+  end
+end

apps/language_server/lib/language_server/providers/folding_range/helpers.ex

@@ -0,0 +1,14 @@
+defmodule ElixirLS.LanguageServer.Providers.FoldingRange.Helpers do
+  @moduledoc false
+
+  def first_and_last_of_list([]), do: :empty_list
+
+  def first_and_last_of_list([head | tail]) do
+    tail
+    |> List.last()
+    |> case do
+      nil -> {head, head}
+      last -> {head, last}
+    end
+  end
+end

apps/language_server/lib/language_server/providers/folding_range/indentation.ex

@@ -0,0 +1,113 @@
+defmodule ElixirLS.LanguageServer.Providers.FoldingRange.Indentation do
+  @moduledoc """
+  Code folding based on indentation level
+
+  Note that we trim trailing empty rows from regions.
+  See the example.
+  """
+
+  alias ElixirLS.LanguageServer.Providers.FoldingRange
+  alias ElixirLS.LanguageServer.Providers.FoldingRange.Line
+
+  @doc """
+  Provides ranges for the source text based on the indentation level.
+
+  ## Example
+
+      iex> alias ElixirLS.LanguageServer.Providers.FoldingRange
+      iex> text = \"""
+      ...> defmodule A do                      # 0
+      ...>   def get_info(args) do             # 1
+      ...>     org =                           # 2
+      ...>       args                          # 3
+      ...>       |> Ecto.assoc(:organization)  # 4
+      ...>       |> Repo.one!()                # 5
+      ...>
+      ...>     user =                          # 7
+      ...>       org                           # 8
+      ...>       |> Organization.user!()       # 9
+      ...>
+      ...>     {:ok, %{org: org, user: user}}  # 11
+      ...>   end                               # 12
+      ...> end                                 # 13
+      ...> \"""
+      iex> FoldingRange.convert_text_to_input(text)
+      ...> |> FoldingRange.Indentation.provide_ranges()
+      {:ok, [
+        %{startLine: 0, endLine: 12, kind?: :region},
+        %{startLine: 1, endLine: 11, kind?: :region},
+        %{startLine: 7, endLine: 9, kind?: :region},
+        %{startLine: 2, endLine: 5, kind?: :region},
+      ]}
+
+  Note that the empty lines 6 and 10 do not appear in the inner most ranges.
+  """
+  @spec provide_ranges(FoldingRange.input()) :: {:ok, [FoldingRange.t()]}
+  def provide_ranges(%{lines: lines}) do
+    ranges =
+      lines
+      |> Enum.map(&extract_cell/1)
+      |> pair_cells()
+      |> pairs_to_ranges()
+
+    {:ok, ranges}
+  end
+
+  defp extract_cell({_line, cell, _first}), do: cell
+
+  @doc """
+  Pairs cells into {start, end} tuples of regions
+  Public function for testing
+  """
+  @spec pair_cells([Line.cell()]) :: [{Line.cell(), Line.cell()}]
+  def pair_cells(cells) do
+    do_pair_cells(cells, [], [], [])
+  end
+
+  # Base case
+  defp do_pair_cells([], _, _, pairs) do
+    pairs
+    |> Enum.map(fn
+      {cell1, cell2, []} -> {cell1, cell2}
+      {cell1, _, empties} -> {cell1, List.last(empties)}
+    end)
+    |> Enum.reject(fn {{r1, _}, {r2, _}} -> r1 + 1 >= r2 end)
+  end
+
+  # Empty row
+  defp do_pair_cells([{_, nil} = head | tail], stack, empties, pairs) do
+    do_pair_cells(tail, stack, [head | empties], pairs)
+  end
+
+  # Empty stack
+  defp do_pair_cells([head | tail], [], empties, pairs) do
+    do_pair_cells(tail, [head], empties, pairs)
+  end
+
+  # Non-empty stack: head is to the right of the top of the stack
+  defp do_pair_cells([{_, x} = head | tail], [{_, y} | _] = stack, _, pairs) when x > y do
+    do_pair_cells(tail, [head | stack], [], pairs)
+  end
+
+  # Non-empty stack: head is equal to or to the left of the top of the stack
+  defp do_pair_cells([{_, x} = head | tail], stack, empties, pairs) do
+    # If the head is <= to the top of the stack, then we need to pair it with
+    # everything on the stack to the right of it.
+    # The head can also start a new region, so it's pushed onto the stack.
+    {leftovers, new_tail_stack} = stack |> Enum.split_while(fn {_, y} -> x <= y end)
+    new_pairs = leftovers |> Enum.map(&{&1, head, empties})
+    do_pair_cells(tail, [head | new_tail_stack], [], new_pairs ++ pairs)
+  end
+
+  @spec pairs_to_ranges([{Line.cell(), Line.cell()}]) :: [FoldingRange.t()]
+  defp pairs_to_ranges(pairs) do
+    pairs
+    |> Enum.map(fn {{r1, _}, {r2, _}} ->
+      %{
+        startLine: r1,
+        endLine: r2 - 1,
+        kind?: :region
+      }
+    end)
+  end
+end