Format ranges using UTF-16 string representation (#200)

* Format ranges using UTF-16 string representation

* fix formatting

Co-authored-by: Jason Axelson <jason.axelson@gmail.com>

Author: miXwui

apps/language_server/lib/language_server/providers/formatting.ex

@@ -79,7 +79,9 @@ defmodule ElixirLS.LanguageServer.Providers.Formatting do
       if char == "\n" do
         {line + 1, 0}
       else
-        {line, col + 1}
+        # LSP contentChanges positions are based on UTF-16 string representation
+        # https://microsoft.github.io/language-server-protocol/specification#textDocuments
+        {line, col + byte_size(:unicode.characters_to_binary(char, :utf8, :utf16)) / 2}
       end
     end)
   end

apps/language_server/lib/language_server/source_file.ex

@@ -83,7 +83,14 @@ defmodule ElixirLS.LanguageServer.SourceFile do
             [[line, ?\n] | acc]
 
           idx == start_line ->
-            [String.slice(line, 0, start_character) | acc]
+            # LSP contentChanges positions are based on UTF-16 string representation
+            # https://microsoft.github.io/language-server-protocol/specification#textDocuments
+            beginning_utf8 =
+              :unicode.characters_to_binary(line, :utf8, :utf16)
+              |> binary_part(0, start_character * 2)
+              |> :unicode.characters_to_binary(:utf16, :utf8)
+
+            [beginning_utf8 | acc]
 
           idx > start_line ->
             acc
@@ -99,7 +106,18 @@ defmodule ElixirLS.LanguageServer.SourceFile do
             acc
 
           idx == end_line ->
-            [[String.slice(line, end_character..-1), ?\n] | acc]
+            # LSP contentChanges positions are based on UTF-16 string representation
+            # https://microsoft.github.io/language-server-protocol/specification#textDocuments
+            ending_utf8 =
+              :unicode.characters_to_binary(line, :utf8, :utf16)
+              |> (&binary_part(
+                    &1,
+                    end_character * 2,
+                    byte_size(&1) - end_character * 2
+                  )).()
+              |> :unicode.characters_to_binary(:utf16, :utf8)
+
+            [[ending_utf8, ?\n] | acc]
 
           idx > end_line ->
             [[line, ?\n] | acc]

apps/language_server/test/providers/formatting_test.exs

@@ -67,4 +67,109 @@ defmodule ElixirLS.LanguageServer.Providers.FormattingTest do
     assert {:error, :internal_error, msg} = Formatting.format(source_file, uri, project_dir)
     assert String.contains?(msg, "Unable to format")
   end
+
+  test "Proper utf-16 format: emoji 😀" do
+    uri = "file://project/file.ex"
+
+    text = """
+    IO.puts "😀"
+    """
+
+    source_file = %ElixirLS.LanguageServer.SourceFile{
+      text: text,
+      version: 1,
+      dirty?: true
+    }
+
+    project_dir = "/project"
+
+    assert {:ok, changes} = Formatting.format(source_file, uri, project_dir)
+
+    assert changes == [
+             %{
+               "newText" => ")",
+               "range" => %{
+                 "end" => %{"character" => 12, "line" => 0},
+                 "start" => %{"character" => 12, "line" => 0}
+               }
+             },
+             %{
+               "newText" => "(",
+               "range" => %{
+                 "end" => %{"character" => 8, "line" => 0},
+                 "start" => %{"character" => 7, "line" => 0}
+               }
+             }
+           ]
+  end
+
+  test "Proper utf-16 format: emoji 🏳️‍🌈" do
+    uri = "file://project/file.ex"
+
+    text = """
+    IO.puts "🏳️‍🌈"
+    """
+
+    source_file = %ElixirLS.LanguageServer.SourceFile{
+      text: text,
+      version: 1,
+      dirty?: true
+    }
+
+    project_dir = "/project"
+
+    assert {:ok, changes} = Formatting.format(source_file, uri, project_dir)
+
+    assert changes == [
+             %{
+               "newText" => ")",
+               "range" => %{
+                 "end" => %{"character" => 16, "line" => 0},
+                 "start" => %{"character" => 16, "line" => 0}
+               }
+             },
+             %{
+               "newText" => "(",
+               "range" => %{
+                 "end" => %{"character" => 8, "line" => 0},
+                 "start" => %{"character" => 7, "line" => 0}
+               }
+             }
+           ]
+  end
+
+  test "Proper utf-16 format: zalgo" do
+    uri = "file://project/file.ex"
+
+    text = """
+    IO.puts "ẕ̸͇̞̲͇͕̹̙̄͆̇͂̏̊͒̒̈́́̕͘͠͝à̵̢̛̟̞͚̟͖̻̹̮̘͚̻͍̇͂̂̅́̎̉͗́́̃̒l̴̻̳͉̖̗͖̰̠̗̃̈́̓̓̍̅͝͝͝g̷̢͚̠̜̿̊́̋͗̔ȍ̶̹̙̅̽̌̒͌͋̓̈́͑̏͑͊͛͘ ̸̨͙̦̫̪͓̠̺̫̖͙̫̏͂̒̽́̿̂̊́͂͋͜͠͝͝ṭ̴̜͎̮͉̙͍͔̜̾͋͒̓̏̉̄͘͠͝ͅę̷̡̭̹̰̺̩̠͓͌̃̕͜͝ͅͅx̵̧͍̦͈͍̝͖͙̘͎̥͕̾̾̍̀̿̔̄̑̈͝t̸̛͇̀̕"
+    """
+
+    source_file = %ElixirLS.LanguageServer.SourceFile{
+      text: text,
+      version: 1,
+      dirty?: true
+    }
+
+    project_dir = "/project"
+
+    assert {:ok, changes} = Formatting.format(source_file, uri, project_dir)
+
+    assert changes == [
+             %{
+               "newText" => ")",
+               "range" => %{
+                 "end" => %{"character" => 213, "line" => 0},
+                 "start" => %{"character" => 213, "line" => 0}
+               }
+             },
+             %{
+               "newText" => "(",
+               "range" => %{
+                 "end" => %{"character" => 8, "line" => 0},
+                 "start" => %{"character" => 7, "line" => 0}
+               }
+             }
+           ]
+  end
 end