make more regexes unicode aware

attempt to fix a rare crash in complete when get_prefix would return invalid <<186>>
which would later crash

an exception was raised:
    ** (UnicodeConversionError) invalid encoding starting at <<186>>
        (elixir 1.14.3) <REDACTED: user-file-path>:2536: String.to_charlist_1
        (elixir 1.14.3) <REDACTED: user-file-path>:795: Code.Fragment.last_line_1
        (elixir 1.14.3) <REDACTED: user-file-path>:159: Code.Fragment.cursor_context_2

Author: lukaszsamson

apps/language_server/lib/language_server/diagnostics.ex

@@ -92,7 +92,7 @@ defmodule ElixirLS.LanguageServer.Diagnostics do
   end
 
   defp split_type_and_message(message) do
-    case Regex.run(~r/^\*\* \(([\w\.]+?)?\) (.*)/s, message) do
+    case Regex.run(~r/^\*\* \(([\w\.]+?)?\) (.*)/su, message) do
       [_, type, rest] ->
         {type, rest}
 
@@ -121,7 +121,7 @@ defmodule ElixirLS.LanguageServer.Diagnostics do
   end
 
   defp get_message_parts(message) do
-    case Regex.run(~r/^(.*?):(\d+)(:(\d+))?: (.*)/s, message) do
+    case Regex.run(~r/^(.*?):(\d+)(:(\d+))?: (.*)/su, message) do
       [_, file, line, _, column, description] -> {file, line, column, description}
       _ -> nil
     end
@@ -151,8 +151,8 @@ defmodule ElixirLS.LanguageServer.Diagnostics do
   end
 
   defp is_stack?("    " <> str) do
-    Regex.match?(~r/.*\.(ex|erl):\d+: /, str) ||
-      Regex.match?(~r/.*expanding macro: /, str)
+    Regex.match?(~r/.*\.(ex|erl):\d+: /u, str) ||
+      Regex.match?(~r/.*expanding macro: /u, str)
   end
 
   defp is_stack?(_) do
@@ -161,7 +161,7 @@ defmodule ElixirLS.LanguageServer.Diagnostics do
 
   defp extract_line_from_missing_hint(message) do
     case Regex.run(
-           ~r/HINT: it looks like the .+ on line (\d+) does not have a matching /,
+           ~r/HINT: it looks like the .+ on line (\d+) does not have a matching /u,
            message
          ) do
       [_, line] -> String.to_integer(line)
@@ -172,7 +172,7 @@ defmodule ElixirLS.LanguageServer.Diagnostics do
   defp extract_line_from_stacktrace(file, stacktrace) do
     Enum.find_value(stacktrace, fn stack_item ->
       with [_, _, file_relative, line] <-
-             Regex.run(~r/(\(.+?\)\s+)?(.*\.ex):(\d+): /, stack_item),
+             Regex.run(~r/(\(.+?\)\s+)?(.*\.ex):(\d+): /u, stack_item),
            true <- String.ends_with?(file, file_relative) do
         String.to_integer(line)
       else

apps/language_server/lib/language_server/dialyzer.ex

@@ -585,7 +585,7 @@ defmodule ElixirLS.LanguageServer.Dialyzer do
 
   defp dialyzer_raw_warning_message(raw_warning) do
     message = String.trim(to_string(:dialyzer.format_warning(raw_warning)))
-    Regex.replace(~r/^.*:\d+: /, message, "")
+    Regex.replace(~r/^.*:\d+: /u, message, "")
   end
 
   # Because mtime-based stale-checking has 1-second granularity, we err on the side of

apps/language_server/lib/language_server/erlang_source_file.ex

@@ -20,7 +20,7 @@ defmodule ElixirLS.LanguageServer.ErlangSourceFile do
   end
 
   def module_line(file) do
-    find_line_by_regex(file, ~r/^-module/)
+    find_line_by_regex(file, ~r/^-module/u)
   end
 
   def function_line(file, function) do
@@ -30,7 +30,7 @@ defmodule ElixirLS.LanguageServer.ErlangSourceFile do
       |> Atom.to_string()
       |> Regex.escape()
 
-    find_line_by_regex(file, ~r/^#{escaped}\b/)
+    find_line_by_regex(file, ~r/^#{escaped}\b/u)
   end
 
   defp find_line_by_regex(file, regex) do

apps/language_server/lib/language_server/markdown_utils.ex

@@ -1,7 +1,7 @@
 defmodule ElixirLS.LanguageServer.MarkdownUtils do
   # Find the lowest heading level in the fragment
   defp lowest_heading_level(fragment) do
-    case Regex.scan(~r/(#+)/, fragment) do
+    case Regex.scan(~r/(#+)/u, fragment) do
       [] ->
         nil
 
@@ -19,7 +19,7 @@ defmodule ElixirLS.LanguageServer.MarkdownUtils do
     if min_level do
       level_difference = base_level + 1 - min_level
 
-      Regex.replace(~r/(#+)/, fragment, fn _, capture ->
+      Regex.replace(~r/(#+)/u, fragment, fn _, capture ->
         adjusted_level = String.length(capture) + level_difference
         String.duplicate("#", adjusted_level)
       end)

apps/language_server/lib/language_server/mix_tasks/format.ex

@@ -920,7 +920,7 @@ defmodule Mix.Tasks.ElixirLSFormat do
 
       if space? do
         str
-        |> String.split(~r/[\t\s]+/, include_captures: true)
+        |> String.split(~r/[\t\s]+/u, include_captures: true)
         |> Enum.map(fn
           <<start::binary-size(1), _::binary>> = str when start in ["\t", "\s"] ->
             IO.ANSI.format([color[:space], str])

apps/language_server/lib/language_server/providers/code_lens/test/describe_block.ex

@@ -27,7 +27,7 @@ defmodule ElixirLS.LanguageServer.Providers.CodeLens.Test.DescribeBlock do
 
   defp get_name(source_lines, declaration_line) do
     %{"name" => name} =
-      ~r/^\s*describe "(?<name>.*)" do/
+      ~r/^\s*describe "(?<name>.*)" do/u
       |> Regex.named_captures(Enum.at(source_lines, declaration_line - 1))
 
     name

apps/language_server/lib/language_server/providers/completion.ex

@@ -120,11 +120,11 @@ defmodule ElixirLS.LanguageServer.Providers.Completion do
 
     def_before =
       cond do
-        Regex.match?(~r/(defdelegate|defp?)\s*#{prefix}$/, text_before_cursor) ->
+        Regex.match?(~r/(defdelegate|defp?)\s*#{prefix}$/u, text_before_cursor) ->
           :def
 
         Regex.match?(
-          ~r/(defguardp?|defmacrop?)\s*#{prefix}$/,
+          ~r/(defguardp?|defmacrop?)\s*#{prefix}$/u,
           text_before_cursor
         ) ->
           :defmacro
@@ -138,7 +138,7 @@ defmodule ElixirLS.LanguageServer.Providers.Completion do
       |> Enum.slice(0..(line - 1))
       |> Enum.reverse()
       |> Enum.find_value(0, fn line_text ->
-        if Regex.match?(~r/(?<=\s|^)do\s*(#.*)?$/, line_text) do
+        if Regex.match?(~r/(?<=\s|^)do\s*(#.*)?$/u, line_text) do
           String.length(line_text) - String.length(String.trim_leading(line_text))
         end
       end)
@@ -151,8 +151,8 @@ defmodule ElixirLS.LanguageServer.Providers.Completion do
       prefix: prefix,
       remote_calls?: match?({:dot, _, _}, Code.Fragment.cursor_context(prefix)),
       def_before: def_before,
-      pipe_before?: Regex.match?(~r/\|>\s*#{prefix}$/, text_before_cursor),
-      capture_before?: Regex.match?(~r/&#{prefix}$/, text_before_cursor),
+      pipe_before?: Regex.match?(~r/\|>\s*#{prefix}$/u, text_before_cursor),
+      capture_before?: Regex.match?(~r/&#{prefix}$/u, text_before_cursor),
       scope: scope,
       module: env.module,
       line: line,
@@ -221,7 +221,7 @@ defmodule ElixirLS.LanguageServer.Providers.Completion do
 
   defp maybe_add_do(completion_items, context) do
     hint =
-      case Regex.scan(~r/(?<=\s|^)[a-z]+$/, context.text_before_cursor) do
+      case Regex.scan(~r/(?<=\s|^)[a-z]+$/u, context.text_before_cursor) do
         [] -> ""
         [[match]] -> match
       end
@@ -247,7 +247,7 @@ defmodule ElixirLS.LanguageServer.Providers.Completion do
 
   defp maybe_add_keywords(completion_items, context) do
     hint =
-      case Regex.scan(~r/(?<=\s|^)[a-z]+$/, context.text_before_cursor) do
+      case Regex.scan(~r/(?<=\s|^)[a-z]+$/u, context.text_before_cursor) do
         [] -> ""
         [[match]] -> match
       end
@@ -1113,7 +1113,7 @@ defmodule ElixirLS.LanguageServer.Providers.Completion do
   end
 
   defp get_prefix(text_before_cursor) do
-    regex = ~r/[\w0-9\._!\?\:@\->]+$/
+    regex = ~r/[\w0-9\._!\?\:@\->]+$/u
 
     case Regex.run(regex, text_before_cursor) do
       [prefix] -> prefix
@@ -1343,7 +1343,7 @@ defmodule ElixirLS.LanguageServer.Providers.Completion do
   end
 
   defp snippet?(item) do
-    item.kind == :snippet || String.match?(item.insert_text, ~r/\${?\d/)
+    item.kind == :snippet || String.match?(item.insert_text, ~r/\${?\d/u)
   end
 
   # As defined by CompletionItemTag in https://microsoft.github.io/language-server-protocol/specifications/specification-current/

apps/language_server/lib/language_server/providers/execute_command/apply_spec.ex

@@ -49,7 +49,7 @@ defmodule ElixirLS.LanguageServer.Providers.ExecuteCommand.ApplySpec do
       end
 
     cur_line = Enum.at(SourceFile.lines(cur_text), line - 1)
-    [indentation] = Regex.run(~r/^\s*/, cur_line)
+    [indentation] = Regex.run(~r/^\s*/u, cur_line)
 
     # Attempt to format to fit within the preferred line length, fallback to having it all on one
     # line if anything fails

apps/language_server/lib/language_server/providers/execute_command/manipulate_pipes.ex

@@ -337,7 +337,7 @@ defmodule ElixirLS.LanguageServer.Providers.ExecuteCommand.ManipulatePipes do
     |> String.reverse()
     |> String.graphemes()
     |> Enum.reduce_while([], fn c, acc ->
-      if String.match?(c, ~r/[\s\(\[\{]/) do
+      if String.match?(c, ~r/[\s\(\[\{]/u) do
         {:halt, acc}
       else
         {:cont, [c | acc]}

apps/language_server/lib/language_server/providers/on_type_formatting.ex

@@ -57,11 +57,11 @@ defmodule ElixirLS.LanguageServer.Providers.OnTypeFormatting do
   end
 
   defp tokens(line) do
-    Regex.scan(~r/(?:->)|(?:[\w\:]+)/, line) |> List.flatten()
+    Regex.scan(~r/(?:->)|(?:[\w\:]+)/u, line) |> List.flatten()
   end
 
   defp indentation(line) do
-    [indentation] = Regex.run(~r/^\s*/, line)
+    [indentation] = Regex.run(~r/^\s*/u, line)
     indentation
   end