Make tests less flaky (#792)

* monitor group leader in logger backend

* format

* warn on app unload

* raise on unload

* add some logs

* add apps to allowed list

* add one more app

* add wait

* avoid restoring wrong mix deps cache

* format

* cleanup

* disable windows tests due to erlef/setup-beam#167

Author: lukaszsamson

.github/workflows/ci.yml

@@ -49,48 +49,50 @@ jobs:
           mix deps.get --only test
       - run: mix test
 
-  mix_test_windows:
-    name: mix test windows (Elixir ${{matrix.elixir}} | Erlang/OTP ${{matrix.otp}})
-    runs-on: windows-2019
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - elixir: 1.12.x
-            otp: 22.x
-          - elixir: 1.12.x
-            otp: 23.x
-          - elixir: 1.13.x
-            otp: 22.x
-          - elixir: 1.13.x
-            otp: 23.x
-          - elixir: 1.13.x
-            otp: 24.x
-          - elixir: 1.13.x
-            otp: 25.x
-          - elixir: 1.14.x
-            otp: 23.x
-          - elixir: 1.14.x
-            otp: 24.x
-          - elixir: 1.14.x
-            otp: 25.x
-    env:
-      MIX_ENV: test
-    steps:
-      - name: Set git to use original line ending
-        run: |
-          git config --global core.autocrlf false
-      - uses: actions/checkout@v3
-      - uses: erlef/setup-beam@v1
-        with:
-          otp-version: ${{matrix.otp}}
-          elixir-version: ${{matrix.elixir}}
-      - name: Install Dependencies
-        run: |
-          mix local.hex --force
-          mix local.rebar --force
-          mix deps.get --only test
-      - run: mix test
+  # TODO reenable when https://github.com/erlef/setup-beam/issues/167 is fixed
+  # currently windows builds are failing with 403s on setup_beam action
+  # mix_test_windows:
+  #   name: mix test windows (Elixir ${{matrix.elixir}} | Erlang/OTP ${{matrix.otp}})
+  #   runs-on: windows-2019
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       include:
+  #         - elixir: 1.12.x
+  #           otp: 22.x
+  #         - elixir: 1.12.x
+  #           otp: 23.x
+  #         - elixir: 1.13.x
+  #           otp: 22.x
+  #         - elixir: 1.13.x
+  #           otp: 23.x
+  #         - elixir: 1.13.x
+  #           otp: 24.x
+  #         - elixir: 1.13.x
+  #           otp: 25.x
+  #         - elixir: 1.14.x
+  #           otp: 23.x
+  #         - elixir: 1.14.x
+  #           otp: 24.x
+  #         - elixir: 1.14.x
+  #           otp: 25.x
+  #   env:
+  #     MIX_ENV: test
+  #   steps:
+  #     - name: Set git to use original line ending
+  #       run: |
+  #         git config --global core.autocrlf false
+  #     - uses: actions/checkout@v3
+  #     - uses: erlef/setup-beam@v1
+  #       with:
+  #         otp-version: ${{matrix.otp}}
+  #         elixir-version: ${{matrix.elixir}}
+  #     - name: Install Dependencies
+  #       run: |
+  #         mix local.hex --force
+  #         mix local.rebar --force
+  #         mix deps.get --only test
+  #     - run: mix test
 
   static_analysis:
     name: static analysis (Elixir ${{matrix.elixir}} | Erlang/OTP ${{matrix.otp}})

apps/elixir_ls_utils/test/support/mix_test.case.ex

@@ -10,7 +10,24 @@ defmodule ElixirLS.Utils.MixTest.Case do
   end
 
   @apps Enum.map(Application.loaded_applications(), &elem(&1, 0))
-  @allowed_apps ~w(docsh xmerl syntax_tools edoc elixir_sense elixir_ls_debugger elixir_ls_utils language_server stream_data)a
+  @allowed_apps ~w(
+    iex
+    elixir_sense
+    elixir_ls_debugger
+    elixir_ls_utils
+    language_server
+    stream_data
+    statistex
+    patch
+    deep_merge
+    erlex
+    benchee
+    path_glob_vendored
+    dialyzer
+    dialyxir_vendored
+    erl2ex
+    jason_vendored
+    )a
 
   setup do
     on_exit(fn ->

apps/language_server/lib/language_server/build.ex

@@ -255,6 +255,21 @@ defmodule ElixirLS.LanguageServer.Build do
   end
 
   defp purge_dep(%Mix.Dep{app: app} = dep) do
+    if app in [
+         :language_server,
+         :elixir_ls_utils,
+         :elixir_sense,
+         :stream_data,
+         :jason_vendored,
+         :path_glob_vendored,
+         :dialyxir_vendored,
+         :erl2ex,
+         :patch,
+         :benchee
+       ] do
+      raise "Unloading #{app}"
+    end
+
     for path <- Mix.Dep.load_paths(dep) do
       Code.delete_path(path)
     end
@@ -338,7 +353,9 @@ defmodule ElixirLS.LanguageServer.Build do
   defp read_cached_deps() do
     # FIXME: Private api
     # we cannot use Mix.Dep.cached() here as it tries to load deps
-    if project = Mix.Project.get() do
+    project = Mix.Project.get()
+    # in test do not try to load cache from elixir_ls
+    if project != nil and project != ElixirLS.LanguageServer.Mixfile do
       env_target = {Mix.env(), Mix.target()}
 
       case Mix.State.read_cache({:cached_deps, project}) do

apps/language_server/lib/language_server/json_rpc_logger_backend.ex

@@ -24,7 +24,9 @@ defmodule Logger.Backends.JsonRpc do
 
   @behaviour :gen_event
 
-  defstruct format: nil,
+  defstruct group_leader: nil,
+            default_group_leader: nil,
+            format: nil,
             level: nil,
             metadata: nil
 
@@ -46,8 +48,10 @@ defmodule Logger.Backends.JsonRpc do
   end
 
   def handle_call({:set_group_leader, pid}, state) do
+    Process.monitor(pid)
+    default_group_leader = Process.group_leader()
     Process.group_leader(self(), pid)
-    {:ok, :ok, state}
+    {:ok, :ok, %{state | group_leader: pid, default_group_leader: default_group_leader}}
   end
 
   @impl true
@@ -74,6 +78,14 @@ defmodule Logger.Backends.JsonRpc do
   end
 
   @impl true
+  def handle_info(
+        {:DOWN, _, :process, pid, _},
+        state = %{group_leader: pid, default_group_leader: default_group_leader}
+      ) do
+    Process.group_leader(self(), default_group_leader)
+    {:ok, %{state | group_leader: nil}}
+  end
+
   def handle_info(_, state) do
     {:ok, state}
   end
@@ -125,10 +137,8 @@ defmodule Logger.Backends.JsonRpc do
   end
 
   defp log_event(level, msg, ts, md, state) do
-    if function_exported?(ElixirLS.Utils.WireProtocol, :send, 1) do
-      output = format_event(level, msg, ts, md, state) |> IO.chardata_to_string()
-      ElixirLS.LanguageServer.JsonRpc.log_message(elixir_log_level_to_lsp(level), output)
-    end
+    output = format_event(level, msg, ts, md, state) |> IO.chardata_to_string()
+    ElixirLS.LanguageServer.JsonRpc.log_message(elixir_log_level_to_lsp(level), output)
 
     state
   end

apps/language_server/test/server_test.exs

@@ -1533,6 +1533,7 @@ defmodule ElixirLS.LanguageServer.ServerTest do
     try do
       func.(server)
     after
+      wait_until_compiled(server)
       stop_supervised(Server)
       stop_supervised(PacketCapture)
       flush_mailbox()