Skip to content

Commit 32fc59f

Browse files
committed
fault tolerant parsing
1 parent 341a815 commit 32fc59f

File tree

9 files changed

+191
-63
lines changed

9 files changed

+191
-63
lines changed

apps/language_server/lib/language_server/parser.ex

Lines changed: 164 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@ defmodule ElixirLS.LanguageServer.Parser do
2020
:ast,
2121
:diagnostics,
2222
:metadata,
23-
:parsed_version
23+
:parsed_version,
24+
:flag
2425
]
2526
end
2627

@@ -32,14 +33,18 @@ defmodule ElixirLS.LanguageServer.Parser do
3233
GenServer.cast(__MODULE__, {:closed, uri})
3334
end
3435

35-
def parse_with_debounce(uri, source_file) do
36+
def parse_with_debounce(uri, source_file = %SourceFile{}) do
3637
GenServer.cast(__MODULE__, {:parse_with_debounce, uri, source_file})
3738
end
3839

39-
def parse_immediate(uri, source_file) do
40+
def parse_immediate(uri, source_file = %SourceFile{}) do
4041
GenServer.call(__MODULE__, {:parse_immediate, uri, source_file})
4142
end
4243

44+
def parse_immediate(uri, source_file = %SourceFile{}, position) do
45+
GenServer.call(__MODULE__, {:parse_immediate, uri, source_file, position})
46+
end
47+
4348
@impl true
4449
def init(_args) do
4550
# TODO get source files on start?
@@ -59,7 +64,7 @@ defmodule ElixirLS.LanguageServer.Parser do
5964
{:noreply, %{state | files: updated_files, debounce_refs: updated_debounce_refs}}
6065
end
6166

62-
def handle_cast({:parse_with_debounce, uri, source_file}, state) do
67+
def handle_cast({:parse_with_debounce, uri, source_file = %SourceFile{}}, state) do
6368
state = if should_parse?(uri, source_file) do
6469
state = update_in(state.debounce_refs[uri], fn old_ref ->
6570
if old_ref do
@@ -89,7 +94,7 @@ defmodule ElixirLS.LanguageServer.Parser do
8994
end
9095

9196
@impl true
92-
def handle_call({:parse_immediate, uri, source_file}, _from, %{files: files, debounce_refs: debounce_refs} = state) do
97+
def handle_call({:parse_immediate, uri, source_file = %SourceFile{}}, _from, %{files: files, debounce_refs: debounce_refs} = state) do
9398
{reply, state} = if should_parse?(uri, source_file) do
9499
{maybe_ref, updated_debounce_refs} = Map.pop(debounce_refs, uri)
95100
if maybe_ref do
@@ -132,6 +137,54 @@ defmodule ElixirLS.LanguageServer.Parser do
132137
{:reply, reply, state}
133138
end
134139

140+
def handle_call({:parse_immediate, uri, source_file = %SourceFile{}, position}, _from, %{files: files, debounce_refs: debounce_refs} = state) do
141+
{reply, state} = if should_parse?(uri, source_file) do
142+
{maybe_ref, updated_debounce_refs} = Map.pop(debounce_refs, uri)
143+
if maybe_ref do
144+
Process.cancel_timer(maybe_ref, info: false)
145+
end
146+
147+
current_version = source_file.version
148+
149+
case files[uri] do
150+
%Context{parsed_version: ^current_version} = file ->
151+
Logger.debug("#{uri} already parsed for cursor position #{inspect(position)}")
152+
file = maybe_fix_missing_env(file, position)
153+
154+
updated_files = Map.put(files, uri, file)
155+
# no change to diagnostics, only update stored metadata
156+
state = %{state | files: updated_files, debounce_refs: updated_debounce_refs}
157+
{file, state}
158+
159+
_other ->
160+
Logger.debug("Parsing #{uri} immediately: languageId #{source_file.language_id}")
161+
# overwrite everything
162+
file = %Context{
163+
source_file: source_file,
164+
path: get_path(uri)
165+
}
166+
|> do_parse(position)
167+
168+
updated_files = Map.put(files, uri, file)
169+
170+
notify_diagnostics_updated(updated_files)
171+
172+
state = %{state | files: updated_files, debounce_refs: updated_debounce_refs}
173+
{file, state}
174+
end
175+
else
176+
Logger.debug("Not parsing #{uri} immediately: languageId #{source_file.language_id}")
177+
# not parsing - respond with empty struct
178+
reply = %Context{
179+
source_file: source_file,
180+
path: get_path(uri)
181+
}
182+
{reply, state}
183+
end
184+
185+
{:reply, reply, state}
186+
end
187+
135188
@impl GenServer
136189
def handle_info(
137190
{:parse_file, uri},
@@ -156,22 +209,112 @@ defmodule ElixirLS.LanguageServer.Parser do
156209
String.ends_with?(uri, [".ex", ".exs", ".eex"]) or source_file.language_id in ["elixir", "eex", "html-eex"]
157210
end
158211

159-
defp do_parse(%Context{source_file: source_file, path: path} = file) do
212+
@dummy_source ""
213+
@dummy_ast Code.string_to_quoted!(@dummy_source)
214+
@dummy_metadata ElixirSense.Core.Metadata.fill(@dummy_source, MetadataBuilder.build(@dummy_ast))
215+
216+
defp maybe_fix_missing_env(%Context{metadata: metadata, flag: flag, source_file: source_file = %SourceFile{}} = file, {line, _character} = cursor_position) do
217+
if Map.has_key?(metadata.lines_to_env, line) do
218+
file
219+
else
220+
case flag do
221+
{_, ^cursor_position} ->
222+
# give up - we already tried
223+
file
224+
{:exact, _} ->
225+
# file does not have parse errors, try to parse again with marker
226+
metadata = case ElixirSense.Core.Parser.try_fix_line_not_found_by_inserting_marker(source_file.text, cursor_position) do
227+
{:ok, acc} ->
228+
Logger.debug("Fixed missing env")
229+
ElixirSense.Core.Metadata.fill(source_file.text, acc)
230+
_ ->
231+
Logger.debug("Not able to fix missing env")
232+
metadata
233+
end
234+
235+
%Context{file |
236+
metadata: metadata,
237+
flag: {:exact, cursor_position}
238+
}
239+
240+
:not_parsable ->
241+
# give up - no support in fault tolerant parser
242+
file
243+
{f, _cursor_position} when f in [:not_parsable, :fixed] ->
244+
# reparse with cursor position
245+
{flag, ast, metadata} = fault_tolerant_parse(source_file, cursor_position)
246+
%Context{file |
247+
ast: ast,
248+
metadata: metadata,
249+
flag: flag
250+
}
251+
end
252+
end
253+
end
254+
255+
defp do_parse(%Context{source_file: source_file = %SourceFile{}, path: path} = file, cursor_position \\ nil) do
160256
{ast, diagnostics} = parse_file(source_file.text, path, source_file.language_id)
161257

162-
metadata = if ast do
163-
acc = MetadataBuilder.build(ast)
164-
ElixirSense.Core.Metadata.fill(source_file.text, acc)
258+
{flag, ast, metadata} = if ast do
259+
# no syntax errors
260+
metadata = MetadataBuilder.build(ast)
261+
|> fix_missing_env(source_file.text, cursor_position)
262+
{{:exact, cursor_position}, ast, metadata}
263+
else
264+
if elixir?(path, source_file.language_id) do
265+
fault_tolerant_parse(source_file, cursor_position)
266+
else
267+
# no support for eex in ElixirSense.Core.Parser
268+
{:not_parsable, @dummy_ast, @dummy_metadata}
269+
end
165270
end
166271

167272
%Context{file |
168273
ast: ast,
169274
diagnostics: diagnostics,
170275
metadata: metadata,
171-
parsed_version: source_file.version
276+
parsed_version: source_file.version,
277+
flag: flag
172278
}
173279
end
174280

281+
defp fault_tolerant_parse(source_file = %SourceFile{}, cursor_position) do
282+
# attempt to parse with fixing syntax errors
283+
options = [
284+
errors_threshold: 3,
285+
cursor_position: cursor_position,
286+
fallback_to_container_cursor_to_quoted: true
287+
]
288+
case ElixirSense.Core.Parser.string_to_ast(source_file.text, options) do
289+
{:ok, ast, modified_source, _error} ->
290+
Logger.debug("Syntax error fixed")
291+
metadata = MetadataBuilder.build(ast)
292+
|> fix_missing_env(modified_source, cursor_position)
293+
{{:fixed, cursor_position}, ast, metadata}
294+
_ ->
295+
Logger.debug("Not able to fix syntax error")
296+
# we can't fix it
297+
{{:not_parsable, cursor_position}, @dummy_ast, @dummy_metadata}
298+
end
299+
end
300+
301+
defp fix_missing_env(acc, source, nil), do: ElixirSense.Core.Metadata.fill(source, acc)
302+
defp fix_missing_env(acc, source, {line, _} = cursor_position) do
303+
acc = if Map.has_key?(acc.lines_to_env, line) do
304+
acc
305+
else
306+
case ElixirSense.Core.Parser.try_fix_line_not_found_by_inserting_marker(source, cursor_position) do
307+
{:ok, acc} ->
308+
Logger.debug("Fixed missing env")
309+
acc
310+
_ ->
311+
Logger.debug("Not able to fix missing env")
312+
acc
313+
end
314+
end
315+
ElixirSense.Core.Metadata.fill(source, acc)
316+
end
317+
175318
defp get_path(uri) do
176319
case uri do
177320
"file:" <> _ -> SourceFile.Path.from_uri(uri)
@@ -186,16 +329,25 @@ defmodule ElixirLS.LanguageServer.Parser do
186329
|> Server.parser_finished()
187330
end
188331

332+
defp elixir?(file, language_id) do
333+
is_binary(file) and (String.ends_with?(file, ".ex") or String.ends_with?(file, ".exs")) or language_id in ["elixir"]
334+
end
335+
336+
defp eex?(file, language_id) do
337+
is_binary(file) and String.ends_with?(file, ".eex") or language_id in ["eex", "html-eex"]
338+
end
339+
189340
defp parse_file(text, file, language_id) do
190341
{result, raw_diagnostics} =
191342
Build.with_diagnostics([log: false], fn ->
192343
try do
193344
parser_options = [
194345
file: file,
195-
columns: true
346+
columns: true,
347+
token_metadata: true
196348
]
197349

198-
ast = if is_binary(file) and String.ends_with?(file, ".eex") or language_id in ["eex", "html-eex"] do
350+
ast = if eex?(file, language_id) do
199351
EEx.compile_string(text,
200352
file: file,
201353
parser_options: parser_options

apps/language_server/lib/language_server/providers/completion.ex

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -95,19 +95,13 @@ defmodule ElixirLS.LanguageServer.Providers.Completion do
9595

9696
def completion(%Parser.Context{source_file: %SourceFile{text: text}, metadata: metadata}, line, character, options) do
9797
lines = SourceFile.lines(text)
98-
line_text = Enum.at(lines, line)
99-
100-
# convert to 1 based utf8 position
101-
line = line + 1
102-
character = SourceFile.lsp_character_to_elixir(line_text, character)
98+
line_text = Enum.at(lines, line - 1)
10399

104100
text_before_cursor = String.slice(line_text, 0, character - 1)
105101
text_after_cursor = String.slice(line_text, (character - 1)..-1//1)
106102

107103
prefix = get_prefix(text_before_cursor)
108104

109-
metadata = metadata || ElixirSense.Core.Parser.parse_string(text, true, true, {line, character})
110-
111105
env = ElixirSense.Core.Metadata.get_env(metadata, {line, character})
112106

113107
scope =

apps/language_server/lib/language_server/providers/definition.ex

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,11 @@ defmodule ElixirLS.LanguageServer.Providers.Definition do
33
textDocument/definition provider utilizing Elixir Sense
44
"""
55

6-
alias ElixirLS.LanguageServer.{Protocol, SourceFile, Parser}
6+
alias ElixirLS.LanguageServer.{Protocol, Parser}
77

88
def definition(uri, %Parser.Context{source_file: source_file, metadata: metadata}, line, character, project_dir) do
9-
{line, character} = SourceFile.lsp_position_to_elixir(source_file.text, {line, character})
10-
119
result =
12-
case ElixirSense.definition(source_file.text, line, character, if(metadata, do: [metadata: metadata], else: [])) do
10+
case ElixirSense.definition(source_file.text, line, character, [metadata: metadata]) do
1311
nil ->
1412
nil
1513

apps/language_server/lib/language_server/providers/document_symbols.ex

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ defmodule ElixirLS.LanguageServer.Providers.DocumentSymbols do
66
"""
77

88
alias ElixirLS.LanguageServer.Providers.SymbolUtils
9-
alias ElixirLS.LanguageServer.SourceFile
9+
alias ElixirLS.LanguageServer.{SourceFile, Parser}
1010
require ElixirLS.LanguageServer.Protocol, as: Protocol
1111

1212
defmodule Info do
@@ -24,16 +24,10 @@ defmodule ElixirLS.LanguageServer.Providers.DocumentSymbols do
2424
:deprecated
2525
]
2626

27-
@max_parser_errors 6
27+
def symbols(uri, %Parser.Context{ast: ast, source_file: source_file}, hierarchical) do
28+
symbols = extract_modules(ast) |> Enum.reject(&is_nil/1)
2829

29-
def symbols(uri, text, hierarchical) do
30-
case list_symbols(text) do
31-
{:ok, symbols} ->
32-
{:ok, build_symbols(symbols, uri, text, hierarchical)}
33-
34-
{:error, :compilation_error} ->
35-
{:error, :server_error, "Cannot parse source file", false}
36-
end
30+
{:ok, build_symbols(symbols, uri, source_file.text, hierarchical)}
3731
end
3832

3933
defp build_symbols(symbols, uri, text, hierarchical)
@@ -48,16 +42,6 @@ defmodule ElixirLS.LanguageServer.Providers.DocumentSymbols do
4842
|> List.flatten()
4943
end
5044

51-
defp list_symbols(src) do
52-
case ElixirSense.string_to_quoted(src, {1, 1}, @max_parser_errors,
53-
line: 1,
54-
token_metadata: true
55-
) do
56-
{:ok, quoted_form} -> {:ok, extract_modules(quoted_form) |> Enum.reject(&is_nil/1)}
57-
{:error, _error} -> {:error, :compilation_error}
58-
end
59-
end
60-
6145
# Identify and extract the module symbol, and the symbols contained within the module
6246
defp extract_modules({:__block__, [], ast}) do
6347
ast |> Enum.map(&extract_modules(&1)) |> List.flatten()

apps/language_server/lib/language_server/providers/hover.ex

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,8 @@ defmodule ElixirLS.LanguageServer.Providers.Hover do
88
"""
99

1010
def hover(%Parser.Context{source_file: source_file, metadata: metadata}, line, character) do
11-
{line, character} = SourceFile.lsp_position_to_elixir(source_file.text, {line, character})
12-
1311
response =
14-
case ElixirSense.docs(source_file.text, line, character, if(metadata, do: [metadata: metadata], else: [])) do
12+
case ElixirSense.docs(source_file.text, line, character, [metadata: metadata]) do
1513
nil ->
1614
nil
1715

apps/language_server/lib/language_server/providers/implementation.ex

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,10 @@ defmodule ElixirLS.LanguageServer.Providers.Implementation do
33
textDocument/implementation provider utilizing Elixir Sense
44
"""
55

6-
alias ElixirLS.LanguageServer.{Protocol, SourceFile, Parser}
6+
alias ElixirLS.LanguageServer.{Protocol, Parser}
77

88
def implementation(uri, %Parser.Context{source_file: source_file, metadata: metadata}, line, character, project_dir) do
9-
{line, character} = SourceFile.lsp_position_to_elixir(source_file.text, {line, character})
10-
locations = ElixirSense.implementations(source_file.text, line, character, if(metadata, do: [metadata: metadata], else: []))
9+
locations = ElixirSense.implementations(source_file.text, line, character, [metadata: metadata])
1110

1211
results =
1312
for location <- locations, do: Protocol.Location.new(location, uri, source_file.text, project_dir)

apps/language_server/lib/language_server/providers/references.ex

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,10 @@ defmodule ElixirLS.LanguageServer.Providers.References do
1515
require Logger
1616

1717
def references(%Parser.Context{source_file: source_file, metadata: metadata}, uri, line, character, _include_declaration, project_dir) do
18-
{line, character} = SourceFile.lsp_position_to_elixir(source_file.text, {line, character})
19-
2018
Build.with_build_lock(fn ->
2119
trace = ElixirLS.LanguageServer.Tracer.get_trace()
2220

23-
ElixirSense.references(source_file.text, line, character, trace, if(metadata, do: [metadata: metadata], else: []))
21+
ElixirSense.references(source_file.text, line, character, trace, [metadata: metadata])
2422
|> Enum.map(fn elixir_sense_reference ->
2523
elixir_sense_reference
2624
|> build_reference(uri, source_file.text, project_dir)

apps/language_server/lib/language_server/providers/signature_help.ex

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,8 @@ defmodule ElixirLS.LanguageServer.Providers.SignatureHelp do
77
def trigger_characters(), do: ["(", ","]
88

99
def signature(%Parser.Context{source_file: %SourceFile{} = source_file, metadata: metadata}, line, character) do
10-
{line, character} = SourceFile.lsp_position_to_elixir(source_file.text, {line, character})
11-
1210
response =
13-
case ElixirSense.signature(source_file.text, line, character, if(metadata, do: [metadata: metadata], else: [])) do
11+
case ElixirSense.signature(source_file.text, line, character, [metadata: metadata]) do
1412
%{active_param: active_param, signatures: signatures} ->
1513
%{
1614
"activeSignature" => 0,

0 commit comments

Comments
 (0)