fix: skip zero-width error tokens in last_non_whitespace_byte (#566)

to ensure that `let x = 1 # comment` is treated as an incomplete expression

Author: pfitzseb

src/core/parse_stream.jl

@@ -970,10 +970,11 @@ function last_non_whitespace_byte(stream::ParseStream)
     for i = length(stream.output):-1:1
         node = stream.output[i]
         if is_terminal(node)
-            if !(kind(node) in KSet"Comment Whitespace NewlineWs ErrorEofMultiComment")
+            if kind(node) in KSet"Comment Whitespace NewlineWs ErrorEofMultiComment" || kind(node) == K"error" && node.byte_span == 0
+                byte_pos -= node.byte_span
+            else
                 return byte_pos - 1
             end
-            byte_pos -= node.byte_span
         end
     end
     return first_byte(stream) - 1

test/hooks.jl

@@ -497,6 +497,7 @@ end
                 "Issue53126()." => :other
                 "using " => :other
                 "global xxx::Number = Base." => :other
+                "let x = 1 # comment" => :other
             ]
             @testset "$(repr(str))" begin
                 # Test :statement parsing