Fix tokenization of emitting comments followed by \r\n (#455)

Before this patch the `\r` character would be part of the comment and
not the following NewlineWs (which would simply be a `\n`-style
NewlineWs instead of a `\r\n` as one would expect).

Author: fredrikekre

src/tokenize.jl

@@ -747,8 +747,8 @@ function lex_comment(l::Lexer)
     if peekchar(l) != '='
         valid = true
         while true
-            pc = peekchar(l)
-            if pc == '\n' || pc == EOF_CHAR
+            pc, ppc = dpeekchar(l)
+            if pc == '\n' || (pc == '\r' && ppc == '\n') || pc == EOF_CHAR
                 return emit(l, valid ? K"Comment" : K"ErrorInvalidUTF8")
             end
             valid &= isvalid(pc)

test/tokenize.jl

@@ -221,6 +221,8 @@ end
     @test toks("#=   #=   =#") == ["#=   #=   =#"=>K"ErrorEofMultiComment"]
     @test toks("#=#==#=#") == ["#=#==#=#"=>K"Comment"]
     @test toks("#=#==#=")  == ["#=#==#="=>K"ErrorEofMultiComment"]
+    # comment terminated by \r\n
+    @test toks("#\r\n") == ["#" => K"Comment", "\r\n" => K"NewlineWs"]
 end