Remove DOTOP_FLAG from tokenizer and parser (#568)

This implements the first of a series of AST format changes described
in #567. In particular, this removes the DOTOP_FLAG. Currently, we already
do not emit the DOTOP_FLAG on terminals, they always get split into one
dot token and one identifier token (although we did set it on the intermediate
tokens that came out of the lexer). The only four kinds for which DOTOP_FLAG
was ever set in the final AST were `=`, `op=`, `&&` and `||`. This introduces
separate head kinds for each of these (similar to how there are already
separate head calls for `dotcall` and `dot). Otherwise the AST structure
should be unchanged.

Author: Keno

LICENSE.md

@@ -1,17 +1,17 @@
 The JuliaSyntax.jl package is licensed under the MIT "Expat" License:
 
 > Copyright (c) 2021 Julia Computing and contributors
-> 
+>
 > Permission is hereby granted, free of charge, to any person obtaining a copy
 > of this software and associated documentation files (the "Software"), to deal
 > in the Software without restriction, including without limitation the rights
 > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 > copies of the Software, and to permit persons to whom the Software is
 > furnished to do so, subject to the following conditions:
-> 
+>
 > The above copyright notice and this permission notice shall be included in all
 > copies or substantial portions of the Software.
-> 
+>
 > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

src/core/parse_stream.jl

@@ -376,7 +376,6 @@ function _buffer_lookahead_tokens(lexer, lookahead)
         was_whitespace = is_whitespace(k)
         had_whitespace |= was_whitespace
         f = EMPTY_FLAGS
-        raw.dotop      && (f |= DOTOP_FLAG)
         raw.suffix     && (f |= SUFFIXED_FLAG)
         push!(lookahead, SyntaxToken(SyntaxHead(k, f), k,
                                      had_whitespace, raw.endbyte + 2))

src/integration/expr.jl

@@ -313,9 +313,13 @@ end
         op = args[2]
         rhs = args[3]
         headstr = string(args[2], '=')
-        if is_dotted(nodehead)
-            headstr = '.'*headstr
-        end
+        retexpr.head = Symbol(headstr)
+        retexpr.args = Any[lhs, rhs]
+    elseif k == K".op=" && length(args) == 3
+        lhs = args[1]
+        op = args[2]
+        rhs = args[3]
+        headstr = '.' * string(args[2], '=')
         retexpr.head = Symbol(headstr)
         retexpr.args = Any[lhs, rhs]
     elseif k == K"macrocall"

src/julia/julia_parse_stream.jl

@@ -1,6 +1,4 @@
 # Token flags - may be set for operator kinded tokens
-# Operator is dotted
-const DOTOP_FLAG = RawFlags(1<<1)
 # Operator has a suffix
 const SUFFIXED_FLAG = RawFlags(1<<2)
 
@@ -112,12 +110,6 @@ Return true for postfix operator calls such as the `'ᵀ` call node parsed from
 """
 is_postfix_op_call(x) = call_type_flags(x) == POSTFIX_OP_FLAG
 
-"""
-    is_dotted(x)
-
-Return true for dotted syntax tokens
-"""
-is_dotted(x) = has_flags(x, DOTOP_FLAG)
 
 """
     is_suffixed(x)
@@ -126,12 +118,6 @@ Return true for operators which have suffixes, such as `+₁`
 """
 is_suffixed(x) = has_flags(x, SUFFIXED_FLAG)
 
-"""
-    is_decorated(x)
-
-Return true for operators which are decorated with a dot or suffix.
-"""
-is_decorated(x) = is_dotted(x) || is_suffixed(x)
 
 """
     numeric_flags(x)
@@ -144,11 +130,7 @@ numeric_flags(x) = numeric_flags(flags(x))
 function untokenize(head::SyntaxHead; unique=true, include_flag_suff=true)
     str = (is_error(kind(head)) ? untokenize(kind(head); unique=false) :
            untokenize(kind(head); unique=unique))::String
-    if is_dotted(head)
-        str = "."*str
-    end
     if include_flag_suff
-        # Ignore DOTOP_FLAG - it's represented above with . prefix
         is_trivia(head)  && (str = str*"-t")
         is_infix_op_call(head)   && (str = str*"-i")
         is_prefix_op_call(head)  && (str = str*"-pre")
@@ -313,3 +295,32 @@ function bump_split(stream::ParseStream, split_spec::Vararg{Any, N}) where {N}
     stream.peek_count = 0
     return position(stream)
 end
+
+function peek_dotted_op_token(ps, allow_whitespace=false)
+    # Peek the next token, but if it is a dot, peek the next one as well
+    t = peek_token(ps)
+    isdotted = kind(t) == K"."
+    if isdotted
+        t2 = peek_token(ps, 2)
+        if !is_operator(t2) || (!allow_whitespace && preceding_whitespace(t2))
+            isdotted = false
+        else
+            t = t2
+        end
+    end
+    return (isdotted, t)
+end
+
+function bump_dotted(ps, isdot, flags=EMPTY_FLAGS; emit_dot_node=false, remap_kind=K"None")
+    if isdot
+        if emit_dot_node
+            dotmark = position(ps)
+            bump(ps, TRIVIA_FLAG) # TODO: NOTATION_FLAG
+        else
+            bump(ps, TRIVIA_FLAG) # TODO: NOTATION_FLAG
+        end
+    end
+    pos = bump(ps, flags, remap_kind=remap_kind)
+    isdot && emit_dot_node && (pos = emit(ps, dotmark, K"."))
+    return pos
+end

src/julia/kinds.jl

@@ -293,7 +293,9 @@ register_kinds!(JuliaSyntax, 0, [
     "BEGIN_ASSIGNMENTS"
         "BEGIN_SYNTACTIC_ASSIGNMENTS"
         "="
+        ".="
         "op="  # Updating assignment operator ( $= %= &= *= += -= //= /= <<= >>= >>>= \= ^= |= ÷= ⊻= )
+        ".op="
         ":="
         "END_SYNTACTIC_ASSIGNMENTS"
         "~"
@@ -470,11 +472,13 @@ register_kinds!(JuliaSyntax, 0, [
     # Level 4
     "BEGIN_LAZYOR"
         "||"
+        ".||"
     "END_LAZYOR"
 
     # Level 5
     "BEGIN_LAZYAND"
         "&&"
+        ".&&"
     "END_LAZYAND"
 
     # Level 6