add parseatom and parseall (#724)

Author: simeonschaub

Project.toml

@@ -1,6 +1,6 @@
 name = "Compat"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "3.19.0"
+version = "3.20.0"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

README.md

@@ -55,6 +55,13 @@ changes in `julia`.
 
 ## Supported features
 
+* `Compat.parseatom(text::AbstractString, pos::Integer; filename="none")` parses a single
+  atom from `text` starting at index `pos`. Returns a `Tuple` consisting of the
+  parsed expression and the index to resume parsing from. ([#35243]) (since Compat 3.20)
+
+* `Compat.parseall(text::AbstractString; filename="none")` parses the whole string `text`
+  and returns the parsed expression. ([#35243]) (since Compat 3.20)
+
 * `mul!(C, A, B, alpha, beta)` now performs in-place multiply add ([#29634]). (since Compat 3.19).
 
 * `reinterpret(reshape, T, a::AbstractArray{S})` reinterprets `a` to have eltype `T` while
@@ -211,3 +218,4 @@ Note that you should specify the correct minimum version for `Compat` in the
 [#37517]: https://github.com/JuliaLang/julia/pull/37517
 [#37559]: https://github.com/JuliaLang/julia/pull/37559
 [#29634]: https://github.com/JuliaLang/julia/pull/29634
+[#35243]: https://github.com/JuliaLang/julia/pull/35243

src/Compat.jl

@@ -762,6 +762,50 @@ if VERSION < v"1.3.0-alpha.115"
     end
 end
 
+# https://github.com/JuliaLang/julia/pull/35243
+if VERSION < v"1.6.0-DEV.15"
+    _replace_filename(@nospecialize(x), filename, line_offset=0) = x
+    function _replace_filename(x::LineNumberNode, filename, line_offset=0)
+        return LineNumberNode(x.line + line_offset, filename)
+    end
+    function _replace_filename(ex::Expr, filename, line_offset=0)
+        return Expr(
+            ex.head,
+            Any[_replace_filename(i, filename, line_offset) for i in ex.args]...,
+        )
+    end
+
+    function parseatom(text::AbstractString, pos::Integer; filename="none")
+        ex, i = Meta.parse(text, pos, greedy=false)
+        return _replace_filename(ex, Symbol(filename)), i
+    end
+
+    function _skip_newlines(text, line, i)
+        while i <= lastindex(text) && isspace(text[i])
+            line += text[i] == '\n'
+            i = nextind(text, i)
+        end
+        return line, i
+    end
+    
+    function parseall(text::AbstractString; filename="none")
+        filename = Symbol(filename)
+        ex = Expr(:toplevel)
+        line, prev_i = _skip_newlines(text, 1, firstindex(text))
+        ex_n, i = Meta.parse(text, prev_i)
+        while ex_n !== nothing
+            push!(ex.args, LineNumberNode(line, filename))
+            push!(ex.args, _replace_filename(ex_n, filename, line-1))
+            line += count(==('\n'), SubString(text, prev_i:prevind(text, i)))
+            line, prev_i = _skip_newlines(text, line, i)
+            ex_n, i = Meta.parse(text, prev_i)
+        end
+        return ex
+    end
+else
+    using .Meta: parseatom, parseall
+end
+
 include("iterators.jl")
 include("deprecated.jl")
 

test/runtests.jl

@@ -698,6 +698,59 @@ end
     @test  Cmut == mul!(Cmut, B, x, alpha, beta) ≈ ((B * x * alpha) + (C * beta))
 end
 
+# https://github.com/JuliaLang/julia/pull/35243
+@testset "parseatom and parseall" begin
+    @test Compat.parseatom(raw"foo$(@bar)baz", 5; filename="foo") ==
+        (Expr(:macrocall, Symbol("@bar"), LineNumberNode(1, :foo)), 11)
+
+    ex = Compat.parseall(
+        raw"""
+        begin
+            @a b
+            @c() + 1
+        end
+        """;
+        filename="foo",
+    )
+    @test ex == Expr(:toplevel,
+        LineNumberNode(1, :foo),
+        Expr(:block,
+            LineNumberNode(2, :foo),
+            Expr(:macrocall, Symbol("@a"), LineNumberNode(2, :foo), :b),
+            LineNumberNode(3, :foo),
+            Expr(:call,
+                 :+,
+                 Expr(:macrocall, Symbol("@c"), LineNumberNode(3, :foo)),
+                 1,
+            ),
+        ),
+    )
+
+    ex = Compat.parseall(
+        raw"""
+        
+        begin a = 1 end
+
+        begin
+            b = 2
+        end
+        """;
+        filename="foo",
+    )
+    @test ex == Expr(:toplevel,
+        LineNumberNode(2, :foo),
+        Expr(:block,
+            LineNumberNode(2, :foo),
+            :(a = 1),
+        ),
+        LineNumberNode(4, :foo),
+        Expr(:block,
+            LineNumberNode(5, :foo),
+            :(b = 2),
+        ),
+    )
+end
+
 include("iterators.jl")
 
 nothing