Swappable entry point Core._parse for Julia parser

Add Core._parse binding as the internal entry point for parsing of Julia
code. This is called by both C and Julia code which wishes to parse
Julia. For now, it points at the flisp parser by default.

Refactor include() implementation to be done in Julia code, and parsing
rearrangement:

* Implement include() and include_string() in Julia. This also reunifies
  the versions of include() in MainInclude and Base which had started
  diverging.
* Add internal Core.Compiler.fl_parse and set this as the default value
  of the Core._parse binding.
* Use Core._parse from Meta
* Add `Meta.parseall()` for top-level parsing. For now as an internal
  function, but this will likely be made public shortly.

Refactoring in C runtime code:

* Remove flisp code from jl_parse_eval_all. This makes the top level
  parse-lower-eval loop independent of flisp internals.
* Remove jl_parse_eval_all from use as the include implementation (still
  used during bootstrap). This also allowed the removal of mapexpr
  handling from the C code.
* Keep jl_load and jl_load_file_string from julia.h unchanged for
  compatibility with the existing C API. (No julia code `ccall`s
  these anymore)
* Unify flisp parsing to always parse from string rather than directly
  from file.

Author: c42f

base/Base.jl

@@ -365,31 +365,8 @@ for m in methods(include)
 end
 # These functions are duplicated in client.jl/include(::String) for
 # nicer stacktraces. Modifications here have to be backported there
-include(mod::Module, _path::AbstractString) = include(identity, mod, _path)
-function include(mapexpr::Function, mod::Module, _path::AbstractString)
-    path, prev = _include_dependency(mod, _path)
-    for callback in include_callbacks # to preserve order, must come before Core.include
-        invokelatest(callback, mod, path)
-    end
-    tls = task_local_storage()
-    tls[:SOURCE_PATH] = path
-    local result
-    try
-        # result = Core.include(mod, path)
-        if mapexpr === identity
-            result = ccall(:jl_load, Any, (Any, Cstring), mod, path)
-        else
-            result = ccall(:jl_load_rewrite, Any, (Any, Cstring, Any), mod, path, mapexpr)
-        end
-    finally
-        if prev === nothing
-            delete!(tls, :SOURCE_PATH)
-        else
-            tls[:SOURCE_PATH] = prev
-        end
-    end
-    return result
-end
+include(mod::Module, _path::AbstractString) = _include(identity, mod, _path)
+include(mapexpr::Function, mod::Module, _path::AbstractString) = _include(mapexpr, mod, _path)
 
 end_base_include = time_ns()
 

base/boot.jl

@@ -740,4 +740,19 @@ Unsigned(x::Union{Float32, Float64, Bool}) = UInt(x)
 Integer(x::Integer) = x
 Integer(x::Union{Float32, Float64}) = Int(x)
 
+# Binding for the julia parser, called as
+#
+#    Core._parse(text, filename, offset, options)
+#
+# Parse Julia code from the buffer `text`, starting at `offset` and attributing
+# it to `filename`. `text` may be a `String` or `svec(ptr::Ptr{UInt8},
+# len::Int)` for a raw unmanaged buffer. `options` should be one of `:atom`,
+# `:statement` or `:all`, indicating how much the parser will consume.
+#
+# `_parse` must return an `svec` containing an `Expr` and the new offset as an
+# `Int`.
+#
+# The internal jl_parse which will call into Core._parse if not `nothing`.
+_parse = nothing
+
 ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Core, true)

base/client.jl

@@ -154,8 +154,7 @@ function eval_user_input(errio, @nospecialize(ast), show_value::Bool)
 end
 
 function _parse_input_line_core(s::String, filename::String)
-    ex = ccall(:jl_parse_all, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
-               s, sizeof(s), filename, sizeof(filename))
+    ex = Meta.parseall(s, filename=filename)
     if ex isa Expr && ex.head === :toplevel
         if isempty(ex.args)
             return nothing
@@ -439,30 +438,12 @@ end
 # MainInclude exists to hide Main.include and eval from `names(Main)`.
 baremodule MainInclude
 using ..Base
-include(mapexpr::Function, fname::AbstractString) = Base.include(mapexpr, Main, fname)
-# We inline the definition of include from loading.jl/include_relative to get one-frame stacktraces
-# for the common case of include(fname).  Otherwise we would use:
-#    include(fname::AbstractString) = Base.include(Main, fname)
+# These definitions calls Base._include rather than Base.include to get
+# one-frame stacktraces for the common case of using include(fname) in Main.
+include(mapexpr::Function, fname::AbstractString) = Base._include(mapexpr, Main, fname)
 function include(fname::AbstractString)
-    mod = Main
     isa(fname, String) || (fname = Base.convert(String, fname)::String)
-    path, prev = Base._include_dependency(mod, fname)
-    for callback in Base.include_callbacks # to preserve order, must come before Core.include
-        Base.invokelatest(callback, mod, path)
-    end
-    tls = Base.task_local_storage()
-    tls[:SOURCE_PATH] = path
-    local result
-    try
-        result = ccall(:jl_load, Any, (Any, Cstring), mod, path)
-    finally
-        if prev === nothing
-            Base.delete!(tls, :SOURCE_PATH)
-        else
-            tls[:SOURCE_PATH] = prev
-        end
-    end
-    return result
+    Base._include(identity, Main, fname)
 end
 eval(x) = Core.eval(Main, x)
 end

base/compiler/compiler.jl

@@ -113,5 +113,9 @@ include("compiler/optimize.jl") # TODO: break this up further + extract utilitie
 include("compiler/bootstrap.jl")
 ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
 
+include("compiler/parsing.jl")
+Core.eval(Core, :(_parse = Compiler.fl_parse))
+
 end # baremodule Compiler
 ))
+

base/compiler/parsing.jl

@@ -0,0 +1,19 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# Call Julia's builtin flisp-based parser. `offset` is 0-based offset into the
+# byte buffer or string.
+function fl_parse(text::Union{Core.SimpleVector,String},
+                  filename::String, offset, options)
+    if text isa Core.SimpleVector
+        # Will be generated by C entry points jl_parse_string etc
+        text, text_len = text
+    else
+        text_len = sizeof(text)
+    end
+    ccall(:jl_fl_parse, Any, (Ptr{UInt8}, Csize_t, Any, Csize_t, Any),
+          text, text_len, filename, offset, options)
+end
+
+function fl_parse(text::AbstractString, filename::AbstractString, offset, options)
+    fl_parse(String(text), String(filename), offset, options)
+end

base/loading.jl

@@ -1082,14 +1082,31 @@ The optional first argument `mapexpr` can be used to transform the included code
 it is evaluated: for each parsed expression `expr` in `code`, the `include_string` function
 actually evaluates `mapexpr(expr)`.  If it is omitted, `mapexpr` defaults to [`identity`](@ref).
 """
-function include_string(mapexpr::Function, m::Module, txt_::AbstractString, fname::AbstractString="string")
-    txt = String(txt_)
-    if mapexpr === identity
-        ccall(:jl_load_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any),
-            txt, sizeof(txt), String(fname), m)
-    else
-        ccall(:jl_load_rewrite_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any, Any),
-            txt, sizeof(txt), String(fname), m, mapexpr)
+function include_string(mapexpr::Function, mod::Module, code::AbstractString,
+                        filename::AbstractString="string")
+    loc = LineNumberNode(1, Symbol(filename))
+    try
+        ast = Meta.parseall(code, filename=filename)
+        @assert Meta.isexpr(ast, :toplevel)
+        result = nothing
+        line_and_ex = Expr(:toplevel, loc, nothing)
+        for ex in ast.args
+            if ex isa LineNumberNode
+                loc = ex
+                line_and_ex.args[1] = ex
+                continue
+            end
+            ex = mapexpr(ex)
+            # Wrap things to be eval'd in a :toplevel expr to carry line
+            # information as part of the expr.
+            line_and_ex.args[2] = ex
+            result = Core.eval(mod, line_and_ex)
+        end
+        return result
+    catch exc
+        # TODO: Now that stacktraces are more reliable we should remove
+        # LoadError and expose the real error type directly.
+        rethrow(LoadError(filename, loc.line, exc))
     end
 end
 
@@ -1124,7 +1141,28 @@ The optional first argument `mapexpr` can be used to transform the included code
 it is evaluated: for each parsed expression `expr` in `path`, the `include` function
 actually evaluates `mapexpr(expr)`.  If it is omitted, `mapexpr` defaults to [`identity`](@ref).
 """
-Base.include # defined in sysimg.jl
+Base.include # defined in Base.jl
+
+# Full include() implementation which is used after bootstrap
+function _include(mapexpr::Function, mod::Module, _path::AbstractString)
+    @_noinline_meta # Workaround for module availability in _simplify_include_frames
+    path, prev = _include_dependency(mod, _path)
+    for callback in include_callbacks # to preserve order, must come before eval in include_string
+        invokelatest(callback, mod, path)
+    end
+    code = read(path, String)
+    tls = task_local_storage()
+    tls[:SOURCE_PATH] = path
+    try
+        return include_string(mapexpr, mod, code, path)
+    finally
+        if prev === nothing
+            delete!(tls, :SOURCE_PATH)
+        else
+            tls[:SOURCE_PATH] = prev
+        end
+    end
+end
 
 """
     evalfile(path::AbstractString, args::Vector{String}=String[])

base/meta.jl

@@ -149,6 +149,15 @@ struct ParseError <: Exception
     msg::AbstractString
 end
 
+function _parse_string(text::AbstractString, filename::AbstractString,
+                       index::Integer, options)
+    if index < 1 || index > ncodeunits(text) + 1
+        throw(BoundsError(text, index))
+    end
+    ex, offset = Core._parse(text, filename, index-1, options)
+    ex, offset+1
+end
+
 """
     parse(str, start; greedy=true, raise=true, depwarn=true)
 
@@ -171,19 +180,11 @@ julia> Meta.parse("x = 3, y = 5", 5)
 """
 function parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool=true,
                depwarn::Bool=true)
-    # pos is one based byte offset.
-    # returns (expr, end_pos). expr is () in case of parse error.
-    bstr = String(str)
-    # For now, assume all parser warnings are depwarns
-    ex, pos = with_logger(depwarn ? current_logger() : NullLogger()) do
-        ccall(:jl_parse_string, Any,
-              (Ptr{UInt8}, Csize_t, Int32, Int32),
-              bstr, sizeof(bstr), pos-1, greedy ? 1 : 0)
-    end
+    ex, pos = _parse_string(str, "none", pos, greedy ? :statement : :atom)
     if raise && isa(ex,Expr) && ex.head === :error
         throw(ParseError(ex.args[1]))
     end
-    return ex, pos+1 # C is zero-based, Julia is 1-based
+    return ex, pos
 end
 
 """
@@ -223,6 +224,15 @@ function parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true)
     return ex
 end
 
+function parseatom(text::AbstractString, pos::Integer; filename="none")
+    return _parse_string(text, filename, pos, :atom)
+end
+
+function parseall(text::AbstractString; filename="none")
+    ex,_ = _parse_string(text, filename, 1, :all)
+    return ex
+end
+
 """
     partially_inline!(code::Vector{Any}, slot_replacements::Vector{Any},
                       type_signature::Type{<:Tuple}, static_param_values::Vector{Any},