Merge pull request #35243 from JuliaLang/cjf/parser-api

Replaceable single entry point for parser

base/Base.jl

@@ -365,31 +365,8 @@ for m in methods(include)
 end
 # These functions are duplicated in client.jl/include(::String) for
 # nicer stacktraces. Modifications here have to be backported there
-include(mod::Module, _path::AbstractString) = include(identity, mod, _path)
-function include(mapexpr::Function, mod::Module, _path::AbstractString)
-    path, prev = _include_dependency(mod, _path)
-    for callback in include_callbacks # to preserve order, must come before Core.include
-        invokelatest(callback, mod, path)
-    end
-    tls = task_local_storage()
-    tls[:SOURCE_PATH] = path
-    local result
-    try
-        # result = Core.include(mod, path)
-        if mapexpr === identity
-            result = ccall(:jl_load, Any, (Any, Cstring), mod, path)
-        else
-            result = ccall(:jl_load_rewrite, Any, (Any, Cstring, Any), mod, path, mapexpr)
-        end
-    finally
-        if prev === nothing
-            delete!(tls, :SOURCE_PATH)
-        else
-            tls[:SOURCE_PATH] = prev
-        end
-    end
-    return result
-end
+include(mod::Module, _path::AbstractString) = _include(identity, mod, _path)
+include(mapexpr::Function, mod::Module, _path::AbstractString) = _include(mapexpr, mod, _path)
 
 end_base_include = time_ns()
 

base/boot.jl

@@ -740,4 +740,19 @@ Unsigned(x::Union{Float32, Float64, Bool}) = UInt(x)
 Integer(x::Integer) = x
 Integer(x::Union{Float32, Float64}) = Int(x)
 
+# Binding for the julia parser, called as
+#
+#    Core._parse(text, filename, offset, options)
+#
+# Parse Julia code from the buffer `text`, starting at `offset` and attributing
+# it to `filename`. `text` may be a `String` or `svec(ptr::Ptr{UInt8},
+# len::Int)` for a raw unmanaged buffer. `options` should be one of `:atom`,
+# `:statement` or `:all`, indicating how much the parser will consume.
+#
+# `_parse` must return an `svec` containing an `Expr` and the new offset as an
+# `Int`.
+#
+# The internal jl_parse which will call into Core._parse if not `nothing`.
+_parse = nothing
+
 ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Core, true)

base/client.jl

@@ -154,8 +154,7 @@ function eval_user_input(errio, @nospecialize(ast), show_value::Bool)
 end
 
 function _parse_input_line_core(s::String, filename::String)
-    ex = ccall(:jl_parse_all, Any, (Ptr{UInt8}, Csize_t, Ptr{UInt8}, Csize_t),
-               s, sizeof(s), filename, sizeof(filename))
+    ex = Meta.parseall(s, filename=filename)
     if ex isa Expr && ex.head === :toplevel
         if isempty(ex.args)
             return nothing
@@ -439,30 +438,12 @@ end
 # MainInclude exists to hide Main.include and eval from `names(Main)`.
 baremodule MainInclude
 using ..Base
-include(mapexpr::Function, fname::AbstractString) = Base.include(mapexpr, Main, fname)
-# We inline the definition of include from loading.jl/include_relative to get one-frame stacktraces
-# for the common case of include(fname).  Otherwise we would use:
-#    include(fname::AbstractString) = Base.include(Main, fname)
+# These definitions calls Base._include rather than Base.include to get
+# one-frame stacktraces for the common case of using include(fname) in Main.
+include(mapexpr::Function, fname::AbstractString) = Base._include(mapexpr, Main, fname)
 function include(fname::AbstractString)
-    mod = Main
     isa(fname, String) || (fname = Base.convert(String, fname)::String)
-    path, prev = Base._include_dependency(mod, fname)
-    for callback in Base.include_callbacks # to preserve order, must come before Core.include
-        Base.invokelatest(callback, mod, path)
-    end
-    tls = Base.task_local_storage()
-    tls[:SOURCE_PATH] = path
-    local result
-    try
-        result = ccall(:jl_load, Any, (Any, Cstring), mod, path)
-    finally
-        if prev === nothing
-            Base.delete!(tls, :SOURCE_PATH)
-        else
-            tls[:SOURCE_PATH] = prev
-        end
-    end
-    return result
+    Base._include(identity, Main, fname)
 end
 eval(x) = Core.eval(Main, x)
 end

base/compiler/compiler.jl

@@ -113,5 +113,9 @@ include("compiler/optimize.jl") # TODO: break this up further + extract utilitie
 include("compiler/bootstrap.jl")
 ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
 
+include("compiler/parsing.jl")
+Core.eval(Core, :(_parse = Compiler.fl_parse))
+
 end # baremodule Compiler
 ))
+

base/compiler/parsing.jl

@@ -0,0 +1,19 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# Call Julia's builtin flisp-based parser. `offset` is 0-based offset into the
+# byte buffer or string.
+function fl_parse(text::Union{Core.SimpleVector,String},
+                  filename::String, offset, options)
+    if text isa Core.SimpleVector
+        # Will be generated by C entry points jl_parse_string etc
+        text, text_len = text
+    else
+        text_len = sizeof(text)
+    end
+    ccall(:jl_fl_parse, Any, (Ptr{UInt8}, Csize_t, Any, Csize_t, Any),
+          text, text_len, filename, offset, options)
+end
+
+function fl_parse(text::AbstractString, filename::AbstractString, offset, options)
+    fl_parse(String(text), String(filename), offset, options)
+end

base/errorshow.jl

@@ -681,6 +681,41 @@ function is_kw_sorter_name(name::Symbol)
     return !startswith(sn, '#') && endswith(sn, "##kw")
 end
 
+# For improved user experience, filter out frames for include() implementation
+# - see #33065. See also #35371 for extended discussion of internal frames.
+function _simplify_include_frames(trace)
+    i = length(trace)
+    kept_frames = trues(i)
+    first_ignored = nothing
+    while i >= 1
+        frame, _ = trace[i]
+        mod = parentmodule(frame)
+        if isnothing(first_ignored)
+            if mod === Base && frame.func === :_include
+                # Hide include() machinery by default
+                first_ignored = i
+            end
+        else
+            # Hack: allow `mod==nothing` as a workaround for inlined functions.
+            # TODO: Fix this by improving debug info.
+            if mod in (Base,Core,nothing) && 1+first_ignored-i <= 5
+                if frame.func == :eval
+                    kept_frames[i:first_ignored] .= false
+                    first_ignored = nothing
+                end
+            else
+                # Bail out to avoid hiding frames in unexpected circumstances
+                first_ignored = nothing
+            end
+        end
+        i -= 1
+    end
+    if !isnothing(first_ignored)
+        kept_frames[i:first_ignored] .= false
+    end
+    return trace[kept_frames]
+end
+
 function process_backtrace(t::Vector, limit::Int=typemax(Int); skipC = true)
     n = 0
     last_frame = StackTraces.UNKNOWN
@@ -721,7 +756,7 @@ function process_backtrace(t::Vector, limit::Int=typemax(Int); skipC = true)
     if n > 0
         push!(ret, (last_frame, n))
     end
-    return ret
+    return _simplify_include_frames(ret)
 end
 
 function show_exception_stack(io::IO, stack::Vector)

base/loading.jl

@@ -1082,14 +1082,31 @@ The optional first argument `mapexpr` can be used to transform the included code
 it is evaluated: for each parsed expression `expr` in `code`, the `include_string` function
 actually evaluates `mapexpr(expr)`.  If it is omitted, `mapexpr` defaults to [`identity`](@ref).
 """
-function include_string(mapexpr::Function, m::Module, txt_::AbstractString, fname::AbstractString="string")
-    txt = String(txt_)
-    if mapexpr === identity
-        ccall(:jl_load_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any),
-            txt, sizeof(txt), String(fname), m)
-    else
-        ccall(:jl_load_rewrite_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any, Any),
-            txt, sizeof(txt), String(fname), m, mapexpr)
+function include_string(mapexpr::Function, mod::Module, code::AbstractString,
+                        filename::AbstractString="string")
+    loc = LineNumberNode(1, Symbol(filename))
+    try
+        ast = Meta.parseall(code, filename=filename)
+        @assert Meta.isexpr(ast, :toplevel)
+        result = nothing
+        line_and_ex = Expr(:toplevel, loc, nothing)
+        for ex in ast.args
+            if ex isa LineNumberNode
+                loc = ex
+                line_and_ex.args[1] = ex
+                continue
+            end
+            ex = mapexpr(ex)
+            # Wrap things to be eval'd in a :toplevel expr to carry line
+            # information as part of the expr.
+            line_and_ex.args[2] = ex
+            result = Core.eval(mod, line_and_ex)
+        end
+        return result
+    catch exc
+        # TODO: Now that stacktraces are more reliable we should remove
+        # LoadError and expose the real error type directly.
+        rethrow(LoadError(filename, loc.line, exc))
     end
 end
 
@@ -1124,7 +1141,28 @@ The optional first argument `mapexpr` can be used to transform the included code
 it is evaluated: for each parsed expression `expr` in `path`, the `include` function
 actually evaluates `mapexpr(expr)`.  If it is omitted, `mapexpr` defaults to [`identity`](@ref).
 """
-Base.include # defined in sysimg.jl
+Base.include # defined in Base.jl
+
+# Full include() implementation which is used after bootstrap
+function _include(mapexpr::Function, mod::Module, _path::AbstractString)
+    @_noinline_meta # Workaround for module availability in _simplify_include_frames
+    path, prev = _include_dependency(mod, _path)
+    for callback in include_callbacks # to preserve order, must come before eval in include_string
+        invokelatest(callback, mod, path)
+    end
+    code = read(path, String)
+    tls = task_local_storage()
+    tls[:SOURCE_PATH] = path
+    try
+        return include_string(mapexpr, mod, code, path)
+    finally
+        if prev === nothing
+            delete!(tls, :SOURCE_PATH)
+        else
+            tls[:SOURCE_PATH] = prev
+        end
+    end
+end
 
 """
     evalfile(path::AbstractString, args::Vector{String}=String[])

base/meta.jl

@@ -149,6 +149,15 @@ struct ParseError <: Exception
     msg::AbstractString
 end
 
+function _parse_string(text::AbstractString, filename::AbstractString,
+                       index::Integer, options)
+    if index < 1 || index > ncodeunits(text) + 1
+        throw(BoundsError(text, index))
+    end
+    ex, offset = Core._parse(text, filename, index-1, options)
+    ex, offset+1
+end
+
 """
     parse(str, start; greedy=true, raise=true, depwarn=true)
 
@@ -171,19 +180,11 @@ julia> Meta.parse("x = 3, y = 5", 5)
 """
 function parse(str::AbstractString, pos::Integer; greedy::Bool=true, raise::Bool=true,
                depwarn::Bool=true)
-    # pos is one based byte offset.
-    # returns (expr, end_pos). expr is () in case of parse error.
-    bstr = String(str)
-    # For now, assume all parser warnings are depwarns
-    ex, pos = with_logger(depwarn ? current_logger() : NullLogger()) do
-        ccall(:jl_parse_string, Any,
-              (Ptr{UInt8}, Csize_t, Int32, Int32),
-              bstr, sizeof(bstr), pos-1, greedy ? 1 : 0)
-    end
+    ex, pos = _parse_string(str, "none", pos, greedy ? :statement : :atom)
     if raise && isa(ex,Expr) && ex.head === :error
         throw(ParseError(ex.args[1]))
     end
-    return ex, pos+1 # C is zero-based, Julia is 1-based
+    return ex, pos
 end
 
 """
@@ -223,6 +224,15 @@ function parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true)
     return ex
 end
 
+function parseatom(text::AbstractString, pos::Integer; filename="none")
+    return _parse_string(text, filename, pos, :atom)
+end
+
+function parseall(text::AbstractString; filename="none")
+    ex,_ = _parse_string(text, filename, 1, :all)
+    return ex
+end
+
 """
     partially_inline!(code::Vector{Any}, slot_replacements::Vector{Any},
                       type_signature::Type{<:Tuple}, static_param_values::Vector{Any},

base/stacktraces.jl

@@ -268,22 +268,28 @@ function show(io::IO, frame::StackFrame; full_path::Bool=false)
     end
 end
 
+function Base.parentmodule(frame::StackFrame)
+    if frame.linfo isa MethodInstance
+        def = frame.linfo.def
+        if def isa Module
+            return def
+        else
+            return (def::Method).module
+        end
+    else
+        # The module is not always available (common reasons include inlined
+        # frames and frames arising from the interpreter)
+        nothing
+    end
+end
+
 """
     from(frame::StackFrame, filter_mod::Module) -> Bool
 
 Returns whether the `frame` is from the provided `Module`
 """
 function from(frame::StackFrame, m::Module)
-    finfo = frame.linfo
-    result = false
-
-    if finfo isa MethodInstance
-        frame_m = finfo.def
-        isa(frame_m, Method) && (frame_m = frame_m.module)
-        result = nameof(frame_m) === nameof(m)
-    end
-
-    return result
+    return parentmodule(frame) === m
 end
 
 end

src/.gitignore

@@ -20,3 +20,7 @@
 /julia_version.h
 /flisp/host
 /support/host
+
+# Clang compilation database
+/compile_commands*.json
+.clangd/