Enable JuliaSyntax.jl as the defult parser

* Vendor JuliaSyntax into Base via deps directory
* Install JuliaSyntax as the Julia parser unless the environment
  variable JULIA_USE_NEW_PARSER=0 is set.
* Add a function to set the Core._parse binding.
  Required because we'd like to set the binding during Base.__init__.
  This can be done with `Core.eval` but that doesn't work well in
  incremental compilation mode.

Also accommodate JuliaSyntax within tests:

* When JuliaSyntax is enabled, ignore error messages in parser tests
  which are tested separately upstream - error messages are inherently
  expressed a bit differently when they go alongside full source
  location info.
* Accommodate a small number of incompatibilities where in JuliaSyntax
  - `import .Mod.x as (a.b)` is a syntax not lowering error
  - `f(2x for x=1:10, y` is `Expr(:incomplete)` not `Expr(:error)`
  - `incomplete_tag` is more precise for `:block` vs `:other`
  - `global const` without an assignment is a syntax error, in keeping
    with plain `const` without assignment being a syntax error (not
    lowering error).
* Adjust a few tests to be more precise about testing lowering vs the
  parser.
* Make Meta.parse doctest compatible with JuliaSyntax errors

Makefile

@@ -365,6 +365,10 @@ endif
 	# Remove various files which should not be installed
 	-rm -f $(DESTDIR)$(datarootdir)/julia/base/version_git.sh
 	-rm -f $(DESTDIR)$(datarootdir)/julia/test/Makefile
+	-rm -f $(DESTDIR)$(datarootdir)/julia/base/*/source-extracted
+	-rm -f $(DESTDIR)$(datarootdir)/julia/base/*/build-configured
+	-rm -f $(DESTDIR)$(datarootdir)/julia/base/*/build-compiled
+	-rm -f $(DESTDIR)$(datarootdir)/julia/base/*/build-checked
 	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/source-extracted
 	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-configured
 	-rm -f $(DESTDIR)$(datarootdir)/julia/stdlib/$(VERSDIR)/*/build-compiled

NEWS.md

@@ -4,6 +4,9 @@ Julia v1.10 Release Notes
 New language features
 ---------------------
 
+* JuliaSyntax.jl is now used as the default parser, providing better diagnostics and faster
+  parsing. Set environment variable `JULIA_USE_NEW_PARSER` to `0` to switch back to the old
+  parser if necessary (and if you find this necessary, please file an issue) ([#46372]).
 * `⥺` (U+297A, `\leftarrowsubset`) and `⥷` (U+2977, `\leftarrowless`)
   may now be used as binary operators with arrow precedence. ([#45962])
 

base/.gitignore

@@ -8,3 +8,4 @@
 /version_git.jl
 /version_git.jl.phony
 /userimg.jl
+/JuliaSyntax

base/Base.jl

@@ -489,6 +489,10 @@ a_method_to_overwrite_in_test() = inferencebarrier(1)
 include(mod::Module, _path::AbstractString) = _include(identity, mod, _path)
 include(mapexpr::Function, mod::Module, _path::AbstractString) = _include(mapexpr, mod, _path)
 
+# External libraries vendored into Base
+Core.println("JuliaSyntax/src/JuliaSyntax.jl")
+include(@__MODULE__, "JuliaSyntax/src/JuliaSyntax.jl")
+
 end_base_include = time_ns()
 
 const _sysimage_modules = PkgId[]
@@ -600,6 +604,9 @@ function __init__()
     _require_world_age[] = get_world_counter()
     # Prevent spawned Julia process from getting stuck waiting on Tracy to connect.
     delete!(ENV, "JULIA_WAIT_FOR_TRACY")
+    if get_bool_env("JULIA_USE_NEW_PARSER", true) === true
+        JuliaSyntax.enable_in_core!()
+    end
     nothing
 end
 

base/boot.jl

@@ -825,9 +825,11 @@ Integer(x::Union{Float16, Float32, Float64}) = Int(x)
 # `_parse` must return an `svec` containing an `Expr` and the new offset as an
 # `Int`.
 #
-# The internal jl_parse which will call into Core._parse if not `nothing`.
+# The internal jl_parse will call into Core._parse if not `nothing`.
 _parse = nothing
 
+_setparser!(parser) = setglobal!(Core, :_parse, parser)
+
 # support for deprecated uses of internal _apply function
 _apply(x...) = Core._apply_iterate(Main.Base.iterate, x...)
 

base/compiler/compiler.jl

@@ -171,7 +171,7 @@ include("compiler/bootstrap.jl")
 ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
 
 include("compiler/parsing.jl")
-Core.eval(Core, :(_parse = Compiler.fl_parse))
+Core._setparser!(fl_parse)
 
 end # baremodule Compiler
 ))

base/meta.jl

@@ -254,20 +254,22 @@ syntax errors will raise an error; otherwise, `parse` will return an expression
 raise an error upon evaluation.  If `depwarn` is `false`, deprecation warnings will be
 suppressed.
 
-```jldoctest
+```jldoctest; filter=r"(?<=Expr\\(:error).*|(?<=Expr\\(:incomplete).*"
 julia> Meta.parse("x = 3")
 :(x = 3)
 
-julia> Meta.parse("x = ")
-:($(Expr(:incomplete, "incomplete: premature end of input")))
-
 julia> Meta.parse("1.0.2")
-ERROR: Base.Meta.ParseError("invalid numeric constant \\\"1.0.\\\"")
-Stacktrace:
+ERROR: ParseError:
+# Error @ none:1:1
+1.0.2
+└──┘ ── invalid numeric constant
 [...]
 
 julia> Meta.parse("1.0.2"; raise = false)
-:($(Expr(:error, "invalid numeric constant \"1.0.\"")))
+:(\$(Expr(:error, "invalid numeric constant \"1.0.\"")))
+
+julia> Meta.parse("x = ")
+:(\$(Expr(:incomplete, "incomplete: premature end of input")))
 ```
 """
 function parse(str::AbstractString; raise::Bool=true, depwarn::Bool=true)

contrib/generate_precompile.jl

@@ -153,7 +153,6 @@ if Artifacts !== nothing
     """
 end
 
-
 Pkg = get(Base.loaded_modules,
           Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg"),
           nothing)

deps/JuliaSyntax.mk

@@ -0,0 +1,16 @@
+$(eval $(call git-external,JuliaSyntax,JULIASYNTAX,,,$(BUILDDIR)))
+
+$(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/build-compiled: $(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/source-extracted
+	@# no build steps
+	echo 1 > $@
+
+$(eval $(call symlink_install,JuliaSyntax,$$(JULIASYNTAX_SRC_DIR),$$(JULIAHOME)/base))
+
+clean-JuliaSyntax:
+	-rm -f $(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/build-compiled
+get-JuliaSyntax: $(JULIASYNTAX_SRC_FILE)
+extract-JuliaSyntax: $(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/source-extracted
+configure-JuliaSyntax: extract-JuliaSyntax
+compile-JuliaSyntax: $(BUILDDIR)/$(JULIASYNTAX_SRC_DIR)/build-compiled
+fastcheck-JuliSyntax: check-JuliSyntax
+check-JuliSyntax: compile-JuliSyntax

deps/JuliaSyntax.version

@@ -0,0 +1,4 @@
+JULIASYNTAX_BRANCH = main
+JULIASYNTAX_SHA1 = ec51994833d78f8c5525bc1647f448dfadc370c1
+JULIASYNTAX_GIT_URL := https://github.com/JuliaLang/JuliaSyntax.jl.git
+JULIASYNTAX_TAR_URL = https://api.github.com/repos/JuliaLang/JuliaSyntax.jl/tarball/$1

deps/Makefile

@@ -36,7 +36,7 @@ BUILDDIR := $(BUILDDIR)$(MAYBE_HOST)
 # prevent installing libs into usr/lib64 on opensuse
 unexport CONFIG_SITE
 
-DEP_LIBS :=
+DEP_LIBS := JuliaSyntax
 
 ifeq ($(USE_SYSTEM_LIBBLASTRAMPOLINE), 0)
 DEP_LIBS += blastrampoline
@@ -188,7 +188,7 @@ DEP_LIBS_STAGED := $(DEP_LIBS)
 DEP_LIBS_STAGED_ALL := llvm llvm-tools clang llvmunwind unwind libuv pcre \
 	openlibm dsfmt blastrampoline openblas lapack gmp mpfr patchelf utf8proc \
 	objconv mbedtls libssh2 nghttp2 curl libgit2 libwhich zlib p7zip csl \
-	libsuitesparse lld libtracyclient ittapi
+	libsuitesparse lld libtracyclient ittapi JuliaSyntax
 DEP_LIBS_ALL := $(DEP_LIBS_STAGED_ALL)
 
 ifneq ($(USE_BINARYBUILDER_OPENBLAS),0)
@@ -248,4 +248,7 @@ include $(SRCDIR)/libwhich.mk
 include $(SRCDIR)/p7zip.mk
 include $(SRCDIR)/libtracyclient.mk
 
+# vendored Julia libs
+include $(SRCDIR)/JuliaSyntax.mk
+
 include $(SRCDIR)/tools/uninstallers.mk

deps/checksums/JuliaSyntax-ec51994833d78f8c5525bc1647f448dfadc370c1.tar.gz/md5

@@ -0,0 +1 @@
+b1d1ccb00e422eb8b70b2120d7083bf3

deps/checksums/JuliaSyntax-ec51994833d78f8c5525bc1647f448dfadc370c1.tar.gz/sha512

@@ -0,0 +1 @@
+e6df6dc2b5d2a5618da0d553eed793e1192147175d84d51f725c0ea8f7b6be92fbeb37de9abee2b2f548b0f0736f836ec7e3e20e93c12f77e1a2b2058bbfd6db

test/cmdlineargs.jl

@@ -924,7 +924,7 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no`
     close(in)
     close(err.in)
     txt = readline(err)
-    @test startswith(txt, "ERROR: syntax: incomplete")
+    @test startswith(txt, r"ERROR: (syntax: incomplete|ParseError:)")
 end
 
 # Issue #29855

test/show.jl

@@ -633,7 +633,7 @@ end
 @test_repr "::@m(x, y) + z"
 @test_repr "[@m(x) y z]"
 @test_repr "[@m(x) y; z]"
-@test_repr "let @m(x), y=z; end"
+test_repr("let @m(x), y=z; end", true)
 
 @test repr(:(@m x y))    == ":(#= $(@__FILE__):$(@__LINE__) =# @m x y)"
 @test string(:(@m x y))  ==   "#= $(@__FILE__):$(@__LINE__) =# @m x y"

test/strings/basic.jl

@@ -250,8 +250,6 @@ end
     @test string(sym) == string(Char(0xdcdb))
     @test String(sym) == string(Char(0xdcdb))
     @test Meta.lower(Main, sym) === sym
-    @test Meta.parse(string(Char(0xe0080)," = 1"), 1, raise=false)[1] ==
-        Expr(:error, "invalid character \"\Ue0080\" near column 1")
 end
 
 @testset "Symbol and gensym" begin
@@ -761,11 +759,6 @@ function getData(dic)
 end
 @test getData(Dict()) == ",,,,,,,,,,,,,,,,,,"
 
-@testset "unrecognized escapes in string/char literals" begin
-    @test_throws Meta.ParseError Meta.parse("\"\\.\"")
-    @test_throws Meta.ParseError Meta.parse("\'\\.\'")
-end
-
 @testset "thisind" begin
     let strs = Any["∀α>β:α+1>β", s"∀α>β:α+1>β",
                    SubString("123∀α>β:α+1>β123", 4, 18),