load path: change default LOAD_PATH (#25709)

also:

- rename "site" directory to "stdlib" since that's what it is now

- use JULIA_LOAD_PATH as-is instead unconditionally appending the
  system package and stdlib directories to it

- change default DEPOT_PATH to include system paths: one for
  arch-specific and one for arch-independent packages

- delete comment about bundled code going in a versioned directory
  as it no longer applies since installed packages can and should
  be shared across different Julia versions

- update Pkg3 and tests to work correctly when the stdlib directory
  isn't always included in LOAD_PATH

fix failing tests

Makefile

@@ -1,20 +1,13 @@
 JULIAHOME := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
 include $(JULIAHOME)/Make.inc
 
-# TODO: Code bundled with Julia should be installed into a versioned directory,
-# prefix/share/julia/VERSDIR, so that in the future one can have multiple
-# major versions of Julia installed concurrently. Third-party code that
-# is not controlled by Pkg should be installed into
-# prefix/share/julia/site/VERSDIR (not prefix/share/julia/VERSDIR/site ...
-# so that prefix/share/julia/VERSDIR can be overwritten without touching
-# third-party code).
 VERSDIR := v`cut -d. -f1-2 < $(JULIAHOME)/VERSION`
 
 default: $(JULIA_BUILD_MODE) # contains either "debug" or "release"
 all: debug release
 
 # sort is used to remove potential duplicates
-DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/site $(build_man1dir))
+DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/stdlib $(build_man1dir))
 ifneq ($(BUILDROOT),$(JULIAHOME))
 BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src ui doc deps test test/embedding)
 BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS))
@@ -46,8 +39,8 @@ endif
 $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
 $(foreach link,base $(JULIAHOME)/test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia,$(notdir $(link)))))
 
-build_defaultpkgdir = $(build_datarootdir)/julia/site/$(shell echo $(VERSDIR))
-$(eval $(call symlink_target,$(JULIAHOME)/stdlib,$(build_datarootdir)/julia/site,$(shell echo $(VERSDIR))))
+build_defaultpkgdir = $(build_datarootdir)/julia/stdlib/$(shell echo $(VERSDIR))
+$(eval $(call symlink_target,$(JULIAHOME)/stdlib,$(build_datarootdir)/julia/stdlib,$(shell echo $(VERSDIR))))
 
 julia_flisp.boot.inc.phony: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony
@@ -284,7 +277,7 @@ endef
 
 install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index.html
 	@$(MAKE) $(QUIET_MAKE) all
-	@for subdir in $(bindir) $(datarootdir)/julia/site/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir); do \
+	@for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir); do \
 		mkdir -p $(DESTDIR)$$subdir; \
 	done
 

base/initdefs.jl

@@ -48,7 +48,9 @@ function init_depot_path(BINDIR = Sys.BINDIR)
         depots = split(ENV["JULIA_DEPOT_PATH"], Sys.iswindows() ? ';' : ':')
         append!(empty!(DEPOT_PATH), map(expanduser, depots))
     else
-        push!(DEPOT_PATH, joinpath(homedir(), ".julia"))
+        push!(empty!(DEPOT_PATH), joinpath(homedir(), ".julia"))
+        push!(DEPOT_PATH, abspath(BINDIR, "..", "local", "share", "julia"))
+        push!(DEPOT_PATH, abspath(BINDIR, "..", "share", "julia"))
     end
 end
 
@@ -115,14 +117,15 @@ function parse_load_path(str::String)
     return envs
 end
 
+const default_named = parse_load_path("@v#.#.#|@v#.#|@v#|@default|@!v#.#")
+
 function init_load_path(BINDIR = Sys.BINDIR)
-    if !Base.creating_sysimg
-        load_path = get(ENV, "JULIA_LOAD_PATH", "@|@v#.#.#|@v#.#|@v#|@default|@!v#.#")
-        append!(empty!(LOAD_PATH), parse_load_path(load_path))
-    end
     vers = "v$(VERSION.major).$(VERSION.minor)"
-    push!(LOAD_PATH, abspath(BINDIR, "..", "local", "share", "julia", "site", vers))
-    push!(LOAD_PATH, abspath(BINDIR, "..", "share", "julia", "site", vers))
+    stdlib = abspath(BINDIR, "..", "share", "julia", "stdlib", vers)
+    load_path = Base.creating_sysimg           ? Any[stdlib] :
+                haskey(ENV, "JULIA_LOAD_PATH") ? parse_load_path(ENV["JULIA_LOAD_PATH"]) :
+                                                 Any[CurrentEnv(); default_named; stdlib]
+    append!(empty!(LOAD_PATH), load_path)
 end
 
 const atexit_hooks = []

base/loading.jl

@@ -196,7 +196,7 @@ end
 
 find_env(env::Function) = find_env(env())
 
-load_path() = filter(env -> env ≠ nothing, map(find_env, LOAD_PATH))
+load_path() = String[env for env in map(find_env, LOAD_PATH) if env ≠ nothing]
 
 ## package identification: determine unique identity of package to be loaded ##
 

doc/src/manual/environment-variables.md

@@ -68,23 +68,13 @@ and a global configuration search path of
 ### `JULIA_LOAD_PATH`
 
 A separated list of absolute paths that are to be appended to the variable
-[`LOAD_PATH`](@ref). (In Unix-like systems, the path separator is `:`; in Windows
-systems, the path separator is `;`.) The `LOAD_PATH` variable is where
-[`Base.require`](@ref) and `Base.load_in_path()` look for code; it defaults to the absolute
-paths
-
-```
-$JULIA_HOME/../local/share/julia/site/v$(VERSION.major).$(VERSION.minor)
-$JULIA_HOME/../share/julia/site/v$(VERSION.major).$(VERSION.minor)
-```
-
-so that, e.g., version 0.6 of Julia on a Linux system with a Julia executable at
-`/bin/julia` will have a default `LOAD_PATH` of
-
-```
-/local/share/julia/site/v0.6
-/share/julia/site/v0.6
-```
+[`LOAD_PATH`](@ref). (In Unix-like systems, the path separator is `:`; in
+Windows systems, the path separator is `;`.) The `LOAD_PATH` variable is where
+[`Base.require`](@ref) and `Base.load_in_path()` look for code; it defaults to
+the absolute path
+`$JULIA_HOME/../share/julia/stdlib/v$(VERSION.major).$(VERSION.minor)` so that,
+e.g., version 0.7 of Julia on a Linux system with a Julia executable at
+`/bin/julia` will have a default `LOAD_PATH` of `/share/julia/stdlib/v0.7`.
 
 ### `JULIA_PKGDIR`
 

src/common_symbols2.inc

@@ -127,7 +127,6 @@ jl_symbol("pipe_writer"),
 jl_symbol("idxfloor"),
 jl_symbol("id"),
 jl_symbol("ComplexF32"),
-jl_symbol("/home/jeff/src/julia/usr/share/julia/site/v0.7/Pkg/src/resolve/versionweight.jl"),
 jl_symbol("Int32"),
 jl_symbol("indices.jl"),
 jl_symbol("iobuffer.jl"),

stdlib/Pkg3/bin/genstdlib.jl

@@ -6,7 +6,7 @@ prefix = joinpath(homedir(), ".julia", "registries", "Stdlib")
 
 # TODO: use Sys.STDLIBDIR instead once implemented
 let vers = "v$(VERSION.major).$(VERSION.minor)"
-global stdlibdir = realpath(abspath(Sys.BINDIR, "..", "share", "julia", "site", vers))
+global stdlibdir = realpath(abspath(Sys.BINDIR, "..", "share", "julia", "stdlib", vers))
 isdir(stdlibdir) || error("stdlib directory does not exist: $stdlibdir")
 end
 juliadir = dirname(stdlibdir)

stdlib/Pkg3/src/API.jl

@@ -352,7 +352,7 @@ function build(ctx::Context, pkgs::Vector{PackageSpec}; kwargs...)
 end
 
 init() = init(Context())
-init(path::String) = init(Context(), path)
+init(path::String) = init(Context(env=EnvCache(path)), path)
 function init(ctx::Context, path::String=pwd())
     print_first_command_header()
     Context!(ctx; env = EnvCache(joinpath(path, "Project.toml")))

stdlib/Pkg3/src/Operations.jl

@@ -753,9 +753,8 @@ function with_dependencies_loadable_at_toplevel(f, mainctx::Context, pkg::Packag
         end
         write_env(localctx, display_diff = false)
         will_resolve && build_versions(localctx, new)
-        withenv("JULIA_LOAD_PATH" => joinpath(tmpdir)) do
-            f()
-        end
+        sep = Sys.iswindows() ? ';' : ':'
+        withenv(f, "JULIA_LOAD_PATH" => "$tmpdir$sep$(Types.stdlib_dir())")
     end
 end
 
@@ -823,7 +822,6 @@ function build_versions(ctx::Context, uuids::Vector{UUID}; might_need_to_resolve
         log_file = splitext(build_file)[1] * ".log"
         printpkgstyle(ctx, :Building,
             rpad(name * " ", max_name + 1, "─"), "→ ", Types.pathrepr(ctx, log_file))
-
         code = """
             empty!(Base.DEPOT_PATH)
             append!(Base.DEPOT_PATH, $(repr(map(abspath, DEPOT_PATH))))

stdlib/Pkg3/src/Types.jl

@@ -467,17 +467,33 @@ mutable struct EnvCache
             for entry in LOAD_PATH
                 project_file = Base.find_env(entry)
                 project_file isa String && !isdir(project_file) && break
+                project_file = nothing
+            end
+            if project_file == nothing
+                project_dir = nothing
+                for entry in LOAD_PATH
+                    project_dir = Base.find_env(entry)
+                    project_dir isa String && isdir(project_dir) && break
+                    project_dir = nothing
+                end
+                project_dir == nothing && error("No Pkg3 environment found in LOAD_PATH")
+                project_file = joinpath(project_dir, Base.project_names[end])
             end
-            project_file === nothing && error("No Pkg3 environment found in LOAD_PATH")
         elseif env isa AbstractEnv
             project_file = Base.find_env(env)
             project_file === nothing && error("package environment does not exist: $env")
         elseif env isa String
-            isdir(env) && error("environment is a package directory: $env")
-            project_file = endswith(env, ".toml") ? abspath(env) :
-                                                    abspath(env, Base.project_names[end])
+            if isdir(env)
+                isempty(readdir(env)) || error("environment is a package directory: $env")
+                project_file = joinpath(env, Base.project_names[end])
+            else
+                project_file = endswith(env, ".toml") ? abspath(env) :
+                    abspath(env, Base.project_names[end])
+            end
         end
-        @assert project_file isa String && (isfile(project_file) || !ispath(project_file))
+        @assert project_file isa String &&
+            (isfile(project_file) || !ispath(project_file) ||
+             isdir(project_file) && isempty(readdir(project_file)))
         project_dir = dirname(project_file)
         git = ispath(joinpath(project_dir, ".git")) ? LibGit2.GitRepo(project_dir) : nothing
 
@@ -529,7 +545,7 @@ is_project_uuid(env::EnvCache, uuid::UUID) =
 ###########
 # Context #
 ###########
-stdlib_dir() = joinpath(Sys.BINDIR, "..", "share", "julia", "site", "v$(VERSION.major).$(VERSION.minor)")
+stdlib_dir() = joinpath(Sys.BINDIR, "..", "share", "julia", "stdlib", "v$(VERSION.major).$(VERSION.minor)")
 stdlib_path(stdlib::String) = joinpath(stdlib_dir(), stdlib)
 function gather_stdlib_uuids()
     stdlibs = Dict{UUID,String}()

stdlib/Pkg3/test/utils.jl

@@ -13,8 +13,7 @@ function temp_pkg_dir(fn::Function)
                 pushfirst!(DEPOT_PATH, depot_dir)
                 # Add the standard library paths back
                 vers = "v$(VERSION.major).$(VERSION.minor)"
-                push!(LOAD_PATH, abspath(Sys.BINDIR, "..", "local", "share", "julia", "site", vers))
-                push!(LOAD_PATH, abspath(Sys.BINDIR, "..", "share", "julia", "site", vers))
+                push!(LOAD_PATH, abspath(Sys.BINDIR, "..", "share", "julia", "stdlib", vers))
                 fn(env_dir)
             end
         end
@@ -34,4 +33,4 @@ function cd_tempdir(f)
     end
 end
 
-isinstalled(pkg) = Base.locate_package(Base.PkgId(pkg.uuid, pkg.name)) !== nothing
+isinstalled(pkg) = Base.locate_package(Base.PkgId(pkg.uuid, pkg.name)) !== nothing

test/choosetests.jl

@@ -2,7 +2,7 @@
 
 using Random, Sockets
 
-const STDLIB_DIR = joinpath(Sys.BINDIR, "..", "share", "julia", "site", "v$(VERSION.major).$(VERSION.minor)")
+const STDLIB_DIR = joinpath(Sys.BINDIR, "..", "share", "julia", "stdlib", "v$(VERSION.major).$(VERSION.minor)")
 const STDLIBS = readdir(STDLIB_DIR)
 
 """