build: break up monolithic sysimg output work

Makefile

@@ -149,9 +149,12 @@ $(build_datarootdir)/julia/julia-config.jl: $(JULIAHOME)/contrib/julia-config.jl
 $(build_depsbindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(build_depsbindir)
 	@$(call PRINT_CC, $(HOSTCC) -o $(build_depsbindir)/stringreplace $(JULIAHOME)/contrib/stringreplace.c)
 
-julia-base-cache: julia-sysimg-$(JULIA_BUILD_MODE) | $(DIRS) $(build_datarootdir)/julia
-	@JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) $(call spawn, $(JULIA_EXECUTABLE) --startup-file=no $(call cygpath_w,$(JULIAHOME)/etc/write_base_cache.jl) \
-		$(call cygpath_w,$(build_datarootdir)/julia/base.cache))
+julia-base-cache: $(build_datarootdir)/julia/base.cache
+
+# this depends on the phony target julia-sysimg- to force rebuilding
+$(build_datarootdir)/julia/base.cache: $(JULIAHOME)/etc/write_base_cache.jl julia-sysimg-$(JULIA_BUILD_MODE) | $(DIRS) $(build_datarootdir)/julia
+	@JULIA_BINDIR=$(call cygpath_w,$(build_bindir)) \
+		$(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no $(call cygpath_w,$<) $(call cygpath_w,$@)
 
 # public libraries, that are installed in $(prefix)/lib
 JL_TARGETS := julia julialoader
@@ -359,8 +362,10 @@ endif
 	cp -R -L $(build_includedir)/julia/* $(DESTDIR)$(includedir)/julia
 	# Copy system image
 	$(INSTALL_M) $(build_private_libdir)/sys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir)
+	$(INSTALL_M) $(build_private_libdir)/fullsys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir)
 ifeq ($(BUNDLE_DEBUG_LIBS),1)
 	$(INSTALL_M) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) $(DESTDIR)$(private_libdir)
+	$(INSTALL_M) $(build_private_libdir)/fullsys-debug.$(SHLIB_EXT) $(DESTDIR)$(private_libdir)
 endif
 
 	# Copy in all .jl sources as well
@@ -616,7 +621,9 @@ LLVM_SIZE := $(build_depsbindir)/llvm-size$(EXE)
 endif
 build-stats:
 	@printf $(JULCOLOR)' ==> ./julia binary sizes\n'$(ENDCOLOR)
-	$(call spawn,$(LLVM_SIZE) -A $(call cygpath_w,$(build_private_libdir)/sys.$(SHLIB_EXT)) \
+	$(call spawn,$(LLVM_SIZE) -A \
+		$(call cygpath_w,$(build_private_libdir)/fullsys.$(SHLIB_EXT)) \
+		$(call cygpath_w,$(build_private_libdir)/sys.$(SHLIB_EXT)) \
 		$(call cygpath_w,$(build_shlibdir)/libjulia.$(SHLIB_EXT)) \
 		$(call cygpath_w,$(build_bindir)/julia$(EXE)))
 	@printf $(JULCOLOR)' ==> ./julia launch speedtest\n'$(ENDCOLOR)

base/.gitignore

@@ -7,4 +7,3 @@
 /uv_constants.jl
 /version_git.jl
 /version_git.jl.phony
-/userimg.jl

base/Base.jl

@@ -424,4 +424,11 @@ end
 
 end
 
+# Ensure this file is also tracked
+pushfirst!(_included_files, (Main, abspath(@__FILE__)))
+
 end # baremodule Base
+
+# Set up Main module
+using .Base
+import Base.MainInclude: eval, include

base/initdefs.jl

@@ -94,6 +94,7 @@ function append_default_depot_path!(DEPOT_PATH)
     path in DEPOT_PATH || push!(DEPOT_PATH, path)
     path = abspath(Sys.BINDIR::String, "..", "share", "julia")
     path in DEPOT_PATH || push!(DEPOT_PATH, path)
+    return DEPOT_PATH
 end
 
 function init_depot_path()
@@ -112,6 +113,7 @@ function init_depot_path()
     else
         append_default_depot_path!(DEPOT_PATH)
     end
+    nothing
 end
 
 ## LOAD_PATH & ACTIVE_PROJECT ##
@@ -217,9 +219,7 @@ function parse_load_path(str::String)
 end
 
 function init_load_path()
-    if Base.creating_sysimg
-        paths = ["@stdlib"]
-    elseif haskey(ENV, "JULIA_LOAD_PATH")
+    if haskey(ENV, "JULIA_LOAD_PATH")
         paths = parse_load_path(ENV["JULIA_LOAD_PATH"])
     else
         paths = filter!(env -> env !== nothing,

doc/Manifest.toml

@@ -20,10 +20,10 @@ uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 version = "0.8.3"
 
 [[Documenter]]
-deps = ["Base64", "Dates", "DocStringExtensions", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"]
-git-tree-sha1 = "fb1ff838470573adc15c71ba79f8d31328f035da"
+deps = ["Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"]
+git-tree-sha1 = "a4875e0763112d6d017126f3944f4133abb342ae"
 uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
-version = "0.25.2"
+version = "0.25.5"
 
 [[DocumenterLaTeX]]
 deps = ["Documenter", "Test"]
@@ -32,9 +32,15 @@ uuid = "cd674d7a-5f81-5cf3-af33-235ef1834b99"
 version = "0.2.0"
 
 [[Downloads]]
-deps = ["ArgTools", "LibCURL"]
+deps = ["ArgTools", "LibCURL", "NetworkOptions"]
 uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 
+[[IOCapture]]
+deps = ["Logging"]
+git-tree-sha1 = "377252859f740c217b936cebcd918a44f9b53b59"
+uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
+version = "0.1.1"
+
 [[InteractiveUtils]]
 deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
@@ -54,7 +60,7 @@ deps = ["Libdl"]
 uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
 
 [[LibGit2]]
-deps = ["Printf"]
+deps = ["NetworkOptions", "Printf"]
 uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 
 [[Libdl]]
@@ -73,14 +79,17 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 [[MozillaCACerts_jll]]
 uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
 
+[[NetworkOptions]]
+uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
+
 [[Parsers]]
 deps = ["Dates"]
-git-tree-sha1 = "6fa4202675c05ba0f8268a6ddf07606350eda3ce"
+git-tree-sha1 = "b417be52e8be24e916e34b3d70ec2da7bdf56a68"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "1.0.11"
+version = "1.0.12"
 
 [[Pkg]]
-deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "UUIDs"]
+deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [[Printf]]
@@ -108,6 +117,10 @@ uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
 deps = ["Dates"]
 uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 
+[[Tar]]
+deps = ["ArgTools", "SHA"]
+uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+
 [[Test]]
 deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

src/init.c

@@ -540,7 +540,7 @@ static const char *absformat(const char *in)
     return out;
 }
 
-static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel)
+static const char *jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel, int just_repl)
 {   // this function resolves the paths in jl_options to absolute file locations as needed
     // and it replaces the pointers to `julia_bindir`, `julia_bin`, `image_file`, and output file paths
     // it may fail, print an error, and exit(1) if any of these paths are longer than PATH_MAX
@@ -569,7 +569,24 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel)
         jl_options.julia_bindir = abspath(jl_options.julia_bindir, 0);
     free(free_path);
     free_path = NULL;
+    const char *image_file = NULL;
     if (jl_options.image_file) {
+        if (jl_options.image_file == jl_get_default_sysimg_path() && just_repl) {
+            // if we're just launching a REPL for the default environment
+            // interpose a slightly more complete environment at startup
+            free_path = (char*)malloc_s(PATH_MAX);
+            int n = snprintf(free_path, PATH_MAX, "%s" PATHSEPSTRING "%sfull",
+                             jl_options.julia_bindir, jl_options.image_file);
+            if (n >= PATH_MAX || n < 0) {
+                jl_error("fatal error: jl_options.image_file path too long");
+            }
+            // swap filename and "full" prefix in string to form the target name
+            char *dirp = strrchr(free_path, PATHSEPSTRING[0]) + strlen(PATHSEPSTRING);
+            memmove(dirp + 4, dirp, n - (dirp - free_path) - 4);
+            memcpy(dirp, "full", 4);
+            image_file = free_path;
+            free_path = NULL;
+        }
         if (rel == JL_IMAGE_JULIA_HOME && !isabspath(jl_options.image_file)) {
             // build time path, relative to JULIA_BINDIR
             free_path = (char*)malloc_s(PATH_MAX);
@@ -579,13 +596,13 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel)
                 jl_error("fatal error: jl_options.image_file path too long");
             }
             jl_options.image_file = free_path;
+            free_path = NULL;
         }
-        if (jl_options.image_file)
+        else {
             jl_options.image_file = abspath(jl_options.image_file, 0);
-        if (free_path) {
-            free(free_path);
-            free_path = NULL;
         }
+        if (image_file == NULL)
+            image_file = jl_options.image_file;
     }
     if (jl_options.outputo)
         jl_options.outputo = abspath(jl_options.outputo, 0);
@@ -609,6 +626,8 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel)
             }
         }
     }
+
+    return image_file;
 }
 
 static void jl_set_io_wait(int v)
@@ -626,7 +645,7 @@ static void restore_fp_env(void)
     }
 }
 
-void _julia_init(JL_IMAGE_SEARCH rel)
+void _julia_init(JL_IMAGE_SEARCH rel, int just_repl)
 {
     jl_init_timing();
     // Make sure we finalize the tls callback before starting any threads.
@@ -728,15 +747,15 @@ void _julia_init(JL_IMAGE_SEARCH rel)
 
     jl_gc_enable(0);
 
-    jl_resolve_sysimg_location(rel);
+    const char *image_file = jl_resolve_sysimg_location(rel, just_repl);
     // loads sysimg if available, and conditionally sets jl_options.cpu_target
-    if (jl_options.image_file)
-        jl_preload_sysimg_so(jl_options.image_file);
+    if (image_file)
+        jl_preload_sysimg_so(image_file);
     if (jl_options.cpu_target == NULL)
         jl_options.cpu_target = "native";
 
-    if (jl_options.image_file) {
-        jl_restore_system_image(jl_options.image_file);
+    if (image_file) {
+        jl_restore_system_image(image_file);
     }
     else {
         jl_init_types();
@@ -752,7 +771,7 @@ void _julia_init(JL_IMAGE_SEARCH rel)
     jl_init_flisp();
     jl_init_serializer();
 
-    if (!jl_options.image_file) {
+    if (!image_file) {
         jl_core_module = jl_new_module(jl_symbol("Core"));
         jl_core_module->parent = jl_core_module;
         jl_type_typename->mt->module = jl_core_module;
@@ -786,7 +805,7 @@ void _julia_init(JL_IMAGE_SEARCH rel)
 
     jl_gc_enable(1);
 
-    if (jl_options.image_file && (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) {
+    if (image_file && (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) {
         jl_array_t *init_order = jl_module_init_order;
         JL_GC_PUSH1(&init_order);
         jl_module_init_order = NULL;