Support building of system image in binary builds

Makefile

@@ -123,11 +123,11 @@ $(build_private_libdir)/sys%$(SHLIB_EXT): $(build_private_libdir)/sys%o
 
 $(build_private_libdir)/sys0.o:
 	@$(QUIET_JULIA) cd base && \
-	$(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys0) sysimg.jl
+	$(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --build $(call cygpath_w,$(build_private_libdir)/sys0) sysimg.jl
 
 $(build_private_libdir)/sys.o: VERSION base/*.jl base/pkg/*.jl base/linalg/*.jl base/sparse/*.jl $(build_datarootdir)/julia/helpdb.jl $(build_datarootdir)/man/man1/julia.1 $(build_private_libdir)/sys0.$(SHLIB_EXT)
 	@$(QUIET_JULIA) cd base && \
-	$(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys) \
+	$(call spawn,$(JULIA_EXECUTABLE)) -C $(JULIA_CPU_TARGET) --build $(call cygpath_w,$(build_private_libdir)/sys) \
 		-J$(call cygpath_w,$(build_private_libdir))/$$([ -e $(build_private_libdir)/sys.ji ] && echo sys.ji || echo sys0.ji) -f sysimg.jl \
 		|| (echo "*** This error is usually fixed by running 'make clean'. If the error persists, try 'make cleanall'. ***" && false)
 
@@ -258,6 +258,8 @@ endif
 	# Copy system image
 	$(INSTALL_F) $(build_private_libdir)/sys.ji $(DESTDIR)$(private_libdir)
 	$(INSTALL_M) $(build_private_libdir)/sys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir)
+	# Copy in system image build script
+	$(INSTALL_M) contrib/build_sysimg.jl $(DESTDIR)$(datarootdir)/julia/
 	# Copy in all .jl sources as well
 	cp -R -L $(build_datarootdir)/julia $(DESTDIR)$(datarootdir)/
 	# Remove git repository of juliadoc
@@ -332,10 +334,8 @@ ifeq ($(JULIA_CPU_TARGET), native)
 endif
 
 ifeq ($(OS), WINNT)
-ifeq ($(ARCH),x86_64)
-	# If we are running on WIN64, also delete sys.dll until we switch to llvm3.5+
+	# If we are running on WINNT, also delete sys.dll until it stops causing issues (#8895, among others)
 	-rm -f $(DESTDIR)$(private_libdir)/sys.$(SHLIB_EXT)
-endif
 
 	[ ! -d dist-extras ] || ( cd dist-extras && \
 		cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll $(bindir) && \

contrib/build_sysimg.jl

@@ -0,0 +1,168 @@
+#!/usr/bin/env julia
+
+# Build a system image binary at sysimg_path.dlext.  By default, put the system image
+# next to libjulia (except on Windows, where it goes in $JULIA_HOME\..\lib\julia)
+# Allow insertion of a userimg via userimg_path.  If sysimg_path.dlext is currently loaded into memory,
+# don't continue unless force is set to true.  Allow targeting of a CPU architecture via cpu_target
+@unix_only const default_sysimg_path = joinpath(dirname(Sys.dlpath("libjulia")),"sys")
+@windows_only const default_sysimg_path = joinpath(JULIA_HOME,"..","lib","julia","sys")
+function build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", userimg_path=nothing; force=false)
+    # Quit out if a sysimg is already loaded and is in the same spot as sysimg_path, unless forcing
+    sysimg = dlopen_e("sys")
+    if sysimg != C_NULL
+        if !force && Base.samefile(Sys.dlpath(sysimg), "$(sysimg_path).$(Sys.dlext)")
+            info("System image already loaded at $(Sys.dlpath(sysimg)), set force to override")
+            return
+        end
+    end
+
+    # Canonicalize userimg_path before we enter the base_dir
+    if userimg_path != nothing
+        userimg_path = abspath(userimg_path)
+    end
+
+    # Enter base/ and setup some useful paths
+    base_dir = dirname(Base.find_source_file("sysimg.jl"))
+    cd(base_dir) do
+        try
+            julia = joinpath(JULIA_HOME, "julia")
+            ld = find_system_linker()
+
+            # Ensure we have write-permissions to wherever we're trying to write to
+            try
+                touch("$sysimg_path.ji")
+            catch
+                err_msg =  "Unable to modify $sysimg_path.ji, ensure parent directory exists "
+                err_msg *= "and is writable. Absolute paths work best. Do you need to run this with sudo?)"
+                error( err_msg )
+            end
+
+            # Copy in userimg.jl if it exists...
+            if userimg_path != nothing
+                if !isreadable(userimg_path)
+                    error("$userimg_path is not readable, ensure it is an absolute path!")
+                end
+                cp(userimg_path, "userimg.jl")
+            end
+
+            # Start by building sys0.{ji,o}
+            sys0_path = joinpath(dirname(sysimg_path), "sys0")
+            info("Building sys0.o...")
+            println("$julia -C $cpu_target --build $sys0_path sysimg.jl")
+            run(`$julia -C $cpu_target --build $sys0_path sysimg.jl`)
+
+            # Bootstrap off of that to create sys.{ji,o}
+            info("Building sys.o...")
+            println("$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl")
+            run(`$julia -C $cpu_target --build $sysimg_path -J $sys0_path.ji -f sysimg.jl`)
+
+            if ld != nothing
+                link_sysimg(sysimg_path, ld)
+            else
+                info("System image successfully built at $sysimg_path.ji")
+            end
+
+            if !Base.samefile("$default_sysimg_path.ji", "$sysimg_path.ji")
+                info("To run Julia with this image loaded, run: julia -J $sysimg_path.ji")
+            else
+                info("Julia will automatically load this system image at next startup")
+            end
+        finally
+            # Cleanup userimg.jl
+            if isfile("userimg.jl")
+                rm("userimg.jl")
+            end
+        end
+    end
+end
+
+# Search for a linker to link sys.o into sys.dl_ext.  Honor LD environment variable.
+function find_system_linker()
+    if haskey( ENV, "LD" )
+        if !success(`$(ENV["LD"]) -v`)
+            warn("Using linker override $(ENV["LD"]), but unable to run `$(ENV["LD"]) -v`")
+        end
+        return ENV["LD"]
+    end
+
+    # On Windows, check to see if WinRPM is installed, and if so, see if gcc is installed
+    @windows_only try
+        require("WinRPM")
+        winrpmgcc = joinpath(WinRPM.installdir,"usr","$(Sys.ARCH)-w64-mingw32",
+            "sys-root","mingw","bin","gcc.exe")
+        if success(`$winrpmgcc --version`)
+            return winrpmgcc
+        else
+            throw()
+        end
+    catch
+        warn("Install GCC via `Pkg.add(\"WinRPM\"); WinRPM.install(\"gcc\")` to generate sys.dll for faster startup times")
+    end
+
+
+    # See if `ld` exists
+    try
+        if success(`ld -v`)
+            return "ld"
+        end
+    end
+
+    warn( "No supported linker found; startup times will be longer" )
+end
+
+# Link sys.o into sys.$(dlext)
+function link_sysimg(sysimg_path=default_sysimg_path, ld=find_system_linker())
+    julia_libdir = dirname(Sys.dlpath("libjulia"))
+
+    FLAGS = ["-L$julia_libdir"]
+    if OS_NAME == :Darwin
+        push!(FLAGS, "-dylib")
+        push!(FLAGS, "-undefined")
+        push!(FLAGS, "dynamic_lookup")
+        push!(FLAGS, "-macosx_version_min")
+        push!(FLAGS, "10.7")
+    else
+        push!(FLAGS, "-shared")
+        # on windows we link using gcc for now
+        wl = @windows? "-Wl," : ""
+        push!(FLAGS, wl * "--unresolved-symbols")
+        push!(FLAGS, wl * "ignore-all")
+    end
+    @windows_only append!(FLAGS, ["-ljulia", "-lssp-0"])
+
+    info("Linking sys.$(Sys.dlext)")
+    run(`$ld $FLAGS -o $sysimg_path.$(Sys.dlext) $sysimg_path.o`)
+
+    info("System image successfully built at $sysimg_path.$(Sys.dlext)")
+    @windows_only begin
+        if convert(VersionNumber, Base.libllvm_version) < v"3.5.0"
+            LLVM_msg = "Building sys.dll on Windows against LLVM < 3.5.0 can cause incorrect backtraces!"
+            LLVM_msg *= " Delete generated sys.dll to avoid these problems"
+            warn( LLVM_msg )
+        end
+    end
+end
+
+# When running this file as a script, try to do so with default values.  If arguments are passed
+# in, use them as the arguments to build_sysimg above
+if !isinteractive()
+    if length(ARGS) > 4 || ("--help" in ARGS || "-h" in ARGS)
+        println("Usage: build_sysimg.jl <sysimg_path> <cpu_target> <usrimg_path.jl> [--force] [--help]")
+        println("   <sysimg_path>    is an absolute, extensionless path to store the system image at")
+        println("   <cpu_target>     is an LLVM cpu target to build the system image against")
+        println("   <usrimg_path.lj> is the path to a user image to be baked into the system image")
+        println("   --force          Set if you wish to overwrite the default system image")
+        println("   --help           Print out this help text and exit")
+        println()
+        println(" Example:")
+        println("   build_sysimg.jl /usr/local/lib/julia/sys core2 ~/my_usrimg.jl true")
+        println()
+        println(" Running this script with no arguments is equivalent to calling it via")
+        println("   build_sysimg.jl $(default_sysimg_path) native")
+        return 0
+    end
+
+    force_flag = "--force" in ARGS
+    filter!(x -> x != "--force", ARGS)
+    build_sysimg(ARGS..., force=force_flag)
+end

doc/devdocs/julia.rst

@@ -8,3 +8,4 @@
    :maxdepth: 1
 
    cartesian
+   sysimg

doc/devdocs/sysimg.rst

@@ -0,0 +1,35 @@
+*********************
+System Image Building
+*********************
+
+Building the Julia system image
+-------------------------------
+
+Julia ships with a preparsed system image containing the contents of the ``Base`` module, named ``sys.ji``.  This file is also precompiled into a shared library called ``sys.{so,dll,dylib}`` on as many platforms as possible, so as to give vastly improved startup times.  On systems that do not ship with a precompiled system image file, one can be generated from the source files shipped in Julia's ``DATAROOTDIR/julia/base`` folder.
+
+This operation is useful for multiple reasons.  A user may:
+
+* Build a precompiled shared library system image on a platform that did not ship with one, thereby improving startup times.
+
+* Modify ``Base``, rebuild the system image and use the new ``Base`` next time Julia is started.
+
+* Include a ``userimg.jl`` file that includes packages into the system image, thereby creating a system image that has packages embedded into the startup environment.
+
+Julia now ships with a script that automates the tasks of building the system image, wittingly named ``build_sysimg.jl`` that lives in ``DATAROOTDIR/julia/``.  That is, to include it into a current Julia session, type:
+::
+
+   include(joinpath(JULIA_HOME, Base.DATAROOTDIR, "julia", "build_sysimg.jl"))
+
+This will include a ``build_sysimg()`` function:
+
+.. function:: build_sysimg(sysimg_path=default_sysimg_path, cpu_target="native", userimg_path=nothing; force=false)
+
+   Rebuild the system image. Store it in ``sysimg_path``, which defaults to a file named ``sys.ji`` that sits in the same folder as ``libjulia.{so,dylib}``, except on Windows where it defaults to ``JULIA_HOME/../lib/julia/sys.ji``.
+   Use the cpu instruction set given by ``cpu_target``.  Valid CPU targets are the same as for the ``-C`` option to ``julia``, or the ``-march`` option to ``gcc``.  Defaults to ``native``, which means to use all CPU instructions available on the current processor.
+   Include the user image file given by ``userimg_path``, which should contain directives such as ``using MyPackage`` to include that package in the new system image.
+   New system image will not replace an older image unless ``force`` is set to true.
+
+Note that this file can also be run as a script itself, with command line arguments taking the place of arguments passed to the ``build_sysimg`` function.  For example, to build a system image in ``/tmp/sys.{so,dll,dylib}``, with the ``core2`` CPU instruction set, a user image of ``~/userimg.jl`` and ``force`` set to ``true``, one would execute:
+::
+
+   julia build_sysimg.jl /tmp/sys core2 ~/userimg.jl --force

src/cgutils.cpp

@@ -240,7 +240,7 @@ static void jl_gen_llvm_gv_array()
             ConstantInt::get(T_size,globalUnique+1),
             "jl_globalUnique");
 
-    Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_cpu_string);
+    Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_compileropts.cpu_target);
     new GlobalVariable(*jl_Module,
                        feature_string->getType(),
                        true,
@@ -249,7 +249,7 @@ static void jl_gen_llvm_gv_array()
                        "jl_sysimg_cpu_target");
 
     // For native also store the cpuid
-    if (strcmp(jl_cpu_string,"native") == 0) {
+    if (strcmp(jl_compileropts.cpu_target,"native") == 0) {
         uint32_t info[4];
 
         jl_cpuid((int32_t*)info, 1);

src/codegen.cpp

@@ -506,10 +506,7 @@ static Type *NoopType;
 
 // --- utilities ---
 
-#define XSTR(x) #x
-#define MSTR(x) XSTR(x)
 extern "C" {
-    const char *jl_cpu_string = MSTR(JULIA_TARGET_ARCH);
     int globalUnique = 0;
 }
 
@@ -4510,7 +4507,6 @@ extern "C" void jl_init_codegen(void)
     jl_setup_module(engine_module,false);
 #endif
 
-
 #if !defined(LLVM_VERSION_MAJOR) || (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR == 0)
     jl_ExecutionEngine = EngineBuilder(m).setEngineKind(EngineKind::JIT).create();
 #ifdef JL_DEBUG_BUILD
@@ -4576,9 +4572,9 @@ extern "C" void jl_init_codegen(void)
             TheTriple,
             "",
 #if LLVM35
-            strcmp(jl_cpu_string,"native") ? jl_cpu_string : sys::getHostCPUName().data(),
+            strcmp(jl_compileropts.cpu_target,"native") ? jl_compileropts.cpu_target : sys::getHostCPUName().data(),
 #else
-            strcmp(jl_cpu_string,"native") ? jl_cpu_string : "",
+            strcmp(jl_compileropts.cpu_target,"native") ? jl_compileropts.cpu_target : "",
 #endif
             MAttrs);
     assert(jl_TargetMachine);

src/dump.c

@@ -102,7 +102,6 @@ jl_value_t ***sysimg_gvars = NULL;
 
 extern int globalUnique;
 extern void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType);
-extern const char *jl_cpu_string;
 uv_lib_t *jl_sysimg_handle = NULL;
 uint64_t jl_sysimage_base = 0;
 #ifdef _OS_WINDOWS_
@@ -122,21 +121,21 @@ static void jl_load_sysimg_so(char *fname)
         sysimg_gvars = (jl_value_t***)jl_dlsym(jl_sysimg_handle, "jl_sysimg_gvars");
         globalUnique = *(size_t*)jl_dlsym(jl_sysimg_handle, "jl_globalUnique");
         const char *cpu_target = (const char*)jl_dlsym(jl_sysimg_handle, "jl_sysimg_cpu_target");
-        if (strcmp(cpu_target,jl_cpu_string) != 0)
+        if (strcmp(cpu_target,jl_compileropts.cpu_target) != 0)
             jl_error("Julia and the system image were compiled for different architectures.\n"
                      "Please delete or regenerate sys.{so,dll,dylib}.\n");
         uint32_t info[4];
         jl_cpuid((int32_t*)info, 1);
         if (strcmp(cpu_target, "native") == 0) {
             uint64_t saved_cpuid = *(uint64_t*)jl_dlsym(jl_sysimg_handle, "jl_sysimg_cpu_cpuid");
             if (saved_cpuid != (((uint64_t)info[2])|(((uint64_t)info[3])<<32)))
-                jl_error("Target architecture mismatch. Please delete or regenerate sys.{so,dll,dylib}.");
+                jl_error("Target architecture mismatch. Please delete or regenerate sys.{so,dll,dylib}.\n");
         }
         else if (strcmp(cpu_target,"core2") == 0) {
             int HasSSSE3 = (info[2] & 1<<9);
             if (!HasSSSE3)
                 jl_error("The current host does not support SSSE3, but the system image was compiled for Core2.\n"
-                         "Please delete or regenerate sys.{so,dll,dylib}.");
+                         "Please delete or regenerate sys.{so,dll,dylib}.\n");
         }
 #ifdef _OS_WINDOWS_
         jl_sysimage_base = (intptr_t)jl_sysimg_handle->handle;
@@ -1196,6 +1195,32 @@ jl_value_t *jl_compress_ast(jl_lambda_info_t *li, jl_value_t *ast)
     return v;
 }
 
+// Takes in a path of the form "usr/lib/julia/sys.ji", such as passed in to jl_restore_system_image()
+DLLEXPORT
+const char * jl_get_system_image_cpu_target(const char *fname)
+{
+    // If passed NULL, don't even bother
+    if (!fname)
+        return NULL;
+
+    // First, get "sys" from "sys.ji"
+    char *fname_shlib = (char*)alloca(strlen(fname)+1);
+    strcpy(fname_shlib, fname);
+    char *fname_shlib_dot = strrchr(fname_shlib, '.');
+    if (fname_shlib_dot != NULL)
+        *fname_shlib_dot = 0;
+
+    // Get handle to sys.so
+    uv_lib_t * sysimg_handle = jl_load_dynamic_library_e(fname_shlib, JL_RTLD_DEFAULT | JL_RTLD_GLOBAL);
+
+    // Return jl_sysimg_cpu_target if we can
+    if (sysimg_handle)
+        return (const char *)jl_dlsym(sysimg_handle, "jl_sysimg_cpu_target");
+
+    // If something goes wrong, return NULL
+    return NULL;
+}
+
 DLLEXPORT
 jl_value_t *jl_uncompress_ast(jl_lambda_info_t *li, jl_value_t *data)
 {

src/init.c

@@ -81,6 +81,7 @@ DLLEXPORT void gdblookup(ptrint_t ip);
 
 char *julia_home = NULL;
 jl_compileropts_t jl_compileropts = { NULL, // build_path
+                                      NULL, // cpu_target ("native", "core2", etc...)
                                       0,    // code_coverage
                                       0,    // malloc_log
                                       JL_COMPILEROPT_CHECK_BOUNDS_DEFAULT,
@@ -746,6 +747,13 @@ void julia_init(char *imageFile)
 {
     jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.),
                                     // best to call this first, since it also initializes libuv
+    // If we are able to load the sysimg and get a cpu_target, use that unless user has overridden
+    if (jl_compileropts.cpu_target == NULL) {
+        const char * sysimg_cpu_target = jl_get_system_image_cpu_target(imageFile);
+
+        // If we can't load anything from the sysimg, default to native
+        jl_compileropts.cpu_target = sysimg_cpu_target ? sysimg_cpu_target : "native";
+    }
     jl_page_size = jl_getpagesize();
     jl_arr_xtralloc_limit = uv_get_total_memory() / 100;  // Extra allocation limited to 1% of total RAM 
     jl_find_stack_bottom();

src/julia.h

@@ -867,6 +867,7 @@ DLLEXPORT void jl_init_with_image(char *julia_home_dir, char *image_relative_pat
 DLLEXPORT int jl_is_initialized(void);
 DLLEXPORT extern char *julia_home;
 
+DLLEXPORT const char * jl_get_system_image_cpu_target(const char *fname);
 DLLEXPORT void jl_save_system_image(char *fname);
 DLLEXPORT void jl_restore_system_image(char *fname);
 void jl_init_restored_modules();
@@ -1322,6 +1323,7 @@ void show_execution_point(char *filename, int lno);
 
 typedef struct {
     char *build_path;
+    const char *cpu_target;
     int8_t code_coverage;
     int8_t malloc_log;
     int8_t check_bounds;