add programatic interface to caching modules. renames sys.ji to Base.…

…ji for consistency
JuliaLang · Dec 6, 2014 · da2f13c · da2f13c
1 parent f9bfc9e
commit da2f13c
Show file tree

Hide file tree

Showing 8 changed files with 84 additions and 67 deletions.
diff --git a/Make.inc b/Make.inc
@@ -368,7 +368,7 @@ RANLIB := $(CROSS_COMPILE)ranlib
 
 
 # if not absolute, then relative to the directory of the julia executable
-JCPPFLAGS += "-DJL_SYSTEM_IMAGE_PATH=\"$(build_private_libdir_rel)/sys.ji\""
+JCPPFLAGS += "-DJL_SYSTEM_IMAGE_PATH=\"$(build_private_libdir_rel)/Base.ji\""
 
 # On Windows, we want shared library files to end up in $(build_bindir), instead of $(build_libdir)
 ifeq ($(OS),WINNT)

diff --git a/Makefile b/Makefile
@@ -48,7 +48,7 @@ endif
 debug release: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test $(build_docdir) $(build_sysconfdir)/julia/juliarc.jl $(build_man1dir)/julia.1
 	@$(MAKE) $(QUIET_MAKE) julia-$@
 	@export private_libdir=$(private_libdir) && \
-	$(MAKE) $(QUIET_MAKE) LD_LIBRARY_PATH=$(build_libdir):$(LD_LIBRARY_PATH) JULIA_EXECUTABLE="$(JULIA_EXECUTABLE_$@)" $(build_private_libdir)/sys.$(SHLIB_EXT)
+	$(MAKE) $(QUIET_MAKE) LD_LIBRARY_PATH=$(build_libdir):$(LD_LIBRARY_PATH) JULIA_EXECUTABLE="$(JULIA_EXECUTABLE_$@)" $(build_private_libdir)/Base.$(SHLIB_EXT)
 
 release-candidate: release test
 	@#Check documentation
@@ -123,12 +123,12 @@ $(build_sysconfdir)/julia/juliarc.jl: contrib/windows/juliarc.jl
 endif
 
 # use sys.ji if it exists, otherwise run two stages
-$(build_private_libdir)/sys%ji: $(build_private_libdir)/sys%o
+$(build_private_libdir)/%.ji: $(build_private_libdir)/%.o
 
-.SECONDARY: $(build_private_libdir)/sys.o
-.SECONDARY: $(build_private_libdir)/sys0.o
+.SECONDARY: $(build_private_libdir)/Base.o
+.SECONDARY: $(build_private_libdir)/Base0.o
 
-$(build_private_libdir)/sys%$(SHLIB_EXT): $(build_private_libdir)/sys%o
+$(build_private_libdir)/%.$(SHLIB_EXT): $(build_private_libdir)/%.o
 ifneq ($(USEMSVC), 1)
 	@$(call PRINT_LINK, $(CXX) -shared -fPIC -L$(build_private_libdir) -L$(build_libdir) -L$(build_shlibdir) -o $@ $< \
 		$$([ $(OS) = Darwin ] && echo '' -Wl,-undefined,dynamic_lookup || echo '' -Wl,--unresolved-symbols,ignore-all ) \
@@ -138,17 +138,21 @@ else
 	@true
 endif
 
-$(build_private_libdir)/sys0.o:
+$(build_private_libdir)/Base0.o:
 	@$(call PRINT_JULIA, cd base && \
-	$(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys0) sysimg.jl)
+	$(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)) sysimg.jl)
+	mv $(build_private_libdir)/Base.ji $(build_private_libdir)/Base0.ji
+	mv $(build_private_libdir)/Base.o $(build_private_libdir)/Base0.o
 
 BASE_SRCS := $(wildcard base/*.jl base/*/*.jl base/*/*/*.jl)
 
+# This rule defines $(,) to be an escaped ',' character
 ,:=,
-$(build_private_libdir)/sys.o: VERSION $(BASE_SRCS) $(build_docdir)/helpdb.jl $(build_private_libdir)/sys0.$(SHLIB_EXT)
+
+$(build_private_libdir)/Base.o: VERSION $(BASE_SRCS) $(build_docdir)/helpdb.jl $(build_private_libdir)/Base0.$(SHLIB_EXT)
 	@$(call PRINT_JULIA, cd base && \
-	$(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys) \
-		-J$(call cygpath_w,$(build_private_libdir))/$$([ -e $(build_private_libdir)/sys.ji ] && echo sys.ji || echo sys0.ji) -f sysimg.jl \
+	$(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)) \
+		-J$(call cygpath_w,$(build_private_libdir))/$$([ -e $(build_private_libdir)/Base.ji ] && echo Base.ji || echo Base0.ji) -f sysimg.jl \
 		|| { echo '*** This error is usually fixed by running `make clean`. If the error persists$(,) try `make cleanall`. ***' && false; } )
 
 $(build_bindir)/stringreplace: contrib/stringreplace.c | $(build_bindir)
@@ -263,8 +267,8 @@ endif
 
 	$(INSTALL_F) src/julia.h src/options.h src/support/*.h $(DESTDIR)$(includedir)/julia
 	# Copy system image
-	$(INSTALL_F) $(build_private_libdir)/sys.ji $(DESTDIR)$(private_libdir)
-	$(INSTALL_M) $(build_private_libdir)/sys.$(SHLIB_EXT) $(DESTDIR)$(private_libdir)
+	$(INSTALL_F) $(build_private_libdir)/Base.ji $(DESTDIR)$(private_libdir)
+	$(INSTALL_M) $(build_private_libdir)/Base.$(SHLIB_EXT) $(DESTDIR)$(private_libdir)
 	# Copy in all .jl sources as well
 	cp -R -L $(build_datarootdir)/julia $(DESTDIR)$(datarootdir)/
 	# Copy documentation
@@ -302,7 +306,7 @@ endif
 
 	# Overwrite JL_SYSTEM_IMAGE_PATH in julia binaries
 	for julia in $(DESTDIR)$(bindir)/julia* ; do \
-		$(call spawn,$(build_bindir)/stringreplace $$(strings -t x - $$julia | grep "sys.ji$$" | awk '{print $$1;}' ) "$(private_libdir_rel)/sys.ji" 256 $(call cygpath_w,$$julia)); \
+		$(call spawn,$(build_bindir)/stringreplace $$(strings -t x - $$julia | grep "Base.ji$$" | awk '{print $$1;}' ) "$(private_libdir_rel)/Base.ji" 256 $(call cygpath_w,$$julia)); \
 	done
 endif
 
@@ -337,15 +341,15 @@ ifeq ($(OS), Darwin)
 	-cat ./contrib/mac/juliarc.jl >> $(DESTDIR)$(prefix)/etc/julia/juliarc.jl
 endif
 
-	# purge sys.{dll,so,dylib} as that file is not relocatable across processor architectures
+	# purge Base.{dll,so,dylib} as that file is not relocatable across processor architectures
 ifeq ($(JULIA_CPU_TARGET), native)
-	-rm -f $(DESTDIR)$(private_libdir)/sys.$(SHLIB_EXT)
+	-rm -f $(DESTDIR)$(private_libdir)/Base.$(SHLIB_EXT)
 endif
 
 ifeq ($(OS), WINNT)
 ifeq ($(ARCH),x86_64)
 	# If we are running on WIN64, also delete sys.dll until we switch to llvm3.5+
-	-rm -f $(DESTDIR)$(private_libdir)/sys.$(SHLIB_EXT)
+	-rm -f $(DESTDIR)$(private_libdir)/Base.$(SHLIB_EXT)
 endif
 
 	[ ! -d dist-extras ] || ( cd dist-extras && \

diff --git a/src/Windows.mk b/src/Windows.mk
@@ -47,7 +47,7 @@ LIB = $(LIB);C:\Program Files\llvm\lib\Release
 !endif
 
 CFLAGS = $(CFLAGS) -DCOPY_STACKS -D_CRT_SECURE_NO_WARNINGS
-CFLAGS = $(CFLAGS) -DJL_SYSTEM_IMAGE_PATH=\"../lib/julia/sys.ji\" -DLIBRARY_EXPORTS
+CFLAGS = $(CFLAGS) -DJL_SYSTEM_IMAGE_PATH=\"../lib/julia/Base.ji\" -DLIBRARY_EXPORTS
 
 LIBWINDOWS = \
 	kernel32.lib \

diff --git a/src/dump.c b/src/dump.c
@@ -884,8 +884,8 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc)
         if (tag == 7) {
             jl_tuple_t *parameters = (jl_tuple_t*)jl_deserialize_value(s, NULL);
             dtv = jl_apply_type(dtv, parameters);
-            backref_list.items[pos] = dtv;
         }
+        backref_list.items[pos] = dtv;
         return dtv;
     }
     uint16_t nf = read_uint16(s);
@@ -1345,6 +1345,10 @@ void jl_deserialize_lambdas_from_mod(ios_t *s)
 
 int jl_deserialize_verify_mod_list(ios_t *s)
 {
+    if (!jl_main_module->uuid) {
+        JL_PRINTF(JL_STDERR, "error: Main module uuid state is invalid for module deserialization.\n");
+        return 0;
+    }
     while (1) {
         size_t len = read_int32(s);
         if (len == 0)
@@ -1355,15 +1359,15 @@ int jl_deserialize_verify_mod_list(ios_t *s)
         uint64_t uuid = read_uint64(s);
         jl_module_t *m = (jl_module_t*)jl_get_global(jl_main_module, jl_symbol(name));
         if (!m) {
-            JL_PRINTF(JL_STDERR, "Module %s must be loaded first\n", name);
+            JL_PRINTF(JL_STDERR, "error: Module %s must be loaded first\n", name);
             return 0;
         }
         if (!jl_is_module(m)) {
             ios_close(s);
             jl_errorf("typeassert: expected %s::Module", name);
         }
         if (m->uuid != uuid) {
-            JL_PRINTF(JL_STDERR, "Module %s uuid did not match cache file\n", name);
+            JL_PRINTF(JL_STDERR, "error: Module %s uuid did not match cache file\n", name);
             return 0;
         }
     }
@@ -1380,6 +1384,7 @@ DLLEXPORT void jl_save_system_image(char *fname)
     int en = jl_gc_is_enabled();
     jl_gc_disable();
     htable_reset(&backref_table, 250000);
+    backref_table_numel = 0;
     ios_t f;
     if (ios_file(&f, fname, 1, 1, 1, 1) == NULL) {
         JL_PRINTF(JL_STDERR, "Cannot open system image file \"%s\" for writing.\n", fname);
@@ -1623,6 +1628,7 @@ int jl_save_new_module(char *fname, jl_module_t *mod)
     jl_serialize_mod_list(&f);
     htable_new(&backref_table, 5000);
     ptrhash_put(&backref_table, jl_main_module, (void*)(uintptr_t)0);
+    backref_table_numel = 1;
 
     int en = jl_gc_is_enabled();
     jl_gc_disable();

diff --git a/src/init.c b/src/init.c
@@ -36,7 +36,6 @@
 
 #include "julia.h"
 #include "julia_internal.h"
-#include <stdio.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -628,7 +627,7 @@ void *mach_segv_listener(void *arg)
     (void)arg;
     while (1) {
         int ret = mach_msg_server(exc_server,2048,segv_port,MACH_MSG_TIMEOUT_NONE);
-        printf("mach_msg_server: %s\n", mach_error_string(ret));
+        ios_printf(ios_stderr, "mach_msg_server: %s\n", mach_error_string(ret));
         jl_exit(1);
     }
 }
@@ -998,8 +997,6 @@ DLLEXPORT void jl_install_sigint_handler()
 extern int asprintf(char **str, const char *fmt, ...);
 extern void *__stack_chk_guard;
 
-void jl_compile_all(void);
-
 DLLEXPORT int julia_trampoline(int argc, char **argv, int (*pmain)(int ac,char *av[]))
 {
 #if defined(_OS_WINDOWS_)
@@ -1015,37 +1012,6 @@ DLLEXPORT int julia_trampoline(int argc, char **argv, int (*pmain)(int ac,char *
     p[0] = 0;
     JL_SET_STACK_BASE;
     int ret = pmain(argc, argv);
-    char *build_path = jl_compileropts.build_path;
-    if (build_path) {
-        if (jl_compileropts.compile_enabled == JL_COMPILEROPT_COMPILE_ALL)
-            jl_compile_all();
-        char *build_ji;
-        if (asprintf(&build_ji, "%s.ji",build_path) > 0) {
-            jl_save_system_image(build_ji);
-            free(build_ji);
-            if (jl_compileropts.dumpbitcode == JL_COMPILEROPT_DUMPBITCODE_ON) {
-                char *build_bc;
-                if (asprintf(&build_bc, "%s.bc",build_path) > 0) {
-                    jl_dump_bitcode(build_bc);
-                    free(build_bc);
-                }
-                else {
-                    ios_printf(ios_stderr,"\nWARNING: failed to create string for .bc build path\n");
-                }
-            }
-            char *build_o;
-            if (asprintf(&build_o, "%s.o",build_path) > 0) {
-                jl_dump_objfile(build_o,0);
-                free(build_o);
-            }
-            else {
-                ios_printf(ios_stderr,"\nFATAL: failed to create string for .o build path\n");
-            }
-        }
-        else {
-            ios_printf(ios_stderr,"\nFATAL: failed to create string for .ji build path\n");
-        }
-    }
     p[sizeof(__stack_chk_guard)-1] = a;
     p[sizeof(__stack_chk_guard)-2] = b;
     p[0] = c;

diff --git a/src/jlapi.c b/src/jlapi.c
@@ -46,10 +46,10 @@ int jl_is_initialized(void) { return jl_main_module!=NULL; }
 
 // First argument is the usr/lib directory where libjulia is, or NULL to guess.
 // if that doesn't work, try the full path to the "lib" directory that
-// contains lib/julia/sys.ji
+// contains lib/julia/Base.ji
 // Second argument is the path of a system image file (*.ji) relative to the
 // first argument path, or relative to the default julia home dir. The default
-// is something like ../lib/julia/sys.ji
+// is something like ../lib/julia/Base.ji
 DLLEXPORT void jl_init_with_image(char *julia_home_dir, char *image_relative_path)
 {
     if (jl_is_initialized()) return;

diff --git a/src/toplevel.c b/src/toplevel.c
@@ -5,6 +5,7 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <stdio.h>
 #include <setjmp.h>
 #include <assert.h>
 #include <sys/types.h>
@@ -83,11 +84,14 @@ void jl_module_load_time_initialize(jl_module_t *m)
     }
 }
 
+void jl_compile_all(void);
+
 extern void jl_get_system_hooks(void);
 jl_value_t *jl_eval_module_expr(jl_expr_t *ex)
 {
     static arraylist_t module_stack;
     static int initialized=0;
+    int isbase = 0;
     if (!initialized) {
         arraylist_new(&module_stack, 0);
         initialized = 1;
@@ -106,12 +110,14 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex)
     jl_binding_t *b = jl_get_binding_wr(parent_module, name);
     jl_declare_constant(b);
     if (b->value != NULL) {
+        jl_main_module->uuid = 0;
         JL_PRINTF(JL_STDERR, "Warning: replacing module %s\n", name->name);
     }
     jl_module_t *newm = jl_new_module(name);
     newm->parent = parent_module;
     b->value = (jl_value_t*)newm;
     if (parent_module == jl_main_module && name == jl_symbol("Base")) {
+        isbase = 1;
         // pick up Base module during bootstrap
         jl_old_base_module = jl_base_module;
         jl_base_module = newm;
@@ -156,6 +162,49 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex)
     jl_current_module = last_module;
     jl_current_task->current_module = task_last_m;
 
+    arraylist_push(&module_stack, newm);
+
+    if (jl_current_module == jl_main_module) {
+        while (module_stack.len > 0) {
+            jl_module_load_time_initialize((jl_module_t *) arraylist_pop(&module_stack));
+        }
+
+        char *build_path = jl_compileropts.build_path;
+        if (build_path) {
+            if (jl_compileropts.compile_enabled == JL_COMPILEROPT_COMPILE_ALL)
+                jl_compile_all();
+            char *build_ji;
+            if (asprintf(&build_ji, "%s" PATHSEPSTRING "%s.ji",build_path,newm->name->name) > 0) {
+                if (isbase)
+                    jl_save_system_image(build_ji);
+                else
+                    jl_save_new_module(build_ji, newm);
+                free(build_ji);
+                if (jl_compileropts.dumpbitcode == JL_COMPILEROPT_DUMPBITCODE_ON) {
+                    char *build_bc;
+                    if (asprintf(&build_bc, "%s" PATHSEPSTRING "%s.bc",build_path,newm->name->name) > 0) {
+                        jl_dump_bitcode(build_bc);
+                        free(build_bc);
+                    }
+                    else {
+                        ios_printf(ios_stderr,"\nWARNING: failed to create string for .bc build path\n");
+                    }
+                }
+                char *build_o;
+                if (asprintf(&build_o, "%s" PATHSEPSTRING "%s.o",build_path,newm->name->name) > 0) {
+                    jl_dump_objfile(build_o,0);
+                    free(build_o);
+                }
+                else {
+                    ios_printf(ios_stderr,"\nFATAL: failed to create string for .o build path\n");
+                }
+            }
+            else {
+                ios_printf(ios_stderr,"\nFATAL: failed to create string for .ji build path\n");
+            }
+        }
+    }
+
 #if 0
     // some optional post-processing steps
     size_t i;
@@ -176,14 +225,6 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex)
     }
 #endif
 
-    arraylist_push(&module_stack, newm);
-
-    if (jl_current_module == jl_main_module) {
-        while (module_stack.len > 0) {
-            jl_module_load_time_initialize((jl_module_t *) arraylist_pop(&module_stack));
-        }
-    }
-
     return jl_nothing;
 }
 

diff --git a/ui/repl.c b/ui/repl.c
@@ -87,7 +87,7 @@ static const char *opts =
     " --check-bounds={yes|no}  Emit bounds checks always or never (ignoring declarations)\n"
     " -O, --optimize           Run time-intensive code optimizations\n"
     " --int-literals={32|64}   Select integer literal size independent of platform\n"
-    " --dump-bitcode={yes|no}  Dump bitcode for the system image (used with --build)\n";
+    " --dump-bitcode={yes|no}  Dump bitcode for the cache image (used with --build)\n";
 
 void parse_opts(int *argcp, char ***argvp)
 {