cache compile dep tracking

doc/manual/modules.rst

@@ -257,24 +257,33 @@ the statements in a module often involves compiling a large amount of code.
 Julia provides the ability to create precompiled versions of modules
 to reduce this time.
 
-There are two mechanisms that can achieve this:
-incremental compile and custom system image.
-
-To create a custom system image that can be used when starting Julia with the ``-J`` option,
-recompile Julia after modifying the file ``base/userimg.jl`` to require the desired modules.
-
 To create an incremental precompiled module file, add
-``__precompile__()`` at the top of your module file (before the
-``module`` starts).  This will cause it to be automatically compiled
-the first time it is imported.  Alternatively, you can manually call
-``Base.compilecache(modulename)``.  The resulting cache files will be
-stored in ``Base.LOAD_CACHE_PATH[1]``.  Subsequently, the module is
-automatically recompiled upon ``import`` whenever any of its
-dependencies change; dependencies are modules it imports, the Julia
-build, files it includes, or explicit dependencies declared by
-``include_dependency(path)`` in the module file(s).  Precompiling a
-module also recursively precompiles any modules that are imported
-therein.  If you know that it is *not* safe to precompile your module
+``__precompile__()`` at the top of your module file
+(before the ``module`` starts).
+This will cause it to be automatically compiled the first time it is imported.
+Alternatively, you can manually call ``Base.compilecache(modulename)``.
+The resulting cache files will be stored in ``Base.LOAD_CACHE_PATH[1]``.
+Subsequently, the module is automatically recompiled upon ``import``
+whenever any of its dependencies change;
+dependencies are modules it imports, the Julia build, files it includes,
+or explicit dependencies declared by ``include_dependency(path)`` in the module file(s).
+
+For file dependencies, a change is determined by examining whether the modification time (mtime)
+of each file loaded by ``include`` or added explicity by ``include_dependency`` is unchanged,
+or equal to the modification time truncated to the nearest second
+(to accommodate systems that can't copy mtime with sub-second accuracy).
+It also takes into account whether the path to the file chosen by the search logic in ``require``
+matches the path that had created the precompile file.
+
+It also takes into account the set of dependencies already loaded into the current process
+and won't recompile those modules, even if their files change or disappear,
+in order to avoid creating incompatibilities between the running system and the precompile cache.
+If you want to have changes to the source reflected in the running system,
+you should call ``reload("Module")`` on the module you changed,
+and any module that depended on it in which you want to see the change reflected.
+
+Precompiling a module also recursively precompiles any modules that are imported therein.
+If you know that it is *not* safe to precompile your module
 (for the reasons described below), you should put
 ``__precompile__(false)`` in the module file to cause ``Base.compilecache`` to
 throw an error (and thereby prevent the module from being imported by
@@ -290,12 +299,22 @@ initialization steps that must occur at *runtime* from steps that can
 occur at *compile time*.  For this purpose, Julia allows you to define
 an ``__init__()`` function in your module that executes any
 initialization steps that must occur at runtime.
+This function will not be called during compilation
+(``--output-*`` or ``__precompile__()``).
+You may, of course, call it manually if necessary,
+but the default is to assume this function deals with computing state for
+the local machine, which does not need to be -- or even should not be --
+captured in the compiled image.
+It will be called after the module is loaded into a process,
+including if it is being loaded into an incremental compile
+(``--output-incremental=yes``), but not if it is being loaded
+into a full-compilation process.
 
 In particular, if you define a ``function __init__()`` in a module,
 then Julia will call ``__init__()`` immediately *after* the module is
 loaded (e.g., by ``import``, ``using``, or ``require``) at runtime for
 the *first* time (i.e., ``__init__`` is only called once, and only
-after all statements in the module have been executed).  Because it is
+after all statements in the module have been executed). Because it is
 called after the module is fully imported, any submodules or other
 imported modules have their ``__init__`` functions called *before* the
 ``__init__`` of the enclosing module.
@@ -304,22 +323,25 @@ Two typical uses of ``__init__`` are calling runtime initialization
 functions of external C libraries and initializing global constants
 that involve pointers returned by external libraries.  For example,
 suppose that we are calling a C library ``libfoo`` that requires us
-to call a ``foo_init()`` initialization function at runtime.   Suppose
+to call a ``foo_init()`` initialization function at runtime. Suppose
 that we also want to define a global constant ``foo_data_ptr`` that
 holds the return value of a ``void *foo_data()`` function defined by
 ``libfoo`` — this constant must be initialized at runtime (not at compile
 time) because the pointer address will change from run to run.  You
 could accomplish this by defining the following ``__init__`` function
 in your module::
 
+    const foo_data_ptr = Ref{Ptr{Void}}(0)
     function __init__()
-        ccall((:foo_init,:libfoo), Void, ())
-        global const foo_data_ptr = ccall((:foo_data,:libfoo), Ptr{Void}, ())
+        ccall((:foo_init, :libfoo), Void, ())
+        foo_data_ptr[] = ccall((:foo_data, :libfoo), Ptr{Void}, ())
     end
 
 Notice that it is perfectly possible to define a global inside
 a function like ``__init__``; this is one of the advantages of using a
-dynamic language.
+dynamic language. But by making it a constant at global scope,
+we can ensure that the type is known to the compiler and allow it to generate
+better optimized code.
 Obviously, any other globals in your module that depends on ``foo_data_ptr``
 would also have to be initialized in ``__init__``.
 

src/builtins.c

@@ -44,8 +44,8 @@ JL_DLLEXPORT void JL_NORETURN jl_error(const char *str)
 
 extern int vasprintf(char **str, const char *fmt, va_list ap);
 
-static void JL_NORETURN jl_vexceptionf(jl_datatype_t *exception_type,
-                                       const char *fmt, va_list args)
+static jl_value_t *jl_vexceptionf(jl_datatype_t *exception_type,
+                                  const char *fmt, va_list args)
 {
     if (exception_type == NULL) {
         jl_printf(JL_STDERR, "ERROR: ");
@@ -64,24 +64,38 @@ static void JL_NORETURN jl_vexceptionf(jl_datatype_t *exception_type,
         free(str);
     }
     JL_GC_PUSH1(&msg);
-    jl_throw(jl_new_struct(exception_type, msg));
+    jl_value_t *e = jl_new_struct(exception_type, msg);
+    JL_GC_POP();
+    return e;
 }
 
 JL_DLLEXPORT void JL_NORETURN jl_errorf(const char *fmt, ...)
 {
     va_list args;
     va_start(args, fmt);
-    jl_vexceptionf(jl_errorexception_type, fmt, args);
+    jl_value_t *e = jl_vexceptionf(jl_errorexception_type, fmt, args);
     va_end(args);
+    jl_throw(e);
 }
 
 JL_DLLEXPORT void JL_NORETURN jl_exceptionf(jl_datatype_t *exception_type,
                                             const char *fmt, ...)
 {
     va_list args;
     va_start(args, fmt);
-    jl_vexceptionf(exception_type, fmt, args);
+    jl_value_t *e = jl_vexceptionf(exception_type, fmt, args);
+    va_end(args);
+    jl_throw(e);
+}
+
+jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type,
+                              const char *fmt, ...)
+{
+    va_list args;
+    va_start(args, fmt);
+    jl_value_t *e = jl_vexceptionf(exception_type, fmt, args);
     va_end(args);
+    return e;
 }
 
 JL_DLLEXPORT void JL_NORETURN jl_too_few_args(const char *fname, int min)

src/dump.c

@@ -1084,7 +1084,7 @@ static void write_mod_list(ios_t *s)
 }
 
 // "magic" string and version header of .ji file
-static const int JI_FORMAT_VERSION = 2;
+static const int JI_FORMAT_VERSION = 3;
 static const char JI_MAGIC[] = "\373jli\r\n\032\n"; // based on PNG signature
 static const uint16_t BOM = 0xFEFF; // byte-order marker
 static void write_header(ios_t *s)
@@ -1101,6 +1101,22 @@ static void write_header(ios_t *s)
     ios_write(s, commit, strlen(commit)+1);
 }
 
+// serialize information about the result of deserializing this file
+static void write_work_list(ios_t *s)
+{
+    int i, l = jl_array_len(serializer_worklist);
+    for (i = 0; i < l; i++) {
+        jl_module_t *workmod = (jl_module_t*)jl_array_ptr_ref(serializer_worklist, i);
+        if (workmod->parent == jl_main_module) {
+            size_t l = strlen(jl_symbol_name(workmod->name));
+            write_int32(s, l);
+            ios_write(s, jl_symbol_name(workmod->name), l);
+            write_uint64(s, workmod->uuid);
+        }
+    }
+    write_int32(s, 0);
+}
+
 // serialize the global _require_dependencies array of pathnames that
 // are include depenencies
 static void write_dependency_list(ios_t *s)
@@ -1711,16 +1727,16 @@ static void jl_deserialize_lambdas_from_mod(jl_serializer_state *s)
     }
 }
 
-static int read_verify_mod_list(ios_t *s)
+static jl_value_t *read_verify_mod_list(ios_t *s)
 {
     if (!jl_main_module->uuid) {
-        jl_printf(JL_STDERR, "ERROR: Main module uuid state is invalid for module deserialization.\n");
-        return 0;
+        return jl_get_exceptionf(jl_errorexception_type,
+                "Main module uuid state is invalid for module deserialization.");
     }
     while (1) {
         size_t len = read_int32(s);
         if (len == 0)
-            return 1;
+            return NULL;
         char *name = (char*)alloca(len+1);
         ios_read(s, name, len);
         name[len] = '\0';
@@ -1742,20 +1758,17 @@ static int read_verify_mod_list(ios_t *s)
             m = (jl_module_t*)jl_get_global(jl_main_module, sym);
         }
         if (!m) {
-            jl_printf(JL_STDERR, "ERROR: requiring \"%s\" did not define a corresponding module\n", name);
-            return 0;
+            return jl_get_exceptionf(jl_errorexception_type,
+                    "Requiring \"%s\" did not define a corresponding module.", name);
         }
         if (!jl_is_module(m)) {
             ios_close(s);
-            jl_errorf("invalid module path (%s does not name a module)", name);
+            return jl_get_exceptionf(jl_errorexception_type,
+                "Invalid module path (%s does not name a module).", name);
         }
         if (m->uuid != uuid) {
-            jl_printf(JL_STDERR,
-                      "WARNING: Module %s uuid did not match cache file\n"
-                      "  This is likely because module %s does not support\n"
-                      "  precompilation but is imported by a module that does.\n",
-                      name, name);
-            return 0;
+            return jl_get_exceptionf(jl_errorexception_type,
+                "Module %s uuid did not match cache file.", name);
         }
     }
 }
@@ -1825,13 +1838,13 @@ static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list)
                 jl_declare_constant(b); // this can throw
                 if (b->value != NULL) {
                     if (!jl_is_module(b->value)) {
-                        jl_errorf("invalid redefinition of constant %s",
+                        jl_errorf("Invalid redefinition of constant %s.",
                                   jl_symbol_name(mod->name)); // this also throws
                     }
                     if (jl_generating_output() && jl_options.incremental) {
-                        jl_errorf("cannot replace module %s during incremental precompile", jl_symbol_name(mod->name));
+                        jl_errorf("Cannot replace module %s during incremental precompile.", jl_symbol_name(mod->name));
                     }
-                    jl_printf(JL_STDERR, "WARNING: replacing module %s\n",
+                    jl_printf(JL_STDERR, "WARNING: replacing module %s.\n",
                               jl_symbol_name(mod->name));
                 }
                 b->value = v;
@@ -2098,14 +2111,14 @@ JL_DLLEXPORT void jl_restore_system_image(const char *fname)
         int err = jl_load_sysimg_so();
         if (err != 0) {
             if (jl_sysimg_handle == 0)
-                jl_errorf("system image file \"%s\" not found", fname);
-            jl_errorf("library \"%s\" does not contain a valid system image", fname);
+                jl_errorf("System image file \"%s\" not found.", fname);
+            jl_errorf("Library \"%s\" does not contain a valid system image.", fname);
         }
     }
     else {
         ios_t f;
         if (ios_file(&f, fname, 1, 0, 0, 0) == NULL)
-            jl_errorf("system image file \"%s\" not found", fname);
+            jl_errorf("System image file \"%s\" not found.", fname);
         JL_SIGATOMIC_BEGIN();
         jl_restore_system_image_from_stream(&f);
         ios_close(&f);
@@ -2200,8 +2213,10 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist)
     }
     serializer_worklist = worklist;
     write_header(&f);
-    write_mod_list(&f); // this can throw, keep it early (before any actual initialization)
+    write_work_list(&f);
     write_dependency_list(&f);
+    write_mod_list(&f); // this can return errors during deserialize,
+                        // best to keep it early (before any actual initialization)
 
     arraylist_new(&reinit_list, 0);
     htable_new(&backref_table, 5000);
@@ -2363,19 +2378,31 @@ static int trace_method(jl_typemap_entry_t *entry, void *closure)
     return 1;
 }
 
-static jl_array_t *_jl_restore_incremental(ios_t *f)
+static jl_value_t *_jl_restore_incremental(ios_t *f)
 {
-    if (ios_eof(f)) {
+    if (ios_eof(f) || !jl_read_verify_header(f)) {
         ios_close(f);
-        return NULL;
+        return jl_get_exceptionf(jl_errorexception_type,
+                "Precompile file header verification checks failed.");
+    }
+    { // skip past the mod list
+        size_t len;
+        while ((len = read_int32(f)))
+            ios_skip(f, len + sizeof(uint64_t));
+    }
+    { // skip past the dependency list
+        size_t deplen = read_uint64(f);
+        ios_skip(f, deplen);
     }
-    if (!jl_read_verify_header(f) ||
-        !read_verify_mod_list(f)) {
+
+    // verify that the system state is valid
+    jl_value_t *verify_error = read_verify_mod_list(f);
+    if (verify_error) {
         ios_close(f);
-        return NULL;
+        return verify_error;
     }
-    size_t deplen = read_uint64(f);
-    ios_skip(f, deplen); // skip past the dependency list
+
+    // prepare to deserialize
     arraylist_new(&backref_list, 4000);
     arraylist_push(&backref_list, jl_main_module);
     arraylist_new(&flagref_list, 0);
@@ -2418,28 +2445,24 @@ static jl_array_t *_jl_restore_incremental(ios_t *f)
     jl_init_restored_modules(init_order);
     JL_GC_POP();
 
-    return restored;
+    return (jl_value_t*)restored;
 }
 
 JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz)
 {
     ios_t f;
-    jl_array_t *modules;
     ios_static_buffer(&f, (char*)buf, sz);
-    modules = _jl_restore_incremental(&f);
-    return modules ? (jl_value_t*) modules : jl_nothing;
+    return _jl_restore_incremental(&f);
 }
 
 JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname)
 {
     ios_t f;
-    jl_array_t *modules;
     if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) {
-        jl_printf(JL_STDERR, "Cache file \"%s\" not found\n", fname);
-        return jl_nothing;
+        return jl_get_exceptionf(jl_errorexception_type,
+            "Cache file \"%s\" not found.\n", fname);
     }
-    modules = _jl_restore_incremental(&f);
-    return modules ? (jl_value_t*) modules : jl_nothing;
+    return _jl_restore_incremental(&f);
 }
 
 // --- init ---

src/julia_internal.h

@@ -240,6 +240,7 @@ void jl_set_gs_ctr(uint32_t ctr);
 
 void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args);
 void JL_NORETURN jl_method_error(jl_function_t *f, jl_value_t **args, size_t na);
+jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type, const char *fmt, ...);
 
 JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t);
 

src/module.c

@@ -26,7 +26,9 @@ JL_DLLEXPORT jl_module_t *jl_new_module(jl_sym_t *name)
     m->name = name;
     m->parent = NULL;
     m->istopmod = 0;
-    m->uuid = jl_hrtime();
+    static unsigned int mcounter; // simple counter backup, in case hrtime is not incrementing
+    m->uuid = jl_hrtime() + (++mcounter);
+    if (!m->uuid) m->uuid++; // uuid 0 is invalid
     m->counter = 0;
     htable_new(&m->bindings, 0);
     arraylist_new(&m->usings, 0);