Unify code loading to parse from string rather than file

Always read code into a String before parsing, to simplify and unify the
parsing entry points.

While doing this, rearrange the runtime functions jl_load_rewrite and
jl_load_rewrite_file_string which implement include() and
include_string():

* Put all these closely related functions go together into toplevel.c
* Pass julia types to these, rather than a combination of Julia types
  and C strings.
* Keep jl_load and jl_load_file_string from julia.h unchanged for
  compatibility with the existing C API. (No julia code `ccall`s
  these anymore)

base/Base.jl

@@ -375,12 +375,8 @@ function include(mapexpr::Function, mod::Module, _path::AbstractString)
     tls[:SOURCE_PATH] = path
     local result
     try
-        # result = Core.include(mod, path)
-        if mapexpr === identity
-            result = ccall(:jl_load, Any, (Any, Cstring), mod, path)
-        else
-            result = ccall(:jl_load_rewrite, Any, (Any, Cstring, Any), mod, path, mapexpr)
-        end
+        result = ccall(:jl_load_rewrite, Any, (Any, Any, Any), mod, path,
+                       mapexpr === identity ? nothing : mapexpr)
     finally
         if prev === nothing
             delete!(tls, :SOURCE_PATH)

base/loading.jl

@@ -1083,13 +1083,8 @@ actually evaluates `mapexpr(expr)`.  If it is omitted, `mapexpr` defaults to [`i
 """
 function include_string(mapexpr::Function, m::Module, txt_::AbstractString, fname::AbstractString="string")
     txt = String(txt_)
-    if mapexpr === identity
-        ccall(:jl_load_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any),
-            txt, sizeof(txt), String(fname), m)
-    else
-        ccall(:jl_load_rewrite_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any, Any),
-            txt, sizeof(txt), String(fname), m, mapexpr)
-    end
+    ccall(:jl_load_rewrite_file_string, Any, (Any, Any, Any, Any),
+          m, txt, String(fname), mapexpr === identity ? nothing : mapexpr)
 end
 
 include_string(m::Module, txt::AbstractString, fname::AbstractString="string") =

src/ast.c

@@ -856,37 +856,32 @@ JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len,
     return jl_parse_all(str, len, filename, filename_len);
 }
 
-// parse and eval a whole file, possibly reading from a string (`content`)
-jl_value_t *jl_parse_eval_all(const char *fname,
-                              const char *content, size_t contentlen,
-                              jl_module_t *inmodule,
-                              jl_value_t *mapexpr)
+// Parse a string `text` at top level, attributing source locations to
+// `filename`. Each expression is optionally modified by `mapexpr` (if
+// non-NULL) before evaluating in module `inmodule`.
+jl_value_t *jl_parse_eval_all(jl_module_t *inmodule, jl_value_t *text,
+                              jl_value_t *filename, jl_value_t *mapexpr)
 {
     jl_ptls_t ptls = jl_get_ptls_states();
     if (ptls->in_pure_callback)
         jl_error("cannot use include inside a generated function");
     jl_ast_context_t *ctx = jl_ast_ctx_enter();
     fl_context_t *fl_ctx = &ctx->fl;
     value_t f, ast, expression;
-    size_t len = strlen(fname);
-    f = cvalue_static_cstrn(fl_ctx, fname, len);
+    f = cvalue_static_cstrn(fl_ctx, jl_string_data(filename), jl_string_len(filename));
     fl_gc_handle(fl_ctx, &f);
-    if (content != NULL) {
+    {
         JL_TIMING(PARSING);
-        value_t t = cvalue_static_cstrn(fl_ctx, content, contentlen);
+        value_t t = cvalue_static_cstrn(fl_ctx, jl_string_data(text),
+                                        jl_string_len(text));
         fl_gc_handle(fl_ctx, &t);
         ast = fl_applyn(fl_ctx, 2, symbol_value(symbol(fl_ctx, "jl-parse-all")), t, f);
         fl_free_gc_handles(fl_ctx, 1);
     }
-    else {
-        JL_TIMING(PARSING);
-        assert(memchr(fname, 0, len) == NULL); // was checked already in jl_load
-        ast = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "jl-parse-file")), f);
-    }
     fl_free_gc_handles(fl_ctx, 1);
     if (ast == fl_ctx->F) {
         jl_ast_ctx_leave(ctx);
-        jl_errorf("could not open file %s", fname);
+        jl_errorf("could not open file %s", jl_string_data(filename));
     }
     fl_gc_handle(fl_ctx, &ast);
     fl_gc_handle(fl_ctx, &expression);
@@ -896,7 +891,7 @@ jl_value_t *jl_parse_eval_all(const char *fname,
     size_t last_age = jl_get_ptls_states()->world_age;
     int lineno = 0;
     jl_lineno = 0;
-    jl_filename = fname;
+    jl_filename = jl_string_data(filename);
     jl_module_t *old_module = ctx->module;
     ctx->module = inmodule;
     jl_value_t *form = NULL;
@@ -926,8 +921,8 @@ jl_value_t *jl_parse_eval_all(const char *fname,
                 expression =
                     fl_applyn(fl_ctx, 4,
                               symbol_value(symbol(fl_ctx, "jl-expand-to-thunk-warn")),
-                              expression, symbol(fl_ctx, fname), fixnum(lineno),
-                              iscons(cdr_(ast)) ? fl_ctx->T : fl_ctx->F);
+                              expression, symbol(fl_ctx, jl_string_data(filename)),
+                              fixnum(lineno), iscons(cdr_(ast)) ? fl_ctx->T : fl_ctx->F);
             }
             jl_get_ptls_states()->world_age = jl_world_counter;
             form = scm_to_julia(fl_ctx, expression, inmodule);
@@ -945,7 +940,7 @@ jl_value_t *jl_parse_eval_all(const char *fname,
         }
     }
     JL_CATCH {
-        form = jl_pchar_to_string(fname, len);
+        form = filename;
         result = jl_box_long(jl_lineno);
         err = 1;
         goto finally; // skip jl_restore_excstack
@@ -968,19 +963,6 @@ jl_value_t *jl_parse_eval_all(const char *fname,
     return result;
 }
 
-JL_DLLEXPORT jl_value_t *jl_load_rewrite_file_string(const char *text, size_t len,
-                                                     char *filename, jl_module_t *inmodule,
-                                                     jl_value_t *mapexpr)
-{
-    return jl_parse_eval_all(filename, text, len, inmodule, mapexpr);
-}
-
-JL_DLLEXPORT jl_value_t *jl_load_file_string(const char *text, size_t len,
-                                             char *filename, jl_module_t *inmodule)
-{
-    return jl_parse_eval_all(filename, text, len, inmodule, NULL);
-}
-
 // returns either an expression or a thunk
 jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module_t *inmodule)
 {

src/julia.h

@@ -1596,12 +1596,12 @@ JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *d
 JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz, jl_array_t *depmods);
 
 // front end interface
+// parsing
 JL_DLLEXPORT jl_value_t *jl_parse_all(const char *str, size_t len,
                                       const char *filename, size_t filename_len);
 JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
                                          int pos0, int greedy);
-JL_DLLEXPORT jl_value_t *jl_load_file_string(const char *text, size_t len,
-                                             char *filename, jl_module_t *inmodule);
+// lowering
 JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule);
 JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmodule,
                                             const char *file, int line);
@@ -1610,7 +1610,6 @@ JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t *
 JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule);
 JL_DLLEXPORT jl_value_t *jl_expand_stmt_with_loc(jl_value_t *expr, jl_module_t *inmodule,
                                                  const char *file, int line);
-JL_DLLEXPORT jl_value_t *jl_eval_string(const char *str); // embedding interface
 // deprecated; use jl_parse_all
 JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len,
                                              const char *filename, size_t filename_len);
@@ -1637,10 +1636,15 @@ JL_DLLEXPORT jl_uv_libhandle jl_dlopen(const char *filename, unsigned flags);
 JL_DLLEXPORT int jl_dlclose(jl_uv_libhandle handle);
 JL_DLLEXPORT int jl_dlsym(jl_uv_libhandle handle, const char *symbol, void ** value, int throw_err);
 
-// compiler
+// evaluation
 JL_DLLEXPORT jl_value_t *jl_toplevel_eval(jl_module_t *m, jl_value_t *v);
 JL_DLLEXPORT jl_value_t *jl_toplevel_eval_in(jl_module_t *m, jl_value_t *ex);
+// code loading (parsing + evaluation)
+JL_DLLEXPORT jl_value_t *jl_eval_string(const char *str); // embedding interface
+JL_DLLEXPORT jl_value_t *jl_load_file_string(const char *text, size_t len,
+                                             char *filename, jl_module_t *module);
 JL_DLLEXPORT jl_value_t *jl_load(jl_module_t *module, const char *fname);
+
 JL_DLLEXPORT jl_module_t *jl_base_relative_to(jl_module_t *m JL_PROPAGATES_ROOT);
 
 // tracing

src/julia_internal.h

@@ -473,10 +473,8 @@ jl_array_t *jl_get_loaded_modules(void);
 jl_value_t *jl_toplevel_eval_flex(jl_module_t *m, jl_value_t *e, int fast, int expanded);
 
 jl_value_t *jl_eval_global_var(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *e);
-jl_value_t *jl_parse_eval_all(const char *fname,
-                              const char *content, size_t contentlen,
-                              jl_module_t *inmodule,
-                              jl_value_t *mapexpr);
+jl_value_t *jl_parse_eval_all(jl_module_t *inmodule, jl_value_t *text,
+                              jl_value_t *filename, jl_value_t *mapexpr);
 jl_value_t *jl_interpret_toplevel_thunk(jl_module_t *m, jl_code_info_t *src);
 jl_value_t *jl_interpret_toplevel_expr_in(jl_module_t *m, jl_value_t *e,
                                           jl_code_info_t *src,

src/support/ios.h

@@ -86,19 +86,19 @@ JL_DLLEXPORT size_t ios_read(ios_t *s, char *dest, size_t n) JL_NOTSAFEPOINT;
 JL_DLLEXPORT size_t ios_readall(ios_t *s, char *dest, size_t n) JL_NOTSAFEPOINT;
 JL_DLLEXPORT size_t ios_write(ios_t *s, const char *data, size_t n) JL_NOTSAFEPOINT;
 JL_DLLEXPORT int64_t ios_seek(ios_t *s, int64_t pos) JL_NOTSAFEPOINT; // absolute seek
-JL_DLLEXPORT int64_t ios_seek_end(ios_t *s);
+JL_DLLEXPORT int64_t ios_seek_end(ios_t *s) JL_NOTSAFEPOINT;
 JL_DLLEXPORT int64_t ios_skip(ios_t *s, int64_t offs);  // relative seek
 JL_DLLEXPORT int64_t ios_pos(ios_t *s) JL_NOTSAFEPOINT;  // get current position
 JL_DLLEXPORT int ios_trunc(ios_t *s, size_t size) JL_NOTSAFEPOINT;
 JL_DLLEXPORT int ios_eof(ios_t *s);
 JL_DLLEXPORT int ios_eof_blocking(ios_t *s);
 JL_DLLEXPORT int ios_flush(ios_t *s);
-JL_DLLEXPORT void ios_close(ios_t *s);
+JL_DLLEXPORT void ios_close(ios_t *s) JL_NOTSAFEPOINT;
 JL_DLLEXPORT int ios_isopen(ios_t *s);
 JL_DLLEXPORT char *ios_take_buffer(ios_t *s, size_t *psize);  // release buffer to caller
 // set buffer space to use
 JL_DLLEXPORT int ios_setbuf(ios_t *s, char *buf, size_t size, int own) JL_NOTSAFEPOINT;
-JL_DLLEXPORT int ios_bufmode(ios_t *s, bufmode_t mode);
+JL_DLLEXPORT int ios_bufmode(ios_t *s, bufmode_t mode) JL_NOTSAFEPOINT;
 JL_DLLEXPORT int ios_get_readable(ios_t *s);
 JL_DLLEXPORT int ios_get_writable(ios_t *s);
 JL_DLLEXPORT void ios_set_readonly(ios_t *s);

src/toplevel.c

@@ -865,25 +865,90 @@ JL_DLLEXPORT jl_value_t *jl_infer_thunk(jl_code_info_t *thk, jl_module_t *m)
     return (jl_value_t*)jl_any_type;
 }
 
-JL_DLLEXPORT jl_value_t *jl_load_rewrite(jl_module_t *module, const char *fname, jl_value_t *mapexpr)
+
+//------------------------------------------------------------------------------
+// Code loading: combined parse+eval for include() and include_string()
+
+// Parse julia code from the string `text`, attributing it to `filename`. Eval
+// the resulting statements into `module` after applying `mapexpr` to each one
+// (if not one of NULL or nothing).
+JL_DLLEXPORT jl_value_t *jl_load_rewrite_file_string(jl_module_t *module,
+                                                     jl_value_t *text,
+                                                     jl_value_t *filename,
+                                                     jl_value_t *mapexpr)
 {
-    uv_stat_t stbuf;
-    if (jl_stat(fname, (char*)&stbuf) != 0 || (stbuf.st_mode & S_IFMT) != S_IFREG) {
-        jl_errorf("could not open file %s", fname);
+    if (!jl_is_string(text) || !jl_is_string(filename)) {
+        jl_errorf("Expected `String`s for `text` and `filename`");
     }
-    return jl_parse_eval_all(fname, NULL, 0, module, mapexpr);
+    return jl_parse_eval_all(module, text, filename,
+                             mapexpr == jl_nothing ? NULL : mapexpr);
+}
+
+// Synchronously read content of entire file into a julia String
+static jl_value_t *jl_file_content_as_string(jl_value_t *filename)
+{
+    const char* fname = jl_string_data(filename);
+    ios_t f;
+    if (ios_file(&f, fname, 1, 0, 0, 0) == NULL)
+        jl_errorf("File \"%s\" not found", fname);
+    ios_bufmode(&f, bm_none);
+    ios_seek_end(&f);
+    size_t len = ios_pos(&f);
+    jl_value_t *text = jl_alloc_string(len);
+    ios_seek(&f, 0);
+    if (ios_readall(&f, jl_string_data(text), len) != len)
+        jl_errorf("Error reading file \"%s\"", fname);
+    ios_close(&f);
+    return text;
 }
 
-JL_DLLEXPORT jl_value_t *jl_load(jl_module_t *module, const char *fname)
+// Load and parse julia code from the file `filename`. Eval the resulting
+// statements into `module` after applying `mapexpr` to each one.
+JL_DLLEXPORT jl_value_t *jl_load_rewrite(jl_module_t *module,
+                                         jl_value_t *filename,
+                                         jl_value_t *mapexpr)
 {
-    return jl_load_rewrite(module, fname, NULL);
+    jl_value_t *text = jl_file_content_as_string(filename);
+    JL_GC_PUSH1(&text);
+    jl_value_t *result = jl_load_rewrite_file_string(module, text, filename, mapexpr);
+    JL_GC_POP();
+    return result;
 }
 
-// load from filename given as a String object
-JL_DLLEXPORT jl_value_t *jl_load_(jl_module_t *module, jl_value_t *str)
+// Code loading - julia.h C API with native C types
+
+// Parse julia code from `filename` and eval into `module`.
+JL_DLLEXPORT jl_value_t *jl_load(jl_module_t *module, const char *filename)
+{
+    jl_value_t *filename_ = NULL;
+    JL_GC_PUSH1(&filename_);
+    filename_ = jl_cstr_to_string(filename);
+    jl_value_t *result = jl_load_rewrite(module, filename_, jl_nothing);
+    JL_GC_POP();
+    return result;
+}
+
+// Parse julia code from the string `text` of length `len`, attributing it to
+// `filename`. Eval the resulting statements into `module`.
+JL_DLLEXPORT jl_value_t *jl_load_file_string(const char *text, size_t len,
+                                             char *filename, jl_module_t *module)
+{
+    jl_value_t *text_ = NULL;
+    jl_value_t *filename_ = NULL;
+    JL_GC_PUSH2(&text_, &filename_);
+    text_ = jl_pchar_to_string(text, len);
+    filename_ = jl_cstr_to_string(filename);
+    jl_value_t *result = jl_load_rewrite_file_string(module, text_, filename_, jl_nothing);
+    JL_GC_POP();
+    return result;
+}
+
+
+//--------------------------------------------------
+// Code loading helpers for bootstrap
+JL_DLLEXPORT jl_value_t *jl_load_(jl_module_t *module, jl_value_t *filename)
 {
-    // assume String has a hidden '\0' at the end
-    return jl_load(module, (const char*)jl_string_data(str));
+    return jl_load_rewrite(module, filename, jl_nothing);
 }
 
 JL_DLLEXPORT jl_value_t *jl_prepend_cwd(jl_value_t *str)

test/loading.jl

@@ -562,7 +562,7 @@ end
 end
 
 # normalization of paths by include (#26424)
-@test_throws ErrorException("could not open file $(joinpath(@__DIR__, "notarealfile.jl"))") include("./notarealfile.jl")
+@test_throws ErrorException("File \"$(joinpath(@__DIR__, "notarealfile.jl"))\" not found") include("./notarealfile.jl")
 
 old_act_proj = Base.ACTIVE_PROJECT[]
 pushfirst!(LOAD_PATH, "@")