add a precompile command that can be used to precompile all dependenc…

…ies (#254)

stdlib/Pkg3/docs/src/index.md

@@ -322,6 +322,7 @@ Downloaded MacroTools ─ v0.4.0
 The dependencies of the unregistered package (here `MacroTools`) got installed.
 For unregistered packages we could have given a branch (or commit SHA) to track using `#`, just like for registered packages.
 
+
 ## Developing packages
 
 Let’s say we found a bug in one of our dependencies, e.g. `JSON` that we want to fix. We can get the full git-repo using the `develop` command
@@ -352,7 +353,7 @@ It is also possible to give a local path as the argument to `develop` which will
 
 Overriding the dependency path for a non registered package is done by giving the git-repo url as an argument to `develop`.
 
-### Updating dependencies
+## Updating dependencies
 
 When new versions of packages the project is using  are released, it is a good idea to update. Simply calling `up` will try to update *all* the dependencies of the project. Sometimes this is not what you want. You can specify a subset of the dependencies to upgrade by giving them as arguments to `up`, e.g:
 
@@ -375,7 +376,17 @@ If you just want install the packages that are given by the current `Manifest.to
 (HelloWorld) pkg> up --manifest --fixed
 ```
 
-### Preview mode
+## Precompiling the project
+
+The REPL command `precompile` can be used to precompile all the dependencies in the project. You can for example do
+
+```
+(HelloWorld) pkg> update; precompile
+```
+
+do update the dependencies and then precompile them.
+
+## Preview mode
 
 If you just want to see the effects of running a command, but not change your state you can `preview` a command.
 For example:
@@ -393,7 +404,7 @@ or
 will show you the effects adding `Plots`, or doing a full upgrade, respectively, would have on your project.
 However, nothing would be installed and your `Project.toml` and `Manfiest.toml` are untouched.
 
-### Using someone elses project
+## Using someone elses project
 
 Simple clone their project using e.g. `git clone`, `cd` to the project directory and call
 
@@ -402,3 +413,5 @@ Simple clone their project using e.g. `git clone`, `cd` to the project directory
 ```
 
 This will install the packages at the same state that the project you cloned was using.
+
+

stdlib/Pkg3/src/API.jl

@@ -65,7 +65,7 @@ function update_registry(ctx)
     # Update the registry
     errors = Tuple{String, String}[]
     if ctx.preview
-        info("Skipping updating registry in preview mode")
+        @info("Skipping updating registry in preview mode")
     else
         for reg in registries()
             if isdir(joinpath(reg, ".git"))
@@ -399,4 +399,60 @@ function dir(pkg::String, paths::String...)
 end
 
 
+function precompile(ctx::Context)
+    printpkgstyle(ctx, :Precompiling, "project...")
+
+    pkgids = [Base.PkgId(UUID(uuid), name) for (name, uuid) in ctx.env.project["deps"] if !(UUID(uuid) in  keys(ctx.stdlibs))]
+    if ctx.env.pkg !== nothing && isfile( joinpath( dirname(ctx.env.project_file), "src", ctx.env.pkg.name * ".jl"))
+        push!(pkgids, Base.PkgId(ctx.env.pkg.uuid, ctx.env.pkg.name))
+    end
+
+    needs_to_be_precompiled = String[]
+    for pkg in pkgids
+        paths = Base.find_all_in_cache_path(pkg)
+        sourcepath = Base.locate_package(pkg)
+        if sourcepath == nothing
+            cmderror("couldn't find path to $(pkg.name) when trying to precompilie project")
+        end
+        found_matching_precompile = false
+        for path_to_try in paths::Vector{String}
+            staledeps = Base.stale_cachefile(sourcepath, path_to_try)
+            if !(staledeps isa Bool)
+                found_matching_precompile = true
+            end
+        end
+        if !found_matching_precompile
+            # Only precompile packages that has contains `__precompile__` or `__precompile__(true)`
+            source = read(sourcepath, String)
+            if occursin(r"__precompile__\(\)|__precompile__\(true\)", source)
+                push!(needs_to_be_precompiled, pkg.name)
+            end
+        end
+    end
+
+    # Perhaps running all the imports in the same process would avoid some overheda.
+    # Julia starts pretty fast though (0.3 seconds)
+    code = join(["import " * pkg for pkg in needs_to_be_precompiled], '\n') * "\nexit(0)"
+    for (i, pkg) in enumerate(needs_to_be_precompiled)
+        code = """
+            empty!(Base.DEPOT_PATH)
+            append!(Base.DEPOT_PATH, $(repr(map(abspath, DEPOT_PATH))))
+            empty!(Base.DL_LOAD_PATH)
+            append!(Base.DL_LOAD_PATH, $(repr(map(abspath, Base.DL_LOAD_PATH))))
+            empty!(Base.LOAD_PATH)
+            append!(Base.LOAD_PATH, $(repr(Base.LOAD_PATH)))
+            import $pkg
+        """
+        printpkgstyle(ctx, :Precompiling, pkg, " [$i of $(length(needs_to_be_precompiled))]")
+        run(pipeline(ignorestatus(```
+        $(Base.julia_cmd()) -O$(Base.JLOptions().opt_level) --color=no --history-file=no
+        --startup-file=$(Base.JLOptions().startupfile != 2 ? "yes" : "no")
+        --compiled-modules="yes"
+        --depwarn=no
+        --eval $code
+        ```)))
+    end
+    return nothing
+end
+
 end # module

stdlib/Pkg3/src/REPLMode.jl

@@ -14,7 +14,7 @@ using ..Types, ..Display, ..Operations
 ############
 @enum(CommandKind, CMD_HELP, CMD_STATUS, CMD_SEARCH, CMD_ADD, CMD_RM, CMD_UP,
                    CMD_TEST, CMD_GC, CMD_PREVIEW, CMD_INIT, CMD_BUILD, CMD_FREE,
-                   CMD_PIN, CMD_CHECKOUT, CMD_DEVELOP, CMD_GENERATE)
+                   CMD_PIN, CMD_CHECKOUT, CMD_DEVELOP, CMD_GENERATE, CMD_PRECOMPILE)
 
 struct Command
     kind::CommandKind
@@ -52,6 +52,7 @@ const cmds = Dict(
     "develop"   => CMD_DEVELOP,
     "dev"       => CMD_DEVELOP,
     "generate"  => CMD_GENERATE,
+    "precompile" => CMD_PRECOMPILE,
 )
 
 #################
@@ -163,14 +164,22 @@ end
 ################
 # REPL parsing #
 ################
-const lex_re = r"^[\?\./\+\-](?!\-) | [^@\#\s]+\s*=\s*[^@\#\s]+ | \#\s*[^@\#\s]* | @\s*[^@\#\s]* | [^@\#\s]+"x
+const lex_re = r"^[\?\./\+\-](?!\-) | [^@\#\s;]+\s*=\s*[^@\#\s;]+ | \#\s*[^@\#\s;]* | @\s*[^@\#\s;]* | [^@\#\s;]+|;"x
 
 const Token = Union{Command, Option, VersionRange, String, Rev}
 
-function tokenize(cmd::String)::Vector{Token}
+function tokenize(cmd::String)::Vector{Vector{Token}}
+    words = map(m->m.match, eachmatch(lex_re, cmd))
+    commands = Vector{Token}[]
+    while !isempty(words)
+        push!(commands, tokenize!(words))
+    end
+    return commands
+end
+
+function tokenize!(words::Vector{<:AbstractString})::Vector{Token}
     print_first_command_header()
     tokens = Token[]
-    words = map(m->m.match, eachmatch(lex_re, cmd))
     help_mode = false
     preview_mode = false
     # First parse a Command or a modifier (help / preview) + Command
@@ -196,7 +205,9 @@ function tokenize(cmd::String)::Vector{Token}
     # Now parse the arguments / options to the command
     while !isempty(words)
         word = popfirst!(words)
-        if first(word) == '-'
+        if word == ";"
+            return tokens
+        elseif first(word) == '-'
             push!(tokens, parse_option(word))
         elseif first(word) == '@'
             push!(tokens, VersionRange(strip(word[2:end])))
@@ -209,14 +220,17 @@ function tokenize(cmd::String)::Vector{Token}
     return tokens
 end
 
+
 #############
 # Execution #
 #############
 
 function do_cmd(repl::REPL.AbstractREPL, input::String; do_rethrow=false)
     try
-        tokens = tokenize(input)
-        do_cmd!(tokens, repl)
+        commands = tokenize(input)
+        for command in commands
+            do_cmd!(command, repl)
+        end
     catch err
         if do_rethrow
             rethrow(err)
@@ -269,6 +283,7 @@ function do_cmd!(tokens::Vector{Token}, repl)
     cmd.kind == CMD_PIN      ? Base.invokelatest(           do_pin!, ctx, tokens) :
     cmd.kind == CMD_FREE     ? Base.invokelatest(          do_free!, ctx, tokens) :
     cmd.kind == CMD_GENERATE ? Base.invokelatest(      do_generate!, ctx, tokens) :
+    cmd.kind == CMD_PRECOMPILE ? Base.invokelatest(  do_precompile!, ctx, tokens) :
         cmderror("`$cmd` command not yet implemented")
     return
 end
@@ -283,6 +298,8 @@ backspace when the input line is empty or press Ctrl+C.
 
     pkg> [--env=...] cmd [opts] [args]
 
+Multiple commands can be given on the same line by interleaving a `;` between the commands.
+
 **Environment**
 
 The `--env` meta option determines which project environment to manipulate. By
@@ -323,6 +340,8 @@ What action you want the package manager to take:
 `develop`: clone the full package repo locally for development
 
 `free`: undos a `pin` or `develop`
+
+`precompile`: precompile all the project dependencies
 """
 
 const helps = Dict(
@@ -466,6 +485,10 @@ const helps = Dict(
     pkg> develop Example#c37b675
     pkg> develop https://github.com/JuliaLang/Example.jl#master
     ```
+    """, CMD_PRECOMPILE => md"""
+        precompile
+
+    Precompile all the dependencies of the project by running `import` on all of them in a new process.
     """
 )
 
@@ -712,6 +735,13 @@ function do_generate!(ctx::Context, tokens::Vector{Token})
     API.generate(pkg)
 end
 
+function do_precompile!(ctx::Context, tokens::Vector{Token})
+    if !isempty(tokens)
+        cmderror("`precompile` does not take any arguments")
+    end
+    API.precompile(ctx)
+end
+
 
 ######################
 # REPL mode creation #
@@ -817,7 +847,7 @@ function completions(full, index)
 
         # tokenize input, don't offer any completions for invalid commands
         tokens = try
-            tokenize(join(pre_words[1:end-1], ' '))
+            tokenize(join(pre_words[1:end-1], ' '))[end]
         catch
             return String[], 0:-1, false
         end

stdlib/Pkg3/test/repl.jl

@@ -56,7 +56,7 @@ temp_pkg_dir() do project_path; cd(project_path) do; mktempdir() do tmp_pkg_path
     @test Pkg3.installed()[TEST_PKG.name] > v
     pkg = "UnregisteredWithoutProject"
     p = git_init_package(tmp_pkg_path, joinpath(@__DIR__, "test_packages/$pkg"))
-    Pkg3.REPLMode.pkgstr("add $p")
+    Pkg3.REPLMode.pkgstr("add $p; precompile")
     @eval import $(Symbol(pkg))
     @test Pkg3.installed()[pkg] == v"0.0"
     Pkg3.test("UnregisteredWithoutProject")
@@ -138,7 +138,7 @@ temp_pkg_dir() do project_path; cd(project_path) do
                     cp(p2_path, p2_new_path)
                     Pkg3.REPLMode.pkgstr("develop $(p1_new_path)")
                     Pkg3.REPLMode.pkgstr("develop $(p2_new_path)")
-                    Pkg3.REPLMode.pkgstr("build")
+                    Pkg3.REPLMode.pkgstr("build; precompile")
                     @test locate_name("UnregisteredWithProject") == joinpath(p1_new_path, "src", "UnregisteredWithProject.jl")
                     @test locate_name("UnregisteredWithoutProject") == joinpath(p2_new_path, "src", "UnregisteredWithoutProject.jl")
                     @test Pkg3.installed()["UnregisteredWithProject"] == v"0.1.0"

stdlib/Pkg3/test/test_packages/UnregisteredWithoutProject/src/UnregisteredWithoutProject.jl

@@ -1,3 +1,4 @@
+__precompile__()
 module UnregisteredWithoutProject
 
 if !isfile(joinpath(@__DIR__, "..", "deps", "deps.jl"))