Merge pull request #28428 from JuliaLang/aa/backports-0.7.0-rc3

Backports for v0.7.0-rc3

base/compiler/ssair/passes.jl

@@ -688,6 +688,9 @@ function getfield_elim_pass!(ir::IRCode, domtree::DomTree)
         compact[idx] = val === nothing ? nothing : val.x
     end
 
+    # Copy the use count, `finish` may modify it and for our predicate
+    # below we need it consistent with the state of the IR here.
+    used_ssas = copy(compact.used_ssas)
     ir = finish(compact)
     # Now go through any mutable structs and see which ones we can eliminate
     for (idx, (intermediaries, defuse)) in defuses
@@ -699,9 +702,9 @@ function getfield_elim_pass!(ir::IRCode, domtree::DomTree)
         nleaves = length(defuse.uses) + length(defuse.defs) + length(defuse.ccall_preserve_uses)
         nuses = 0
         for idx in intermediaries
-            nuses += compact.used_ssas[idx]
+            nuses += used_ssas[idx]
         end
-        nuses_total = compact.used_ssas[idx] + nuses - length(intermediaries)
+        nuses_total = used_ssas[idx] + nuses - length(intermediaries)
         nleaves == nuses_total || continue
         # Find the type for this allocation
         defexpr = ir[SSAValue(idx)]
@@ -717,7 +720,13 @@ function getfield_elim_pass!(ir::IRCode, domtree::DomTree)
         fielddefuse = SSADefUse[SSADefUse() for _ = 1:fieldcount(typ)]
         ok = true
         for use in defuse.uses
-            field = try_compute_fieldidx_expr(typ, ir[SSAValue(use)])
+            stmt = ir[SSAValue(use)]
+            # We may have discovered above that this use is dead
+            # after the getfield elim of immutables. In that case,
+            # it would have been deleted. That's fine, just ignore
+            # the use in that case.
+            stmt === nothing && continue
+            field = try_compute_fieldidx_expr(typ, stmt)
             field === nothing && (ok = false; break)
             push!(fielddefuse[field].uses, use)
         end

base/loading.jl

@@ -611,7 +611,8 @@ end
 # or an Exception that describes why it couldn't be loaded
 # and it reconnects the Base.Docs.META
 function _include_from_serialized(path::String, depmods::Vector{Any})
-    restored = ccall(:jl_restore_incremental, Any, (Cstring, Any), path, depmods)
+    sv = ccall(:jl_restore_incremental, Any, (Cstring, Any), path, depmods)
+    restored = sv[1]
     if !isa(restored, Exception)
         for M in restored::Vector{Any}
             M = M::Module
@@ -623,6 +624,7 @@ function _include_from_serialized(path::String, depmods::Vector{Any})
             end
         end
     end
+    isassigned(sv, 2) && ccall(:jl_init_restored_modules, Cvoid, (Any,), sv[2])
     return restored
 end
 

base/options.jl

@@ -26,7 +26,7 @@ struct JLOptions
     warn_overwrite::Int8
     can_inline::Int8
     polly::Int8
-    trace_compile::Int8
+    trace_compile::Ptr{UInt8}
     fast_math::Int8
     worker::Int8
     cookie::Ptr{UInt8}

contrib/generate_precompile.jl

@@ -35,19 +35,12 @@ cd("complet_path\t\t$CTRL_C
 julia_cmd() = (julia = joinpath(Sys.BINDIR, Base.julia_exename()); `$julia`)
 have_repl =  haskey(Base.loaded_modules,
                     Base.PkgId(Base.UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL"))
-have_pkg =  haskey(Base.loaded_modules,
-                    Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg"))
-
-if have_pkg
-    precompile_script *= """
-    tmp = mktempdir()
-    cd(tmp)
-    touch("Project.toml")
-    ] activate .
-    st
-    $CTRL_C
-    rm(tmp; recursive=true)
-    """
+Pkg = get(Base.loaded_modules,
+          Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg"),
+          nothing)
+
+if Pkg !== nothing
+    precompile_script *= Pkg.precompile_script
 end
 
 function generate_precompile_statements()
@@ -82,65 +75,63 @@ function generate_precompile_statements()
         print("Generating precompile statements...")
         sysimg = isempty(ARGS) ? joinpath(dirname(Sys.BINDIR), "lib", "julia", "sys.ji") : ARGS[1]
 
-        # Run a repl process and replay our script
-        stdout_accumulator, stderr_accumulator = IOBuffer(), IOBuffer()
-        with_fake_pty() do slave, master
-            with_fake_pty() do slave_err, master_err
-                done = false
-                withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing,
-                        "TERM" => "") do
-                    p = run(`$(julia_cmd()) -O0 --trace-compile=yes --sysimage $sysimg
-                                           --startup-file=no --color=yes`,
-                            slave, slave, slave_err; wait=false)
-                    readuntil(master, "julia>", keep=true)
-                    for (tty, accumulator) in (master     => stdout_accumulator,
-                                               master_err => stderr_accumulator)
-                        @async begin
-                            while true
-                                done && break
-                                write(accumulator, readavailable(tty))
+        mktemp() do precompile_file, _
+            # Run a repl process and replay our script
+            stdout_accumulator, stderr_accumulator = IOBuffer(), IOBuffer()
+            with_fake_pty() do slave, master
+                with_fake_pty() do slave_err, master_err
+                    done = false
+                    withenv("JULIA_HISTORY" => tempname(), "JULIA_PROJECT" => nothing,
+                            "TERM" => "") do
+                        p = run(`$(julia_cmd()) -O0 --trace-compile=$precompile_file --sysimage $sysimg
+                                               --startup-file=no --color=yes`,
+                                slave, slave, slave_err; wait=false)
+                        readuntil(master, "julia>", keep=true)
+                        for (tty, accumulator) in (master     => stdout_accumulator,
+                                                   master_err => stderr_accumulator)
+                            @async begin
+                                while true
+                                    done && break
+                                    write(accumulator, readavailable(tty))
+                                end
                             end
                         end
-                    end
-                    if have_repl
-                        for l in split(precompile_script, '\n'; keepempty=false)
-                            write(master, l, '\n')
+                        if have_repl
+                            for l in split(precompile_script, '\n'; keepempty=false)
+                                write(master, l, '\n')
+                            end
                         end
+                        write(master, "exit()\n")
+                        wait(p)
+                        done = true
                     end
-                    write(master, "exit()\n")
-                    wait(p)
-                    done = true
                 end
             end
-        end
 
-        stderr_output = String(take!(stderr_accumulator))
-        # println(stderr_output)
-        # stdout_output = String(take!(stdout_accumulator))
-        # println(stdout_output)
-
-        # Extract the precompile statements from stderr
-        statements = Set{String}()
-        for statement in split(stderr_output, '\n')
-            m = match(r"(precompile\(Tuple{.*)", statement)
-            m === nothing && continue
-            statement = m.captures[1]
-            occursin(r"Main.", statement) && continue
-            push!(statements, statement)
-        end
+            # Check what the REPL displayed
+            # stdout_output = String(take!(stdout_accumulator))
+            # println(stdout_output)
 
-        # Load the precompile statements
-        statements_ordered = join(sort(collect(statements)), '\n')
-        # println(statements_ordered)
-        if have_repl
-            # Seems like a reasonable number right now, adjust as needed
-            @assert length(statements) > 700
-        end
+            # Extract the precompile statements from stderr
+            statements = Set{String}()
+            for statement in split(read(precompile_file, String), '\n')
+                occursin("Main.", statement) && continue
+                push!(statements, statement)
+            end
 
-        Base.include_string(PrecompileStagingArea, statements_ordered)
-        print(" $(length(statements)) generated in ")
-        Base.time_print((time() - start_time) * 10^9)
-        println()
+            # Load the precompile statements
+            statements_ordered = join(sort(collect(statements)), '\n')
+            # println(statements_ordered)
+            if have_repl
+                # Seems like a reasonable number right now, adjust as needed
+                @assert length(statements) > 700
+            end
+
+            Base.include_string(PrecompileStagingArea, statements_ordered)
+            print(" $(length(statements)) generated in ")
+            Base.time_print((time() - start_time) * 10^9)
+            println()
+        end
     end
 
     # Fall back to explicit list on Windows, might as well include them

contrib/precompile_explicit.jl

@@ -3,7 +3,7 @@
 # Steps to regenerate this file:
 # 1. Remove all `precompile` calls
 # 2. Rebuild system image
-# 3. Start julia with `--trace-compile=yes and do some stuff
+# 3. Start julia with `--trace-compile=precompiles.txt and do some stuff
 # 5. Run `grep -v '#[0-9]' precompiles.txt >> contrib/precompile_explicit.jl`
 # (filters out closures, which might have different generated names in different environments)
 # This list is only used on Windows, otherwise precompile statements are generated dynamically.

deps/blas.mk

@@ -62,6 +62,15 @@ OPENBLAS_FFLAGS += -mincoming-stack-boundary=2
 endif
 endif
 
+# Work around invalid register errors on 64-bit Windows
+# See discussion in https://github.com/xianyi/OpenBLAS/issues/1708
+# TODO: Remove this once we use a version of OpenBLAS where this is set automatically
+ifeq ($(OS),WINNT)
+ifeq ($(ARCH),x86_64)
+OPENBLAS_CFLAGS += -fno-asynchronous-unwind-tables
+endif
+endif
+
 OPENBLAS_BUILD_OPTS += CFLAGS="$(CFLAGS) $(OPENBLAS_CFLAGS)"
 OPENBLAS_BUILD_OPTS += FFLAGS="$(FFLAGS) $(OPENBLAS_FFLAGS)"
 OPENBLAS_BUILD_OPTS += LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN)"
@@ -73,9 +82,11 @@ endif
 
 # Allow disabling AVX for older binutils
 ifeq ($(OPENBLAS_NO_AVX), 1)
-OPENBLAS_BUILD_OPTS += NO_AVX=1 NO_AVX2=1
+OPENBLAS_BUILD_OPTS += NO_AVX=1 NO_AVX2=1 NO_AVX512=1
 else ifeq ($(OPENBLAS_NO_AVX2), 1)
-OPENBLAS_BUILD_OPTS += NO_AVX2=1
+OPENBLAS_BUILD_OPTS += NO_AVX2=1 NO_AVX512=1
+else ifeq ($(OPENBLAS_NO_AVX512), 1)
+OPENBLAS_BUILD_OPTS += NO_AVX512=1
 endif
 
 # Do not overwrite the "-j" flag

deps/llvm.mk

@@ -506,6 +506,7 @@ $(eval $(call LLVM_PATCH,llvm-D49832-SCEVPred)) # Remove for 7.0
 $(eval $(call LLVM_PATCH,llvm-rL323946-LSRTy)) # Remove for 7.0
 $(eval $(call LLVM_PATCH,llvm-D50010-VNCoercion-ni))
 $(eval $(call LLVM_PATCH,llvm-D50167-scev-umin))
+$(eval $(call LLVM_PATCH,llvm-windows-race))
 endif # LLVM_VER
 
 # Remove hardcoded OS X requirements in compilter-rt cmake build

deps/patches/llvm-D50167-scev-umin.patch

@@ -621,10 +621,46 @@ index bfff7afb5b4..750c1fdfdfb 100644
      bool Proper = true;
      for (const SCEV *NAryOp : NAry->operands()) {
 diff --git a/lib/Analysis/ScalarEvolutionExpander.cpp b/lib/Analysis/ScalarEvolutionExpander.cpp
-index 53ce33bacbe..3179de31b92 100644
+index 01a8732b0b8..8160a1eaa0b 100644
 --- a/lib/Analysis/ScalarEvolutionExpander.cpp
 +++ b/lib/Analysis/ScalarEvolutionExpander.cpp
-@@ -1671,6 +1671,54 @@ Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) {
+@@ -1634,14 +1634,15 @@ Value *SCEVExpander::visitSMaxExpr(const SCEVSMaxExpr *S) {
+   for (int i = S->getNumOperands()-2; i >= 0; --i) {
+     // In the case of mixed integer and pointer types, do the
+     // rest of the comparisons as integer.
+-    if (S->getOperand(i)->getType() != Ty) {
++    Type *OpTy = S->getOperand(i)->getType();
++    if (OpTy->isIntegerTy() != Ty->isIntegerTy()) {
+       Ty = SE.getEffectiveSCEVType(Ty);
+       LHS = InsertNoopCastOfTo(LHS, Ty);
+     }
+     Value *RHS = expandCodeFor(S->getOperand(i), Ty);
+     Value *ICmp = Builder.CreateICmpSGT(LHS, RHS);
+     rememberInstruction(ICmp);
+-    Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smax");
++    Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smin");
+     rememberInstruction(Sel);
+     LHS = Sel;
+   }
+@@ -1658,14 +1659,15 @@ Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) {
+   for (int i = S->getNumOperands()-2; i >= 0; --i) {
+     // In the case of mixed integer and pointer types, do the
+     // rest of the comparisons as integer.
+-    if (S->getOperand(i)->getType() != Ty) {
++    Type *OpTy = S->getOperand(i)->getType();
++    if (OpTy->isIntegerTy() != Ty->isIntegerTy()) {
+       Ty = SE.getEffectiveSCEVType(Ty);
+       LHS = InsertNoopCastOfTo(LHS, Ty);
+     }
+     Value *RHS = expandCodeFor(S->getOperand(i), Ty);
+     Value *ICmp = Builder.CreateICmpUGT(LHS, RHS);
+     rememberInstruction(ICmp);
+-    Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umax");
++    Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umin");
+     rememberInstruction(Sel);
+     LHS = Sel;
+   }
+@@ -1671,6 +1671,56 @@ Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) {
    return LHS;
  }
 
@@ -634,7 +670,8 @@ index 53ce33bacbe..3179de31b92 100644
 +  for (int i = S->getNumOperands()-2; i >= 0; --i) {
 +    // In the case of mixed integer and pointer types, do the
 +    // rest of the comparisons as integer.
-+    if (S->getOperand(i)->getType() != Ty) {
++    Type *OpTy = S->getOperand(i)->getType();
++    if (OpTy->isIntegerTy() != Ty->isIntegerTy()) {
 +      Ty = SE.getEffectiveSCEVType(Ty);
 +      LHS = InsertNoopCastOfTo(LHS, Ty);
 +    }
@@ -658,7 +695,8 @@ index 53ce33bacbe..3179de31b92 100644
 +  for (int i = S->getNumOperands()-2; i >= 0; --i) {
 +    // In the case of mixed integer and pointer types, do the
 +    // rest of the comparisons as integer.
-+    if (S->getOperand(i)->getType() != Ty) {
++    Type *OpTy = S->getOperand(i)->getType();
++    if (OpTy->isIntegerTy() != Ty->isIntegerTy()) {
 +      Ty = SE.getEffectiveSCEVType(Ty);
 +      LHS = InsertNoopCastOfTo(LHS, Ty);
 +    }

deps/patches/llvm-windows-race.patch

@@ -0,0 +1,25 @@
+commit c0be37352f6d91505d964a7d33ad79b40750d370
+Author: Keno Fischer <keno@juliacomputing.com>
+Date:   Fri Aug 3 21:03:32 2018 -0400
+
+    [CrossCompile] Add a dependency llvm-config-host -> llvm-tblgen-host
+
+    These use the same working directory and without a dependency
+    edge between them race to build some of the utility libraries
+    causing build failures.
+
+diff --git a/tools/llvm-config/CMakeLists.txt b/tools/llvm-config/CMakeLists.txt
+index f59402ac4b0..f3b4e60fd99 100644
+--- a/tools/llvm-config/CMakeLists.txt
++++ b/tools/llvm-config/CMakeLists.txt
+@@ -77,5 +77,10 @@ if(CMAKE_CROSSCOMPILING AND NOT LLVM_CONFIG_PATH)
+   add_custom_target(${project}NativeLLVMConfig DEPENDS ${${project}_LLVM_CONFIG_EXE})
+   add_dependencies(${project}NativeLLVMConfig CONFIGURE_LLVM_NATIVE)
+
++  # Add a dependency on the host tblgen, which uses the same working
++  # directory and with which we're otherwise racing to build some
++  # of the utility libraries.
++  add_dependencies(${project}NativeLLVMConfig LLVM-tablegen-host)
++
+   add_dependencies(llvm-config ${project}NativeLLVMConfig)
+ endif(CMAKE_CROSSCOMPILING)

src/dump.c

@@ -2402,7 +2402,7 @@ static jl_array_t *jl_finalize_deserializer(jl_serializer_state *s, arraylist_t
     return init_order;
 }
 
-static void jl_init_restored_modules(jl_array_t *init_order)
+JL_DLLEXPORT void jl_init_restored_modules(jl_array_t *init_order)
 {
     if (!init_order)
         return;
@@ -3118,10 +3118,10 @@ static jl_value_t *_jl_restore_incremental(ios_t *f, jl_array_t *mod_array)
         arraylist_free(tracee_list);
         free(tracee_list);
     }
-    jl_init_restored_modules(init_order);
+    jl_value_t *ret = (jl_value_t*)jl_svec(2, restored, init_order);
     JL_GC_POP();
 
-    return (jl_value_t*)restored;
+    return (jl_value_t*)ret;
 }
 
 JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(const char *buf, size_t sz, jl_array_t *mod_array)