Rebase from master

.travis.yml

@@ -81,7 +81,7 @@ before_install:
         brew install -v staticfloat/juliadeps/libgfortran;
         brew install -v --only-dependencies --HEAD julia;
         BUILDOPTS="-j3 USECLANG=1 LLVM_CONFIG=$(brew --prefix llvm37-julia)/bin/llvm-config-3.7.1 LLVM_SIZE=$(brew --prefix llvm37-julia)/bin/llvm-size-3.7.1";
-        BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1 STAGE2_DEPS=utf8proc";
+        BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1";
         BUILDOPTS="$BUILDOPTS LIBBLAS=-lopenblas LIBBLASNAME=libopenblas LIBLAPACK=-lopenblas LIBLAPACKNAME=libopenblas";
         for lib in LLVM SUITESPARSE ARPACK BLAS FFTW LAPACK GMP MPFR PCRE LIBUNWIND; do
             export BUILDOPTS="$BUILDOPTS USE_SYSTEM_$lib=1";
@@ -111,6 +111,10 @@ script:
     - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 && cd /tmp/julia/share/julia/test &&
         /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN &&
         /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online pkg
-    - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia && rm -rf julia/deps/build/julia-env
+    - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia &&
+        rm -rf julia/deps/build/julia-env &&
+        rm -rf julia/deps/build/libssh2-*/CMakeFiles/Makefile2 &&
+        rm -rf julia/deps/build/curl-*/config.log &&
+        rm -rf julia/deps/build/curl-*/libtool
 # uncomment the following if failures are suspected to be due to the out-of-memory killer
 #    - dmesg

Make.inc

@@ -445,7 +445,9 @@ export CCACHE_CPP2 := yes
 endif
 else #USECCACHE
 CC_BASE := $(shell echo $(CC) | cut -d' ' -f1)
+CC_ARG := $(shell echo $(CC) | cut -d' ' -f2-)
 CXX_BASE := $(shell echo $(CXX) | cut -d' ' -f1)
+CXX_ARG := $(shell echo $(CXX) | cut -d' ' -f2-)
 endif
 
 JFFLAGS := -O2 $(fPIC)
@@ -683,10 +685,12 @@ endif
 
 ifeq ($(OS),WINNT)
 ifneq ($(ARCH),x86_64)
+ifneq ($(USECLANG),1)
 JCFLAGS += -mincoming-stack-boundary=2
 JCXXFLAGS += -mincoming-stack-boundary=2
 endif
 endif
+endif
 
 ifeq ($(USEGCC),1)
 ifeq ($(ISX86),1)

base/inference.jl

@@ -2261,12 +2261,15 @@ function effect_free(e::ANY, linfo::LambdaInfo, allow_volatile::Bool)
                 if !allow_volatile
                     if is_known_call(e, arrayref, linfo) || is_known_call(e, arraylen, linfo)
                         return false
-                    elseif is_known_call(e, getfield, linfo) && !isa(exprtype(e,linfo), Const)
-                        # first argument must be immutable to ensure e is affect_free
-                        a = ea[2]
-                        typ = widenconst(exprtype(a, linfo))
-                        if !isa(typ, DataType) || typ.mutable || typ.abstract
-                            return false
+                    elseif is_known_call(e, getfield, linfo)
+                        et = exprtype(e,linfo)
+                        if !isa(et,Const) && !(isType(et) && isleaftype(et))
+                            # first argument must be immutable to ensure e is affect_free
+                            a = ea[2]
+                            typ = widenconst(exprtype(a, linfo))
+                            if !isa(typ, DataType) || typ.mutable || typ.abstract
+                                return false
+                            end
                         end
                     end
                 end

base/reshapedarray.jl

@@ -36,8 +36,9 @@ start(R::ReshapedArrayIterator) = start(R.iter)
 end
 length(R::ReshapedArrayIterator) = length(R.iter)
 
-reshape(parent::AbstractArray, shp::Tuple)        = _reshape(parent, to_shape(shp))
 reshape(parent::AbstractArray, dims::IntOrInd...) = reshape(parent, dims)
+reshape(parent::AbstractArray, shp::NeedsShaping) = reshape(parent, to_shape(shp))
+reshape(parent::AbstractArray, dims::Dims)        = _reshape(parent, dims)
 
 reshape{T,N}(parent::AbstractArray{T,N}, ndims::Type{Val{N}}) = parent
 function reshape{T,AN,N}(parent::AbstractArray{T,AN}, ndims::Type{Val{N}})
@@ -47,24 +48,34 @@ end
 # dimensionality N, either filling with OneTo(1) or collapsing the
 # product of trailing dims into the last element
 @pure rdims{N}(out::NTuple{N}, inds::Tuple{}, ::Type{Val{N}}) = out
-@pure rdims{N}(out::NTuple{N}, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = (front(out)..., length(last(out)) * prod(map(length, inds)))
+@pure function rdims{N}(out::NTuple{N}, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}})
+    l = length(last(out)) * prod(map(length, inds))
+    (front(out)..., OneTo(l))
+end
 @pure rdims{N}(out::Tuple, inds::Tuple{}, ::Type{Val{N}}) = rdims((out..., OneTo(1)), (), Val{N})
 @pure rdims{N}(out::Tuple, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = rdims((out..., first(inds)), tail(inds), Val{N})
 
-function _reshape(parent::AbstractArray, dims::Dims)
-    n = _length(parent)
-    prod(dims) == n || throw(DimensionMismatch("parent has $n elements, which is incompatible with size $dims"))
-    __reshape((parent, linearindexing(parent)), dims)
-end
-_reshape(R::ReshapedArray, dims::Dims) = _reshape(R.parent, dims)
+# _reshape on Array returns an Array
+_reshape(parent::Vector, dims::Dims{1}) = parent
+_reshape(parent::Array, dims::Dims{1}) = reshape(parent, dims)
+_reshape(parent::Array, dims::Dims) = reshape(parent, dims)
 
 # When reshaping Vector->Vector, don't wrap with a ReshapedArray
-_reshape{T}(v::ReshapedArray{T,1}, dims::Tuple{Int}) = _reshape(v.parent, dims)
-function _reshape(v::AbstractVector, dims::Tuple{Int})
+function _reshape(v::AbstractVector, dims::Dims{1})
     len = dims[1]
     len == length(v) || throw(DimensionMismatch("parent has $(length(v)) elements, which is incompatible with length $len"))
     v
 end
+# General reshape
+function _reshape(parent::AbstractArray, dims::Dims)
+    n = _length(parent)
+    prod(dims) == n || throw(DimensionMismatch("parent has $n elements, which is incompatible with size $dims"))
+    __reshape((parent, linearindexing(parent)), dims)
+end
+
+# Reshaping a ReshapedArray
+_reshape{T}(v::ReshapedArray{T,1}, dims::Dims{1}) = _reshape(v.parent, dims)
+_reshape(R::ReshapedArray, dims::Dims) = _reshape(R.parent, dims)
 
 function __reshape(p::Tuple{AbstractArray,LinearSlow}, dims::Dims)
     parent = p[1]

contrib/windows/msys_build.sh

@@ -178,10 +178,7 @@ echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user
 # libuv since its static lib is no longer included in the binaries
 # openlibm since we need it as a static library to work properly
 # utf8proc since its headers are not in the binary download
-echo 'override STAGE1_DEPS = libuv' >> Make.user
-echo 'override STAGE2_DEPS = utf8proc' >> Make.user
-echo 'override STAGE3_DEPS = ' >> Make.user
-echo 'override STAGE4_DEPS = ' >> Make.user
+echo 'override DEP_LIBS = libuv utf8proc' >> Make.user
 
 if [ -n "$USEMSVC" ]; then
   # Openlibm doesn't build well with MSVC right now
@@ -193,14 +190,14 @@ if [ -n "$USEMSVC" ]; then
   make -C deps install-libuv install-utf8proc
   cp usr/lib/uv.lib usr/lib/libuv.a
   echo 'override CC += -TP' >> Make.user
-  echo 'override STAGE1_DEPS += dsfmt' >> Make.user
+  echo 'override DEP_LIBS += dsfmt' >> Make.user
 
   # Create a modified version of compile for wrapping link
   sed -e 's/-link//' -e 's/cl/link/g' -e 's/ -Fe/ -OUT:/' \
     -e 's|$dir/$lib|$dir/lib$lib|g' deps/srccache/libuv/compile > linkld
   chmod +x linkld
 else
-  echo 'override STAGE1_DEPS += openlibm' >> Make.user
+  echo 'override DEP_LIBS += openlibm' >> Make.user
   make check-whitespace
   make VERBOSE=1 -C base version_git.jl.phony
   echo 'NO_GIT = 1' >> Make.user

deps/Makefile

@@ -46,7 +46,9 @@ CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(DEPS_CFLAGS)" CXX="$(CXX) $(DEPS_CXX
 CMAKE_CC_ARG := $(CC_ARG) $(DEPS_CFLAGS)
 CMAKE_CXX_ARG := $(CXX_ARG) $(DEPS_CXXFLAGS)
 
-CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) -DLIB_INSTALL_DIR=$(build_shlibdir)
+CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix)
+CMAKE_COMMON += -DCMAKE_INSTALL_LIBDIR=$(build_libdir) -DCMAKE_INSTALL_BINDIR=$(build_bindir)
+CMAKE_COMMON += -DLIB_INSTALL_DIR=$(build_shlibdir)
 ifneq ($(VERBOSE), 0)
 CMAKE_COMMON += -DCMAKE_VERBOSE_MAKEFILE=ON
 endif
@@ -88,31 +90,29 @@ MAKE_COMMON := DESTDIR="" prefix=$(build_prefix) bindir=$(build_depsbindir) libd
 # prevent installing libs into usr/lib64 on opensuse
 unexport CONFIG_SITE
 
-STAGE1_DEPS :=
-STAGE2_DEPS :=
 ifeq ($(USE_GPL_LIBS), 1)
-STAGE3_DEPS := suitesparse-wrapper
+DEP_LIBS := suitesparse-wrapper
 else
-STAGE3_DEPS :=
+DEP_LIBS :=
 endif
 
 ifeq ($(USE_SYSTEM_LIBUV), 0)
-STAGE1_DEPS += libuv
+DEP_LIBS += libuv
 endif
 
 ifeq ($(USE_SYSTEM_LIBUNWIND), 0)
 ifeq ($(OS), Linux)
-STAGE1_DEPS += unwind
+DEP_LIBS += unwind
 else ifeq ($(OS), FreeBSD)
-STAGE1_DEPS += unwind
+DEP_LIBS += unwind
 else ifeq ($(OS), Darwin)
-STAGE1_DEPS += osxunwind
+DEP_LIBS += osxunwind
 endif
 endif
 
 ifeq ($(OS), Linux)
 ifeq ($(USE_SYSTEM_PATCHELF), 0)
-STAGE1_DEPS += patchelf
+DEP_LIBS += patchelf
 PATCHELF:=$(build_depsbindir)/patchelf
 else
 PATCHELF:=patchelf
@@ -124,79 +124,79 @@ PATCHELF_BIN := $(CUSTOM_LD_LIBRARY_PATH) $(PATCHELF)
 
 ifeq ($(USE_SYSTEM_OPENLIBM), 0)
 ifeq ($(USE_SYSTEM_LIBM), 0)
-STAGE1_DEPS += openlibm
+DEP_LIBS += openlibm
 endif
 endif
 
 ifeq ($(USE_SYSTEM_OPENSPECFUN), 0)
-STAGE1_DEPS += openspecfun
+DEP_LIBS += openspecfun
 endif
 
 ifeq ($(USE_SYSTEM_DSFMT), 0)
-STAGE1_DEPS += dsfmt
+DEP_LIBS += dsfmt
 endif
 
 ifeq ($(USE_SYSTEM_LLVM), 0)
-STAGE1_DEPS += llvm
+DEP_LIBS += llvm
 endif
 
 ifeq ($(USE_SYSTEM_PCRE), 0)
-STAGE1_DEPS += pcre
+DEP_LIBS += pcre
 endif
 
 ifeq ($(USE_SYSTEM_BLAS), 0)
-STAGE1_DEPS += openblas
+DEP_LIBS += openblas
 ifeq ($(USE_BLAS64), 1)
 ifeq ($(OS), Darwin)
-STAGE1_DEPS += objconv
+DEP_LIBS += objconv
 endif
 endif
 endif
 
 ifeq ($(USE_GPL_LIBS), 1)
 ifeq ($(USE_SYSTEM_FFTW), 0)
-STAGE1_DEPS += fftw
+DEP_LIBS += fftw
 endif
 endif
 
 ifeq ($(USE_SYSTEM_GMP), 0)
-STAGE1_DEPS += gmp
+DEP_LIBS += gmp
 endif
 
+ifeq ($(USE_SYSTEM_LIBGIT2), 0)
 ifeq ($(USE_SYSTEM_MBEDTLS), 0)
-STAGE1_DEPS += mbedtls
+DEP_LIBS += mbedtls
 endif
 
 ifeq ($(USE_SYSTEM_LIBSSH2), 0)
-STAGE2_DEPS += libssh2
+DEP_LIBS += libssh2
 endif
 
 ifneq ($(OS), WINNT)
 ifeq ($(USE_SYSTEM_CURL), 0)
-STAGE3_DEPS += curl
+DEP_LIBS += curl
 endif
 endif
 
-ifeq ($(USE_SYSTEM_LIBGIT2), 0)
-STAGE4_DEPS += libgit2
-endif
+DEP_LIBS += libgit2
+endif # USE_SYSTEM_LIBGIT2
 
 ifeq ($(USE_SYSTEM_MPFR), 0)
-STAGE2_DEPS += mpfr
+DEP_LIBS += mpfr
 endif
 
 ifeq ($(USE_SYSTEM_ARPACK), 0)
-STAGE2_DEPS += arpack
+DEP_LIBS += arpack
 endif
 
 ifeq ($(USE_GPL_LIBS), 1)
 ifeq ($(USE_SYSTEM_SUITESPARSE), 0)
-STAGE2_DEPS += suitesparse
+DEP_LIBS += suitesparse
 endif
 endif
 
 ifeq ($(USE_SYSTEM_UTF8PROC), 0)
-STAGE2_DEPS += utf8proc
+DEP_LIBS += utf8proc
 endif
 
 # Only compile standalone LAPACK if we are not using OpenBLAS.
@@ -205,7 +205,7 @@ endif
 # build LAPACK as the vendor LAPACK may be too old (eg. Apple vecLib)
 ifeq ($(USE_SYSTEM_BLAS), 1)
 ifeq ($(USE_SYSTEM_LAPACK), 0)
-STAGE2_DEPS += lapack
+DEP_LIBS += lapack
 endif
 endif
 
@@ -217,8 +217,6 @@ endif
 
 ## Common build target prefixes
 
-DEP_LIBS := $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS) $(STAGE4_DEPS)
-
 default: install | $(build_prefix)
 get: $(addprefix get-, $(DEP_LIBS))
 configure: $(addprefix configure-, $(DEP_LIBS))

deps/arpack.mk

@@ -32,10 +32,9 @@ ARPACK_OBJ_TARGET := $(build_shlibdir)/libarpack.$(SHLIB_EXT)
 
 ARPACK_MFLAGS := F77="$(FC)" MPIF77="$(FC)"
 ARPACK_FFLAGS += $(FFLAGS) $(JFFLAGS)
-ARPACK_FLAGS := --with-blas="$(LIBBLAS)" --with-lapack="$(LIBLAPACK)" --disable-mpi --enable-shared FFLAGS="$(ARPACK_FFLAGS)" CFLAGS="$(CFLAGS) $(ARPACK_CFLAGS)"
-ifneq ($(OS),WINNT)
-ARPACK_FLAGS += LDFLAGS="$(LDFLAGS) -Wl,-rpath,'$(build_libdir)'"
-endif
+ARPACK_FLAGS := --with-blas="$(LIBBLAS)" --with-lapack="$(LIBLAPACK)" \
+    --disable-mpi --enable-shared FFLAGS="$(ARPACK_FFLAGS)" \
+    CFLAGS="$(CFLAGS) $(ARPACK_CFLAGS)" LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN)"
 
 # ARPACK-NG upstream keeps changing their download filenames
 $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER).tar.gz: | $(SRCDIR)/srccache

deps/blas.mk

@@ -54,7 +54,9 @@ OPENBLAS_BUILD_OPTS += OSNAME=$(OS) CROSS=1 HOSTCC=$(HOSTCC)
 endif
 ifeq ($(OS),WINNT)
 ifneq ($(ARCH),x86_64)
+ifneq ($(USECLANG),1)
 OPENBLAS_BUILD_OPTS += CFLAGS="$(CFLAGS) -mincoming-stack-boundary=2"
+endif
 OPENBLAS_BUILD_OPTS += FFLAGS="$(FFLAGS) -mincoming-stack-boundary=2"
 endif
 endif
@@ -200,10 +202,9 @@ LAPACK_OBJ_TARGET :=
 LAPACK_OBJ_SOURCE :=
 endif
 
-LAPACK_MFLAGS := NOOPT="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS) -O0" OPTS="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS)" FORTRAN="$(FC)" LOADER="$(FC)"
-ifneq ($(OS),WINNT)
-LAPACK_MFLAGS += BLASLIB="-Wl,-rpath,'$(build_libdir)' $(LIBBLAS)"
-endif
+LAPACK_MFLAGS := NOOPT="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS) -O0" \
+    OPTS="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS)" FORTRAN="$(FC)" \
+    LOADER="$(FC)" BLASLIB="$(RPATH_ESCAPED_ORIGIN) $(LIBBLAS)"
 
 $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz: | $(SRCDIR)/srccache
 	$(JLDOWNLOAD) $@ http://www.netlib.org/lapack/$(notdir $@)

deps/curl.mk

@@ -3,29 +3,33 @@
 CURL_SRC_TARGET := $(BUILDDIR)/curl-$(CURL_VER)/lib/.libs/libcurl.$(SHLIB_EXT)
 CURL_OBJ_TARGET := $(build_shlibdir)/libcurl.$(SHLIB_EXT)
 
-ifneq ($(OS),WINNT)
 CURL_LDFLAGS := $(RPATH_ESCAPED_ORIGIN)
-endif
 
 $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2: | $(SRCDIR)/srccache
 	$(JLDOWNLOAD) $@ https://curl.haxx.se/download/curl-$(CURL_VER).tar.bz2
 
-$(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(MBEDTLS_OBJ_TARGET) $(LIBSSH2_OBJ_TARGET)
+$(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2
 	$(JLCHECKSUM) $<
 	cd $(dir $<) && $(TAR) jxf $(notdir $<)
 	touch -c $@
 
+ifeq ($(USE_SYSTEM_MBEDTLS), 0)
+$(BUILDDIR)/curl-$(CURL_VER)/config.status: $(MBEDTLS_OBJ_TARGET)
+endif
+ifeq ($(USE_SYSTEM_LIBSSH2), 0)
+$(BUILDDIR)/curl-$(CURL_VER)/config.status: $(LIBSSH2_OBJ_TARGET)
+endif
 $(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure
 	mkdir -p $(dir $@)
 	cd $(dir $@) && \
 	$< $(CONFIGURE_COMMON) --includedir=$(build_includedir) \
-	--without-ssl --without-gnutls --without-gssapi \
-	--without-libidn --without-libmetalink --without-librtmp \
-	--without-nghttp2 --without-nss --without-polarssl \
-	--without-spnego --disable-ares --disable-ldap \
-	--disable-ldaps --without-zsh-functions-dir \
-	--with-libssh2=$(build_prefix) --with-mbedtls=$(build_prefix) \
-	CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)"
+		--without-ssl --without-gnutls --without-gssapi --without-zlib \
+		--without-libidn --without-libmetalink --without-librtmp \
+		--without-nghttp2 --without-nss --without-polarssl \
+		--without-spnego --without-libpsl --disable-ares \
+		--disable-ldap --disable-ldaps --without-zsh-functions-dir \
+		--with-libssh2=$(build_prefix) --with-mbedtls=$(build_prefix) \
+		CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)"
 	touch -c $@
 
 $(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status