build: use libwhich to improve installation reliability

.travis.yml

@@ -94,8 +94,11 @@ before_install:
             BUILDOPTS="$BUILDOPTS USE_SYSTEM_$lib=1";
         done;
         export LDFLAGS="-L$(brew --prefix openblas-julia)/lib -L$(brew --prefix suite-sparse-julia)/lib";
-        export DYLD_FALLBACK_LIBRARY_PATH="/usr/local/lib:/lib:/usr/lib:$(brew --prefix openblas-julia)/lib:$(brew --prefix suite-sparse-julia)/lib:$(brew --prefix arpack-julia)/lib";
-        export JULIA_MACOS_SPAWN="DYLD_FALLBACK_LIBRARY_PATH=\"$DYLD_FALLBACK_LIBRARY_PATH\" \$1";
+        spawn_DYLD_FALLBACK_LIBRARY_PATH="/usr/local/lib:/lib:/usr/lib";
+        spawn_DYLD_FALLBACK_LIBRARY_PATH+=":$(brew --prefix openblas-julia)/lib";
+        spawn_DYLD_FALLBACK_LIBRARY_PATH+=":$(brew --prefix suite-sparse-julia)/lib";
+        spawn_DYLD_FALLBACK_LIBRARY_PATH+=":$(brew --prefix arpack-julia)/lib";
+        export JULIA_MACOS_SPAWN="DYLD_FALLBACK_LIBRARY_PATH=\"$spawn_DYLD_FALLBACK_LIBRARY_PATH\" \$1";
         export BUILDOPTS="$BUILDOPTS spawn=\$(JULIA_MACOS_SPAWN)";
         make $BUILDOPTS -C contrib -f repackage_system_suitesparse4.make;
         export JULIA_CPU_CORES=2;
@@ -126,13 +129,15 @@ script:
           false; }
     # compile / install Julia
     - make $BUILDOPTS NO_GIT=1 prefix=/tmp/julia install | moreutils/ts -s "%.s"
-    - make $BUILDOPTS NO_GIT=1 build-stats
-    - du -sk /tmp/julia/*
     - if [ `uname` = "Darwin" ]; then
         for name in suitesparseconfig spqr umfpack colamd cholmod amd suitesparse_wrapper; do
             install -pm755 usr/lib/lib${name}*.dylib* /tmp/julia/lib/julia/;
         done;
       fi
+    - make $BUILDOPTS NO_GIT=1 build-stats
+    - du -sk /tmp/julia/*
+    - ls -l /tmp/julia/lib
+    - ls -l /tmp/julia/lib/julia
     - cd .. && mv julia julia2
     # run tests
     - /tmp/julia/bin/julia --sysimage-native-code=no -e 'true' &&
@@ -145,7 +150,7 @@ script:
     # test that the embedding code works on our installation
     - mkdir /tmp/embedding-test &&
         make check -C /tmp/julia/share/doc/julia/examples/embedding \
-             JULIA="DYLD_FALLBACK_LIBRARY_PATH='$DYLD_FALLBACK_LIBRARY_PATH' /tmp/julia/bin/julia" \
+             JULIA="/tmp/julia/bin/julia" \
              BIN=/tmp/embedding-test \
              "$(cd julia2 && make print-CC)"
     # restore initial state and prepare for travis caching

Makefile

@@ -232,61 +232,44 @@ julia-base-cache: julia-sysimg-$(JULIA_BUILD_MODE) | $(DIRS) $(build_datarootdir
 JL_LIBS := julia julia-debug
 
 # private libraries, that are installed in $(prefix)/lib/julia
-JL_PRIVATE_LIBS := ccalltest
+JL_PRIVATE_LIBS-0 := libccalltest
 ifeq ($(USE_GPL_LIBS), 1)
-JL_PRIVATE_LIBS += suitesparse_wrapper
-endif
-ifeq ($(USE_SYSTEM_PCRE),0)
-JL_PRIVATE_LIBS += pcre
+JL_PRIVATE_LIBS-0 += libsuitesparse_wrapper
+endif
+JL_PRIVATE_LIBS-$(USE_SYSTEM_PCRE) += libpcre2-8
+JL_PRIVATE_LIBS-$(USE_SYSTEM_DSFMT) += libdSFMT
+JL_PRIVATE_LIBS-$(USE_SYSTEM_GMP) += libgmp
+JL_PRIVATE_LIBS-$(USE_SYSTEM_MPFR) += libmpfr
+JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSSH2) += libssh2
+JL_PRIVATE_LIBS-$(USE_SYSTEM_MBEDTLS) += libmbedtls libmbedcrypto libmbedx509
+JL_PRIVATE_LIBS-$(USE_SYSTEM_CURL) += libcurl
+JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBGIT2) += libgit2
+JL_PRIVATE_LIBS-$(USE_SYSTEM_ARPACK) += libarpack
+ifeq ($(USE_LLVM_SHLIB),1)
+JL_PRIVATE_LIBS-$(USE_SYSTEM_LLVM) += libLLVM
 endif
+
 ifeq ($(USE_SYSTEM_OPENLIBM),0)
 ifeq ($(USE_SYSTEM_LIBM),0)
-JL_PRIVATE_LIBS += openlibm
-endif
-endif
-ifeq ($(USE_SYSTEM_DSFMT),0)
-JL_PRIVATE_LIBS += dSFMT
-endif
-ifeq ($(USE_SYSTEM_BLAS),0)
-JL_PRIVATE_LIBS += openblas
-else ifeq ($(USE_SYSTEM_LAPACK),0)
-JL_PRIVATE_LIBS += lapack
-endif
-ifeq ($(USE_SYSTEM_GMP),0)
-JL_PRIVATE_LIBS += gmp
-endif
-ifeq ($(USE_SYSTEM_MPFR),0)
-JL_PRIVATE_LIBS += mpfr
-endif
-ifeq ($(USE_SYSTEM_MBEDTLS),0)
-JL_PRIVATE_LIBS += mbedtls mbedcrypto mbedx509
-endif
-ifeq ($(USE_SYSTEM_LIBSSH2),0)
-JL_PRIVATE_LIBS += ssh2
+JL_PRIVATE_LIBS-0 += $(LIBMNAME)
 endif
-ifeq ($(USE_SYSTEM_CURL),0)
-JL_PRIVATE_LIBS += curl
 endif
-ifeq ($(USE_SYSTEM_LIBGIT2),0)
-JL_PRIVATE_LIBS += git2
-endif
-ifeq ($(USE_SYSTEM_ARPACK),0)
-JL_PRIVATE_LIBS += arpack
+
+JL_PRIVATE_LIBS-$(USE_SYSTEM_BLAS) += $(LIBBLASNAME)
+ifneq ($(LIBLAPACKNAME),$(LIBBLASNAME))
+JL_PRIVATE_LIBS-$(USE_SYSTEM_LAPACK) += $(LIBLAPACKNAME)
 endif
-ifeq ($(USE_SYSTEM_SUITESPARSE),0)
+
 ifeq ($(USE_GPL_LIBS), 1)
-JL_PRIVATE_LIBS += amd camd ccolamd cholmod colamd umfpack spqr suitesparseconfig
-endif
-endif
-ifeq ($(USE_SYSTEM_LLVM),0)
-ifeq ($(USE_LLVM_SHLIB),1)
-JL_PRIVATE_LIBS += LLVM
+ifeq ($(USE_SYSTEM_SUITESPARSE),0)
+JL_PRIVATE_LIBS-0 += libamd libcamd libccolamd libcholmod libcolamd libumfpack libspqr libsuitesparseconfig
 endif
 endif
+
 ifeq ($(OS),Darwin)
 ifeq ($(USE_SYSTEM_BLAS),1)
 ifeq ($(USE_SYSTEM_LAPACK),0)
-JL_PRIVATE_LIBS += gfortblas
+JL_PRIVATE_LIBS-0 += libgfortblas
 endif
 endif
 endif
@@ -342,13 +325,17 @@ endif
 			fi \
 		done \
 	done
-	for suffix in $(JL_PRIVATE_LIBS) ; do \
-		for lib in $(build_libdir)/lib$${suffix}*.$(SHLIB_EXT)*; do \
+	for suffix in $(JL_PRIVATE_LIBS-0) ; do \
+		for lib in $(build_libdir)/$${suffix}*.$(SHLIB_EXT)*; do \
 			if [ "$${lib##*.}" != "dSYM" ]; then \
 				$(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) ; \
 			fi \
 		done \
 	done
+	for suffix in $(JL_PRIVATE_LIBS-1) ; do \
+		lib=$(build_private_libdir)/$${suffix}.$(SHLIB_EXT); \
+		$(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) ; \
+	done
 endif
 
 	# Copy public headers

base/Makefile

@@ -104,7 +104,101 @@ else
 endif
 endif
 
-.PHONY: $(BUILDDIR)/build_h.jl.phony $(BUILDDIR)/version_git.jl.phony clean all
+ifneq ($(OS), WINNT)
+# For any USE_SYSTEM_* libraries that will be dynamically loaded by libjulia,
+# put a symlink in the private libdir.
+# System package managers may want to install these links manually,
+# but if not, this will try to create the proper symlink to the right minor version.
+all: symlink_system_libraries
+SYMLINK_SYSTEM_LIBRARIES :=
+
+# if it's a symlink, pick up the symlink target instead, for one level of indirection
+resolve_path = \
+	$1_wd="`pwd`" && \
+	$1_=`readlink -n $${$1} || true` && \
+	if [ -n "$${$1_}" ]; then $1_wd=`dirname "$${$1}"`; $1="$${$1_}"; fi
+## if it's a relative path, make it an absolute path
+resolve_path += && \
+	if [ -z "`echo $${$1} | grep '^/'`" ]; then $1=$${$1_wd}/$${$1}; fi
+ifeq ($(OS), Darwin)
+# try to use the install_name id instead (unless it is an @rpath or such)
+# if it's a relative path, make it an absolute path using the working directory from $1,
+# while in theory incorrect, this has been observed to be a common build mistake for many libraries
+resolve_path += && \
+	$1_=`otool -D $${$1} | tail -n +2 | sed -e 's/^@.*$$//'` && \
+	if [ -n "$${$1_}" ]; then \
+	$1_wd=`dirname "$${$1}"`; $1=$${$1_}; \
+	if [ -z "`echo $${$1} | grep '^/'`" ]; then $1=$${$1_wd}/$${$1}; fi; \
+	fi
+endif
+
+## debug code: `make resolve-path P=<path to test>`
+#resolve_path += && \
+#	echo "$${$1_wd} $${$1}"
+#resolve-path:
+#	$(call resolve_path,P) && \
+#	echo "$$P"
+
+define symlink_system_library
+symlink_$1: $$(build_private_libdir)/$1.$$(SHLIB_EXT)
+$$(build_private_libdir)/$1.$$(SHLIB_EXT):
+	REALPATH=`$$(call spawn,$$(build_depsbindir)/libwhich) -p $$(notdir $$@)` && \
+	$$(call resolve_path,REALPATH) && \
+	[ -e "$$$$REALPATH" ] && \
+	([ ! -e "$$@" ] || rm "$$@") && \
+	echo ln -sf "$$$$REALPATH" "$$@" && \
+	ln -sf "$$$$REALPATH" "$$@"
+ifneq ($2,)
+ifneq ($$(USE_SYSTEM_$2),0)
+SYMLINK_SYSTEM_LIBRARIES += symlink_$1
+endif
+endif
+endef
+
+# the following excludes: libuv.a, libutf8proc.a, suitesparse*
+
+$(eval $(call symlink_system_library,$(LIBMNAME)))
+ifneq ($(USE_SYSTEM_LIBM),0)
+SYMLINK_SYSTEM_LIBRARIES += symlink_$(LIBMNAME)
+else ifneq ($(USE_SYSTEM_OPENLIBM),0)
+SYMLINK_SYSTEM_LIBRARIES += symlink_$(LIBMNAME)
+endif
+
+$(eval $(call symlink_system_library,libpcre2-8,PCRE))
+$(eval $(call symlink_system_library,libdSFMT,DSFMT))
+$(eval $(call symlink_system_library,$(LIBBLASNAME),BLAS))
+ifneq ($(LIBLAPACKNAME),$(LIBBLASNAME))
+$(eval $(call symlink_system_library,$(LIBLAPACKNAME),LAPACK))
+endif
+$(eval $(call symlink_system_library,libgmp,GMP))
+$(eval $(call symlink_system_library,libmpfr,MPFR))
+$(eval $(call symlink_system_library,libarpack,ARPACK))
+$(eval $(call symlink_system_library,libmbedtls,MBEDTLS))
+$(eval $(call symlink_system_library,libssh2,LIBSSH2))
+$(eval $(call symlink_system_library,libcurl,CURL))
+$(eval $(call symlink_system_library,libgit2,LIBGIT2))
+ifneq ($(DISABLE_LIBUNWIND),0)
+$(eval $(call symlink_system_library,libunwind,LIBUNWIND))
+endif
+endif # WINNT
+
+symlink_libLLVM: $(build_private_libdir)/libLLVM.dylib
+$(build_private_libdir)/libLLVM.dylib:
+	REALPATH=`$(LLVM_CONFIG_HOST) --libdir`/libLLVM.$(SHLIB_EXT) && \
+	$(call resolve_path,REALPATH) && \
+	[ -e "$$REALPATH" ] && \
+	([ ! -e "$@" ] || rm "$@") && \
+	echo ln -sf "$$REALPATH" "$@" && \
+	ln -sf "$$REALPATH" "$@"
+ifneq ($(USE_SYSTEM_LLVM),0)
+ifneq ($(USE_LLVM_SHLIB),0)
+SYMLINK_SYSTEM_LIBRARIES += symlink_libLLVM
+endif
+endif
+
+symlink_system_libraries: $(SYMLINK_SYSTEM_LIBRARIES)
+
+.PHONY: $(BUILDDIR)/build_h.jl.phony $(BUILDDIR)/version_git.jl.phony clean all symlink_*
 
 clean:
 	rm -f $(BUILDDIR)/pcre_h.jl

deps/Makefile

@@ -21,7 +21,7 @@ include $(SRCDIR)/tools/git-external.mk
 # if you are adding a new target, it can help to copy an similar, existing target
 #
 # autoconf configure-driven scripts: llvm pcre arpack unwind gmp mpfr patchelf libuv curl
-# custom Makefile rules: openlibm dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv osxunwind
+# custom Makefile rules: openlibm dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv osxunwind libwhich
 # CMake libs: libgit2 libssh2 mbedtls
 #
 # downloaded from git: llvm-svn, libuv, libopenlibm, utf8proc, libgit2, libssh2
@@ -150,6 +150,10 @@ DEP_LIBS += lapack
 endif
 endif
 
+ifneq ($(OS), WINNT)
+DEP_LIBS += libwhich
+endif
+
 DEP_LIBS_STAGED := $(filter-out suitesparse suitesparse-wrapper osxunwind,$(DEP_LIBS)) # unlist targets that have not been converted to use the staged-install
 
 
@@ -168,7 +172,7 @@ uninstall: $(addprefix uninstall-, $(DEP_LIBS_STAGED))
 cleanall: $(addprefix clean-, $(DEP_LIBS))
 distcleanall: $(addprefix distclean-, $(DEP_LIBS))
 	rm -rf $(build_prefix)
-getall: get-llvm get-libuv get-pcre get-openlibm get-dsfmt get-openblas get-lapack get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-objconv get-mbedtls get-libssh2 get-curl get-libgit2
+getall: get-llvm get-libuv get-pcre get-openlibm get-dsfmt get-openblas get-lapack get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-objconv get-mbedtls get-libssh2 get-curl get-libgit2 get-libwhich
 
 include $(SRCDIR)/llvm.mk
 include $(SRCDIR)/libuv.mk
@@ -188,3 +192,4 @@ include $(SRCDIR)/mbedtls.mk
 include $(SRCDIR)/libssh2.mk
 include $(SRCDIR)/curl.mk
 include $(SRCDIR)/libgit2.mk
+include $(SRCDIR)/libwhich.mk

deps/checksums/libwhich-81e9723c0273d78493dc8c8ed570f68d9ce7e89e.tar.gz/md5

@@ -0,0 +1 @@
+22fd8368c7b40209dada50e3205c1294

deps/checksums/libwhich-81e9723c0273d78493dc8c8ed570f68d9ce7e89e.tar.gz/sha512

@@ -0,0 +1 @@
+6fb77b715d70d9bc95a8546c3bf97bd3677c7ea344b88bb5bc3bbfac9dceabe8a8cde7a0f64dec884cde802e4a3000e30837d3f824b5a9242348c4fe061526a3

deps/libwhich.mk

@@ -0,0 +1,35 @@
+## LIBWHICH ##
+LIBWHICH_GIT_URL := git://github.com/vtjnash/libwhich.git
+LIBWHICH_TAR_URL = https://api.github.com/repos/vtjnash/libwhich/tarball/$1
+$(eval $(call git-external,libwhich,LIBWHICH,,,$(BUILDDIR)))
+
+LIBWHICH_OBJ_LIB := $(build_depsbindir)/libwhich
+
+$(BUILDDIR)/$(LIBWHICH_SRC_DIR)/build-compiled: $(BUILDDIR)/$(LIBWHICH_SRC_DIR)/source-extracted
+	$(MAKE) -C $(dir $<) libwhich
+	echo 1 > $@
+
+$(BUILDDIR)/$(LIBWHICH_SRC_DIR)/build-checked: $(BUILDDIR)/$(LIBWHICH_SRC_DIR)/build-compiled
+ifeq ($(OS),$(BUILD_OS))
+	$(MAKE) -C $(dir $@) $(LIBWHICH_MFLAGS) check
+endif
+	echo 1 > $@
+
+define LIBWHICH_INSTALL
+	mkdir -p $2/$$(build_depsbindir)
+	cp $1/libwhich $2/$$(build_depsbindir)
+endef
+$(eval $(call staged-install, \
+	libwhich,$(LIBWHICH_SRC_DIR), \
+	LIBWHICH_INSTALL,,,))
+
+clean-libwhich:
+	-rm $(BUILDDIR)/$(LIBWHICH_SRC_DIR)/build-compiled
+	-$(MAKE) -C $(BUILDDIR)/$(LIBWHICH_SRC_DIR) clean
+
+get-libwhich: $(LIBWHICH_SRC_FILE)
+extract-libwhich: $(BUILDDIR)/$(LIBWHICH_SRC_DIR)/source-extracted
+configure-libwhich: extract-libwhich
+compile-libwhich: $(BUILDDIR)/$(LIBWHICH_SRC_DIR)/build-compiled
+fastcheck-libwhich: check-libwhich
+check-libwhich: $(BUILDDIR)/$(LIBWHICH_SRC_DIR)/build-checked

deps/libwhich.version

@@ -0,0 +1,2 @@
+LIBWHICH_BRANCH=master
+LIBWHICH_SHA1=81e9723c0273d78493dc8c8ed570f68d9ce7e89e