Merge #2280: [Depends] Cleanup Rust and add multi-arch linux support

92bebbe Use statically linked rust toolchain (Fuzzbawls)
7dd7161 Depends: allow for non-x86_64 build hosts for rust linux (Fuzzbawls)
fc7acb3 [Depends] Switch to `cargo vendor` for Rust dependencies (Fuzzbawls)

Pull request description:

  The first commit here comes from zcash/zcash#4725 and provides a significant cleanup in terms of number of files needed in the depends tree.

  The second commit re-works the way we fetch the Rust dependencies to allow depends to be built for linux on non-x86_64 systems. Currently I've compiled native toolchains for `x86_64`, `arm`, `armv7`, and `aarch64`

  The third commit addresses #2127 by re-compiling the rust toolchains with static linking.

  The fourth commit is temporary, and will be removed prior to merging.

Top commit has no ACKs.

Tree-SHA512: c3143bc02f230e7b96f66fcaeebd27912be71e37ae354c9c211364a99f545788af8f3be93131f590ec24c02da7f05dbecc632ebc9a0b45b5c8de70cdc3feefe1

depends/Makefile

@@ -48,10 +48,10 @@ HASH_LENGTH:=11
 DOWNLOAD_CONNECT_TIMEOUT:=30
 DOWNLOAD_RETRIES:=3
 CRATE_REGISTRY:=vendored-sources
+CRATE_ARCHIVE = $(SOURCES_PATH)/vendored-crates.tar.gz
 HOST_ID_SALT ?= salt
 BUILD_ID_SALT ?= salt
 
-LIBRUSTZCASH_OVERRIDE ?=
 
 host:=$(BUILD)
 ifneq ($(HOST),)
@@ -135,7 +135,7 @@ natpmp_packages_$(NO_NATPMP) = $(natpmp_packages)
 
 zmq_packages_$(NO_ZMQ) = $(zmq_packages)
 
-packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(rust_packages) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_)
+packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_)
 native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)
 
 ifneq ($(qt_packages_),)
@@ -169,6 +169,12 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
 	$(AT)echo copying packages: $^
 	$(AT)echo to: $(@D)
 	$(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); )
+	$(AT)if test -f "$(CRATE_ARCHIVE)"; \
+	then echo Extracting pre-vendored crates from $(CRATE_ARCHIVE)...; \
+		tar xf $(CRATE_ARCHIVE) -C $(@D); \
+	else echo Vendoring crates...; \
+		$(@D)/native/bin/cargo vendor --locked --manifest-path $(BASEDIR)/../Cargo.toml $(@D)/$(CRATE_REGISTRY); \
+	fi
 	$(AT)touch $@
 
 $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id)
@@ -186,7 +192,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
             -e 's|@CXXFLAGS@|$(strip $(host_CXXFLAGS) $(host_$(release_type)_CXXFLAGS))|' \
             -e 's|@CPPFLAGS@|$(strip $(host_CPPFLAGS) $(host_$(release_type)_CPPFLAGS))|' \
             -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \
-            -e 's|@rust_target@|$(call rust_target,rust,$(canonical_host),$(host_os))|' \
+            -e 's|@rust_target@|$(call rust_target,native_rust,$(canonical_host),$(host_os))|' \
             -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \
             -e 's|@no_qt@|$(NO_QT)|' \
             -e 's|@no_zmq@|$(NO_ZMQ)|' \
@@ -213,9 +219,13 @@ define check_or_remove_sources
 endef
 
 check-packages:
-	@$(foreach package,$(all_packages),$(call check_or_remove_cached,$(package));)
+	@$(foreach package,$(packages),$(call check_or_remove_cached,$(package));)
+	@$(foreach package,$(rust_crates),$(call check_or_remove_cached,$(package));)
+	@$(foreach package,$(native_packages),$(call check_or_remove_cached,$(package));)
 check-sources:
-	@$(foreach package,$(all_packages),$(call check_or_remove_sources,$(package));)
+	@$(foreach package,$(packages),$(call check_or_remove_sources,$(package));)
+	@$(foreach package,$(rust_crates),$(call check_or_remove_sources,$(package));)
+	@$(foreach package,$(native_packages),$(call check_or_remove_sources,$(package));)
 
 $(host_prefix)/share/config.site: check-packages
 
@@ -230,6 +240,18 @@ clean:
 install: check-packages $(host_prefix)/share/config.site
 
 
+crates_download_dir=$(base_download_dir)/crates
+download-crates: native_rust
+	$(AT)echo Vendoring crates...
+	$(AT)mkdir -p $(SOURCES_PATH)
+	$(AT)rm -rf $(crates_download_dir)
+	$(AT)mkdir -p $(crates_download_dir)
+	$(AT)tar xf $(native_rust_cached) -C $(crates_download_dir)
+	$(AT)$(crates_download_dir)/native/bin/cargo vendor --locked --manifest-path $(BASEDIR)/../Cargo.toml $(crates_download_dir)/$(CRATE_REGISTRY)
+	$(AT)cd $(crates_download_dir); find $(CRATE_REGISTRY) | sort | tar --no-recursion -czf $(CRATE_ARCHIVE) -T -
+	$(AT)rm -rf $(crates_download_dir)
+
+
 download-one: check-sources $(all_sources)
 
 download-osx:
@@ -238,8 +260,8 @@ download-linux:
 	@$(MAKE) -s HOST=x86_64-unknown-linux-gnu download-one
 download-win:
 	@$(MAKE) -s HOST=x86_64-w64-mingw32 download-one
-download: download-osx download-linux download-win
+download: download-crates download-osx download-linux download-win
 
 $(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package))))
 
-.PHONY: install cached clean clean-all download-one download-osx download-linux download-win download check-packages check-sources
+.PHONY: install cached clean clean-all download-crates download-one download-osx download-linux download-win download check-packages check-sources

depends/README.md

@@ -95,8 +95,6 @@ build script logic) are searched for among the host system packages using
 `pkg-config`. It allows building with packages of other (newer) versions</dd>
 <dt>DEBUG</dt>
 <dd>disable some optimizations and enable more runtime checking</dd>
-<dt>LIBRUSTZCASH_OVERRIDE</dt>
-<dd>Path to a local librustzcash repository</dd>
 <dt>HOST_ID_SALT</dt>
 <dd>Optional salt to use when generating host package ids</dd>
 <dt>BUILD_ID_SALT</dt>

depends/funcs.mk

@@ -34,21 +34,6 @@ define fetch_file
       $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(3),$(4),$(5))))
 endef
 
-define generate_crate_checksum
-$(BASEDIR)/cargo-checksum.sh "$($(1)_file_name)" "$(build_SHA256SUM)" "\"$($(1)_sha256_hash)\""
-endef
-
-define generate_unpackaged_crate_checksum
-$(BASEDIR)/cargo-checksum.sh "$($(1)_file_name)" "$(build_SHA256SUM)" "null"
-endef
-
-define vendor_crate_source
-mkdir -p $($(1)_staging_prefix_dir)/$(CRATE_REGISTRY) && \
-cp -r $($(1)_extract_dir) $($(1)_staging_prefix_dir)/$(CRATE_REGISTRY)/$($(1)_crate_name) && \
-cd $($(1)_staging_prefix_dir)/$(CRATE_REGISTRY)/$($(1)_crate_versioned_name) && \
-rm -r `basename $($(1)_patch_dir)` .stamp_* .$($(1)_file_name).hash
-endef
-
 define int_get_build_recipe_hash
 $(eval $(1)_all_file_checksums:=$(shell $(build_SHA256SUM) $(meta_depends) packages/$(1).mk $(addprefix $(PATCHES_PATH)/$(1)/,$($(1)_patches)) | cut -d" " -f1))
 $(eval $(1)_recipe_hash:=$(shell echo -n "$($(1)_all_file_checksums)" | $(build_SHA256SUM) | cut -d" " -f1))
@@ -65,11 +50,11 @@ final_build_id_long+=$($(package)_build_id_long)
 #override platform specific files and hashes
 $(eval $(1)_file_name=$(if $($(1)_exact_file_name),$($(1)_exact_file_name),$(if $($(1)_file_name_$(host_os)),$($(1)_file_name_$(host_os)),$($(1)_file_name))))
 $(eval $(1)_sha256_hash=$(if $($(1)_exact_sha256_hash),$($(1)_exact_sha256_hash),$(if $($(1)_sha256_hash_$(host_os)),$($(1)_sha256_hash_$(host_os)),$($(1)_sha256_hash))))
+$(eval $(1)_download_file=$(if $($(1)_exact_download_file),$($(1)_exact_download_file),$(if $($(1)_download_file_$(host_os)),$($(1)_download_file_$(host_os)),$(if $($(1)_download_file),$($(1)_download_file),$($(1)_file_name)))))
 
 
 #compute package-specific paths
 $(1)_build_subdir?=.
-$(1)_download_file?=$($(1)_file_name)
 $(1)_source_dir:=$(SOURCES_PATH)
 $(1)_source:=$$($(1)_source_dir)/$($(1)_file_name)
 $(1)_staging_dir=$(base_staging_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)