R4R: Infrastructure for reproducible builds

.circleci/config.yml

@@ -4,8 +4,6 @@ defaults: &linux_defaults
   working_directory: /go/src/github.com/cosmos/cosmos-sdk
   docker:
     - image: circleci/golang:1.12.4
-  environment:
-    GOBIN: /tmp/workspace/bin
 
 
 ############
@@ -25,8 +23,7 @@ set_macos_env: &macos_env
     command: |
       echo 'export PATH=$PATH:$HOME/go/bin' >> $BASH_ENV
       echo 'export GOPATH=$HOME/project' >> $BASH_ENV
-      echo 'export GOBIN=$GOPATH/bin' >> $BASH_ENV
-      echo 'export PATH=$PATH:$HOME/go/bin:$GOBIN' >> $BASH_ENV
+      echo 'export PATH=$PATH:$HOME/go/bin:$GOPATH/bin' >> $BASH_ENV
       echo 'export GO111MODULE=on'
 
 ############
@@ -40,12 +37,6 @@ docs_update: &docs_deploy
   environment:
     AWS_REGION: us-east-1
 
-deps: &dependencies
-  run:
-    name: dependencies
-    command: |
-      export PATH="$GOBIN:$PATH"
-
 jobs:
   setup_dependencies:
     <<: *linux_defaults
@@ -59,13 +50,11 @@ jobs:
       - run:
           name: tools
           command: |
-            export PATH="$GOBIN:$PATH"
-            make tools
-      - *dependencies
+            make tools TOOLS_DESTDIR=/tmp/workspace/bin
       - run:
           name: binaries
           command: |
-            export PATH="$GOBIN:$PATH"
+            export PATH=/tmp/workspace/bin:$PATH
             make go-mod-cache
             make install
       - save_cache:
@@ -85,14 +74,13 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Lint source
           command: |
-            export PATH="$GOBIN:$PATH"
+            export PATH=/tmp/workspace/bin:$PATH
             make ci-lint
 
   integration_tests:
@@ -102,14 +90,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test cli
           command: |
-            export PATH="$GOBIN:$PATH"
             make test_cli
 
   test_sim_gaia_nondeterminism:
@@ -119,14 +105,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test individual module simulations
           command: |
-            export PATH="$GOBIN:$PATH"
             make test_sim_gaia_nondeterminism
 
   test_sim_gaia_fast:
@@ -136,14 +120,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test full Gaia simulation
           command: |
-            export PATH="$GOBIN:$PATH"
             make test_sim_gaia_fast
 
   test_sim_gaia_import_export:
@@ -153,14 +135,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test Gaia import/export simulation
           command: |
-            export PATH="$GOBIN:$PATH"
             make test_sim_gaia_import_export
 
   test_sim_gaia_simulation_after_import:
@@ -170,14 +150,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test Gaia import/export simulation
           command: |
-            export PATH="$GOBIN:$PATH"
             make test_sim_gaia_simulation_after_import
 
   test_sim_gaia_multi_seed_long:
@@ -187,14 +165,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test multi-seed Gaia simulation long
           command: |
-            export PATH="$GOBIN:$PATH"
             export GO111MODULE=on
             make runsim
             runsim 500 50 TestFullGaiaSimulation
@@ -206,14 +182,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test multi-seed Gaia simulation short
           command: |
-            export PATH="$GOBIN:$PATH"
             export GO111MODULE=on
             make runsim
             runsim 50 10 TestFullGaiaSimulation
@@ -225,15 +199,13 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - run: mkdir -p /tmp/logs
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Run tests
           command: |
-            export PATH="$GOBIN:$PATH"
             export VERSION="$(git describe --tags --long | sed 's/v\(.*\)/\1/')"
             export GO111MODULE=on
             for pkg in $(go list ./... | grep -v github.com/cosmos/cosmos-sdk/cmd/gaia/cli_test | grep -v '/simulation' | circleci tests split --split-by=timings); do
@@ -254,7 +226,6 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - run:
           name: gather
           command: |
@@ -283,7 +254,6 @@ jobs:
       machine:
         image: circleci/classic:latest
       environment:
-        GOBIN: /home/circleci/.go_workspace/bin
         GOPATH: /home/circleci/.go_workspace/
         GOOS: linux
         GOARCH: amd64

.pending/breaking/gaia/4027-gaiad-and-gaiac

@@ -0,0 +1,2 @@
+#4027 gaiad and gaiacli version commands do not return the checksum of the go.sum file shipped along with the source release tarball.
+Go modules feature guarantees dependencies reproducibility and as long as binaries are built via the Makefile shipped with the sources, no dependendencies can break such guarantee.

.pending/breaking/sdk/4262-GoSumHash-is-no

@@ -0,0 +1 @@
+#4262 GoSumHash is no longer returned by the version command.

Dockerfile

@@ -5,7 +5,7 @@
 FROM golang:alpine AS build-env
 
 # Set up dependencies
-ENV PACKAGES curl make git libc-dev bash gcc linux-headers eudev-dev
+ENV PACKAGES curl make git libc-dev bash gcc linux-headers eudev-dev python
 
 # Set working directory for the build
 WORKDIR /go/src/github.com/cosmos/cosmos-sdk

Makefile

@@ -5,8 +5,7 @@ PACKAGES_SIMTEST=$(shell go list ./... | grep '/simulation')
 VERSION := $(shell echo $(shell git describe --tags) | sed 's/^v//')
 COMMIT := $(shell git log -1 --format='%H')
 LEDGER_ENABLED ?= true
-GOBIN ?= $(GOPATH)/bin
-GOSUM := $(shell which gosum)
+BINDIR ?= $(GOPATH)/bin
 
 export GO111MODULE = on
 
@@ -42,16 +41,17 @@ endif
 build_tags += $(BUILD_TAGS)
 build_tags := $(strip $(build_tags))
 
+whitespace :=
+whitespace += $(whitespace)
+comma := ,
+build_tags_comma_sep := $(subst $(whitespace),$(comma),$(build_tags))
+
 # process linker flags
 
 ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=gaia \
 	-X github.com/cosmos/cosmos-sdk/version.Version=$(VERSION) \
 	-X github.com/cosmos/cosmos-sdk/version.Commit=$(COMMIT) \
-  -X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags)"
-
-ifneq ($(GOSUM),)
-ldflags += -X github.com/cosmos/cosmos-sdk/version.GoSumHash=$(shell $(GOSUM) go.sum)
-endif
+  -X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags_comma_sep)"
 
 ifeq ($(WITH_CLEVELDB),yes)
   ldflags += -X github.com/cosmos/cosmos-sdk/types.DBBackend=cleveldb
@@ -119,6 +119,12 @@ draw_deps: tools
 clean:
 	rm -rf snapcraft-local.yaml build/
 
+distclean: clean
+	rm -rf \
+    gitian-build-darwin/ \
+    gitian-build-linux/ \
+    gitian-build-windows/ \
+    .gitian-builder-cache/
 
 ########################################
 ### Documentation
@@ -164,20 +170,20 @@ test_sim_gaia_fast:
 
 test_sim_gaia_import_export: runsim
 	@echo "Running Gaia import/export simulation. This may take several minutes..."
-	$(GOBIN)/runsim 50 5 TestGaiaImportExport
+	$(BINDIR)/runsim 50 5 TestGaiaImportExport
 
 test_sim_gaia_simulation_after_import: runsim
 	@echo "Running Gaia simulation-after-import. This may take several minutes..."
-	$(GOBIN)/runsim 50 5 TestGaiaSimulationAfterImport
+	$(BINDIR)/runsim 50 5 TestGaiaSimulationAfterImport
 
 test_sim_gaia_custom_genesis_multi_seed: runsim
 	@echo "Running multi-seed custom genesis simulation..."
 	@echo "By default, ${HOME}/.gaiad/config/genesis.json will be used."
-	$(GOBIN)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation
+	$(BINDIR)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation
 
 test_sim_gaia_multi_seed: runsim
 	@echo "Running multi-seed Gaia simulation. This may take awhile!"
-	$(GOBIN)/runsim 400 5 TestFullGaiaSimulation
+	$(BINDIR)/runsim 400 5 TestFullGaiaSimulation
 
 test_sim_benchmark_invariants:
 	@echo "Running simulation invariant benchmarks..."
@@ -186,8 +192,8 @@ test_sim_benchmark_invariants:
 	-SimulationCommit=true -SimulationSeed=57 -v -timeout 24h
 
 # Don't move it into tools - this will be gone once gaia has moved into the new repo
-runsim: $(GOBIN)/runsim
-$(GOBIN)/runsim: cmd/gaia/contrib/runsim/main.go
+runsim: $(BINDIR)/runsim
+$(BINDIR)/runsim: cmd/gaia/contrib/runsim/main.go
 	go install github.com/cosmos/cosmos-sdk/cmd/gaia/contrib/runsim
 
 SIM_NUM_BLOCKS ?= 500

cmd/gaia/Makefile

@@ -5,7 +5,6 @@ VERSION := $(shell echo $(shell git describe --tags) | sed 's/^v//')
 COMMIT := $(shell git log -1 --format='%H')
 LEDGER_ENABLED ?= true
 GOBIN ?= $(GOPATH)/bin
-GOSUM := $(shell which gosum)
 
 export GO111MODULE = on
 
@@ -41,16 +40,17 @@ endif
 build_tags += $(BUILD_TAGS)
 build_tags := $(strip $(build_tags))
 
+whitespace :=
+whitespace += $(whitespace)
+comma := ,
+build_tags_comma_sep := $(subst $(whitespace),$(comma),$(build_tags))
+
 # process linker flags
 
 ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=gaia \
 	-X github.com/cosmos/cosmos-sdk/version.Version=$(VERSION) \
 	-X github.com/cosmos/cosmos-sdk/version.Commit=$(COMMIT) \
-  -X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags)"
-
-ifneq ($(GOSUM),)
-ldflags += -X github.com/cosmos/cosmos-sdk/version.GoSumHash=$(shell $(GOSUM) ../../go.sum)
-endif
+  -X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags_comma_sep)"
 
 ifeq ($(WITH_CLEVELDB),yes)
   ldflags += -X github.com/cosmos/cosmos-sdk/types.DBBackend=cleveldb