R4R: Infrastructure for reproducible builds

.circleci/config.yml

@@ -4,8 +4,6 @@ defaults: &linux_defaults
   working_directory: /go/src/github.com/cosmos/cosmos-sdk
   docker:
     - image: circleci/golang:1.12.4
-  environment:
-    GOBIN: /tmp/workspace/bin
 
 
 ############
@@ -25,8 +23,7 @@ set_macos_env: &macos_env
     command: |
       echo 'export PATH=$PATH:$HOME/go/bin' >> $BASH_ENV
       echo 'export GOPATH=$HOME/project' >> $BASH_ENV
-      echo 'export GOBIN=$GOPATH/bin' >> $BASH_ENV
-      echo 'export PATH=$PATH:$HOME/go/bin:$GOBIN' >> $BASH_ENV
+      echo 'export PATH=$PATH:$HOME/go/bin:$GOPATH/bin' >> $BASH_ENV
       echo 'export GO111MODULE=on'
 
 ############
@@ -40,12 +37,6 @@ docs_update: &docs_deploy
   environment:
     AWS_REGION: us-east-1
 
-deps: &dependencies
-  run:
-    name: dependencies
-    command: |
-      export PATH="$GOBIN:$PATH"
-
 jobs:
   setup_dependencies:
     <<: *linux_defaults
@@ -59,13 +50,11 @@ jobs:
       - run:
           name: tools
           command: |
-            export PATH="$GOBIN:$PATH"
-            make tools
-      - *dependencies
+            make tools TOOLS_DESTDIR=/tmp/workspace/bin
       - run:
           name: binaries
           command: |
-            export PATH="$GOBIN:$PATH"
+            export PATH=/tmp/workspace/bin:$PATH
             make go-mod-cache
             make install
       - save_cache:
@@ -85,14 +74,13 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Lint source
           command: |
-            export PATH="$GOBIN:$PATH"
+            export PATH=/tmp/workspace/bin:$PATH
             make ci-lint
 
   integration_tests:
@@ -102,14 +90,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test cli
           command: |
-            export PATH="$GOBIN:$PATH"
             make test_cli
 
   test_sim_gaia_nondeterminism:
@@ -119,14 +105,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test individual module simulations
           command: |
-            export PATH="$GOBIN:$PATH"
             make test_sim_gaia_nondeterminism
 
   test_sim_gaia_fast:
@@ -136,14 +120,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test full Gaia simulation
           command: |
-            export PATH="$GOBIN:$PATH"
             make test_sim_gaia_fast
 
   test_sim_gaia_import_export:
@@ -153,14 +135,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test Gaia import/export simulation
           command: |
-            export PATH="$GOBIN:$PATH"
             make test_sim_gaia_import_export
 
   test_sim_gaia_simulation_after_import:
@@ -170,14 +150,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test Gaia import/export simulation
           command: |
-            export PATH="$GOBIN:$PATH"
             make test_sim_gaia_simulation_after_import
 
   test_sim_gaia_multi_seed_long:
@@ -187,14 +165,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test multi-seed Gaia simulation long
           command: |
-            export PATH="$GOBIN:$PATH"
             export GO111MODULE=on
             make runsim
             runsim 500 50 TestFullGaiaSimulation
@@ -206,14 +182,12 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Test multi-seed Gaia simulation short
           command: |
-            export PATH="$GOBIN:$PATH"
             export GO111MODULE=on
             make runsim
             runsim 50 10 TestFullGaiaSimulation
@@ -225,15 +199,13 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - run: mkdir -p /tmp/logs
       - restore_cache:
           keys:
             - go-mod-v1-{{ checksum "go.sum" }}
       - run:
           name: Run tests
           command: |
-            export PATH="$GOBIN:$PATH"
             export VERSION="$(git describe --tags --long | sed 's/v\(.*\)/\1/')"
             export GO111MODULE=on
             for pkg in $(go list ./... | grep -v github.com/cosmos/cosmos-sdk/cmd/gaia/cli_test | grep -v '/simulation' | circleci tests split --split-by=timings); do
@@ -254,7 +226,6 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - checkout
-      - *dependencies
       - run:
           name: gather
           command: |
@@ -283,7 +254,6 @@ jobs:
       machine:
         image: circleci/classic:latest
       environment:
-        GOBIN: /home/circleci/.go_workspace/bin
         GOPATH: /home/circleci/.go_workspace/
         GOOS: linux
         GOARCH: amd64

Dockerfile

@@ -5,7 +5,7 @@
 FROM golang:alpine AS build-env
 
 # Set up dependencies
-ENV PACKAGES curl make git libc-dev bash gcc linux-headers eudev-dev
+ENV PACKAGES curl make git libc-dev bash gcc linux-headers eudev-dev python
 
 # Set working directory for the build
 WORKDIR /go/src/github.com/cosmos/cosmos-sdk

Makefile

@@ -5,8 +5,7 @@ PACKAGES_SIMTEST=$(shell go list ./... | grep '/simulation')
 VERSION := $(shell echo $(shell git describe --tags) | sed 's/^v//')
 COMMIT := $(shell git log -1 --format='%H')
 LEDGER_ENABLED ?= true
-GOBIN ?= $(GOPATH)/bin
-GOSUM := $(shell which gosum)
+BINDIR ?= $(GOPATH)/bin
 
 export GO111MODULE = on
 
@@ -49,10 +48,6 @@ ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=gaia \
 	-X github.com/cosmos/cosmos-sdk/version.Commit=$(COMMIT) \
   -X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags)"
 
-ifneq ($(GOSUM),)
-ldflags += -X github.com/cosmos/cosmos-sdk/version.GoSumHash=$(shell $(GOSUM) go.sum)
-endif
-
 ifeq ($(WITH_CLEVELDB),yes)
   ldflags += -X github.com/cosmos/cosmos-sdk/types.DBBackend=cleveldb
 endif
@@ -166,20 +161,20 @@ test_sim_gaia_fast:
 
 test_sim_gaia_import_export: runsim
 	@echo "Running Gaia import/export simulation. This may take several minutes..."
-	$(GOBIN)/runsim 50 5 TestGaiaImportExport
+	$(BINDIR)/runsim 50 5 TestGaiaImportExport
 
 test_sim_gaia_simulation_after_import: runsim
 	@echo "Running Gaia simulation-after-import. This may take several minutes..."
-	$(GOBIN)/runsim 50 5 TestGaiaSimulationAfterImport
+	$(BINDIR)/runsim 50 5 TestGaiaSimulationAfterImport
 
 test_sim_gaia_custom_genesis_multi_seed: runsim
 	@echo "Running multi-seed custom genesis simulation..."
 	@echo "By default, ${HOME}/.gaiad/config/genesis.json will be used."
-	$(GOBIN)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation
+	$(BINDIR)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation
 
 test_sim_gaia_multi_seed: runsim
 	@echo "Running multi-seed Gaia simulation. This may take awhile!"
-	$(GOBIN)/runsim 400 5 TestFullGaiaSimulation
+	$(BINDIR)/runsim 400 5 TestFullGaiaSimulation
 
 test_sim_benchmark_invariants:
 	@echo "Running simulation invariant benchmarks..."
@@ -188,8 +183,8 @@ test_sim_benchmark_invariants:
 	-SimulationCommit=true -SimulationSeed=57 -v -timeout 24h
 
 # Don't move it into tools - this will be gone once gaia has moved into the new repo
-runsim: $(GOBIN)/runsim
-$(GOBIN)/runsim: cmd/gaia/contrib/runsim/main.go
+runsim: $(BINDIR)/runsim
+$(BINDIR)/runsim: cmd/gaia/contrib/runsim/main.go
 	go install github.com/cosmos/cosmos-sdk/cmd/gaia/contrib/runsim
 
 SIM_NUM_BLOCKS ?= 500

cmd/gaia/Makefile

@@ -5,7 +5,6 @@ VERSION := $(shell echo $(shell git describe --tags) | sed 's/^v//')
 COMMIT := $(shell git log -1 --format='%H')
 LEDGER_ENABLED ?= true
 GOBIN ?= $(GOPATH)/bin
-GOSUM := $(shell which gosum)
 
 export GO111MODULE = on
 
@@ -48,10 +47,6 @@ ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=gaia \
 	-X github.com/cosmos/cosmos-sdk/version.Commit=$(COMMIT) \
   -X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags)"
 
-ifneq ($(GOSUM),)
-ldflags += -X github.com/cosmos/cosmos-sdk/version.GoSumHash=$(shell $(GOSUM) ../../go.sum)
-endif
-
 ifeq ($(WITH_CLEVELDB),yes)
   ldflags += -X github.com/cosmos/cosmos-sdk/types.DBBackend=cleveldb
 endif

cmd/gaia/contrib/gitian-build.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -euo pipefail
+
+g_default_dirname="gitian-build-$(date +%Y-%m-%d-%H%M%S)"
+g_dirname=''
+
+f_main() {
+  local l_dirname \
+    l_workdir \
+    l_sdk \
+    l_commit \
+    l_platform \
+    l_result \
+    l_descriptor \
+    l_release_name
+
+  l_dirname=$1
+  l_platform=$2
+  l_sdk=$3
+  pushd ${l_sdk}
+  l_commit=$(git rev-parse HEAD)
+  popd
+
+  l_descriptor=$l_sdk/cmd/gaia/contrib/gitian-descriptors/gitian-${l_platform}.yml
+  [ -f ${l_descriptor} ]
+
+  l_workdir="$(pwd)/${l_dirname}"
+  mkdir ${l_workdir}/
+  echo "Download gitian" >&2
+  git clone https://github.com/devrandom/gitian-builder ${l_workdir}
+
+  echo "Prepare gitian-target docker image" >&2
+  f_prep_docker_image "${l_workdir}"
+
+  echo "Download Go" >&2
+  f_download_go "${l_workdir}"
+
+  echo "Start the build" >&2
+  f_build "${l_workdir}" "${l_sdk}" "${l_commit}" "${l_platform}"
+  echo "You may find the result in $(echo ${l_workdir}/result/*.yml))" >&2
+
+  l_release_name="$(sed -n 's/^name: \"\(.\+\)\"$/\1/p' ${l_descriptor})"
+  [ -n ${l_release_name} ]
+  echo "You can now sign the build with the following command:" >&2
+  echo "cd ${l_workdir} ; bin/gsign -p 'gpg --detach-sign --armor' -s GPG_IDENTITY --release=${l_release_name} ${l_descriptor}" >&2
+  return 0
+}
+
+f_prep_docker_image() {
+  local l_dir=$1
+
+  pushd ${l_dir}
+  bin/make-base-vm --docker --suite bionic --arch amd64
+  popd
+}
+
+f_download_go() {
+  local l_dir l_gopkg
+
+  l_dir=$1
+
+  mkdir -p ${l_dir}/inputs
+  l_gopkg=go1.12.4.linux-amd64.tar.gz
+  curl -L https://dl.google.com/go/${l_gopkg} > ${l_dir}/inputs/${l_gopkg}
+}
+
+f_build() {
+  local l_gitian l_sdk l_platform l_descriptor
+
+  l_gitian=$1
+  l_sdk=$2
+  l_commit=$3
+  l_platform=$4
+  l_descriptor=$l_sdk/cmd/gaia/contrib/gitian-descriptors/gitian-${l_platform}.yml
+
+  [ -f ${l_descriptor} ]
+
+  cd ${l_gitian}
+  export USE_DOCKER=1
+  bin/gbuild $l_descriptor --commit cosmos-sdk=$l_commit
+  libexec/stop-target || echo "warning: couldn't stop target" >&2
+}
+
+f_help() {
+  cat >&2 <<EOF
+Usage: $(basename $0) [-h] GOOS GIT_REPO
+Launch a gitian build from the local clone of cosmos-sdk available at GIT_REPO.
+
+  Options:
+   -h               Display this help and exit
+   -d DIRNAME       Set working directory name
+EOF
+}
+
+while getopts ":d:h" opt; do
+  case "${opt}" in
+    h)  f_help ; exit 0 ;;
+    d)  g_dirname="${OPTARG}"
+  esac
+done
+
+shift "$((OPTIND-1))"
+
+f_main "${g_dirname:-${g_default_dirname}}" $1 $2