Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: release build scripts #1286

Merged
merged 16 commits into from
Apr 19, 2022
8 changes: 4 additions & 4 deletions .build.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

#!/bin/bash

set -ue
Expand All @@ -12,7 +11,7 @@ set -ue
# - DEBUG

# Source builder's functions library
. /usr/local/share/tendermint/buildlib.sh
. /usr/local/share/osmosis/buildlib.sh

# These variables are now available
# - BASEDIR
Expand All @@ -25,14 +24,15 @@ for platform in ${TARGET_PLATFORMS} ; do
# cases except when the target platform is 'windows'.
setup_build_env_for_platform "${platform}"

make clean
echo Building for $(go env GOOS)/$(go env GOARCH) >&2
GOROOT_FINAL="$(go env GOROOT)" \
make build \
LDFLAGS=-buildid=${VERSION} \
VERSION=${VERSION} \
COMMIT=${COMMIT} \
LEDGER_ENABLED=${LEDGER_ENABLED}
LEDGER_ENABLED=${LEDGER_ENABLED} \
LINK_STATICALLY=true \
BUILD_TAGS=muslc
mv ./build/${APP}${OS_FILE_EXT} ${OUTDIR}/${APP}-${VERSION}-$(go env GOOS)-$(go env GOARCH)${OS_FILE_EXT}

# This function restore the build environment variables to their
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* [#1107](https://github.com/osmosis-labs/osmosis/pull/1107) Update to wasmvm v0.24.0, re-enabling building on M1 macs!

### Minor improvements & Bug Fixes
* [#1286](https://github.com/osmosis-labs/osmosis/pull/1286) Fix release build scripts.
* [#1203](https://github.com/osmosis-labs/osmosis/pull/1203) cleanup Makefile and ci workflows
* [#1177](https://github.com/osmosis-labs/osmosis/pull/1177) upgrade to go 1.18
* [#1193](https://github.com/osmosis-labs/osmosis/pull/1193) Setup e2e tests on a single chain; add balances query test
Expand Down
17 changes: 10 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ endif
ifeq (,$(findstring nostrip,$(OSMOSIS_BUILD_OPTIONS)))
ldflags += -w -s
endif
ifeq ($(LINK_STATICALLY),true)
ldflags += -linkmode=external -extldflags "-L/usr/local/lib/ -lwasmvm_muslc -Wl,-z,muldefs -static"
endif
ldflags += $(LDFLAGS)
ldflags := $(strip $(ldflags))

Expand Down Expand Up @@ -96,12 +99,12 @@ $(BUILDDIR)/:
build-reproducible: go.sum
$(DOCKER) rm latest-build || true
$(DOCKER) run --volume=$(CURDIR):/sources:ro \
--env TARGET_PLATFORMS='linux/amd64 darwin/amd64 linux/arm64 windows/amd64' \
--env APP=osmosisd \
--env VERSION=$(VERSION) \
--env COMMIT=$(COMMIT) \
--env LEDGER_ENABLED=$(LEDGER_ENABLED) \
--name latest-build cosmossdk/rbuilder:latest
--env TARGET_PLATFORMS='linux/amd64' \
--env APP=osmosisd \
--env VERSION=$(VERSION) \
--env COMMIT=$(COMMIT) \
--env LEDGER_ENABLED=$(LEDGER_ENABLED) \
--name latest-build osmolabs/rbuilder:latest
$(DOCKER) cp -a latest-build:/home/builder/artifacts/ $(CURDIR)/

build-linux: go.sum
Expand All @@ -125,7 +128,7 @@ draw-deps:
@goviz -i ./cmd/osmosisd -d 2 | dot -Tpng -o dependency-graph.png

clean:
rm -rf $(BUILDDIR)/ artifacts/
rm -rf $(CURDIR)/artifacts/

distclean: clean
rm -rf vendor/
Expand Down
2 changes: 1 addition & 1 deletion contrib/images/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ simd-env:
docker build --build-arg UID=$(shell id -u) --build-arg GID=$(shell id -g) --tag cosmossdk/simd-env simd-env

rbuilder:
docker build --tag cosmossdk/rbuilder rbuilder
docker build --tag osmolabs/rbuilder rbuilder

.PHONY: all simd-env rbuilder
35 changes: 28 additions & 7 deletions contrib/images/rbuilder/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,16 +1,37 @@
FROM golang:1.15.2-alpine3.12

FROM golang:1.18-bullseye

ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get --no-install-recommends -y install \
pciutils build-essential git wget \
lsb-release dpkg-dev curl bsdmainutils fakeroot
RUN mkdir -p /usr/local/share/cosmos-sdk/
COPY buildlib.sh /usr/local/share/cosmos-sdk/
RUN mkdir -p /usr/local/share/osmosis/

# Deploy the shell functions library.
COPY buildlib.sh /usr/local/share/osmosis/

# Create the 'builder' user.
RUN useradd -ms /bin/bash -U builder
ARG APP
ARG DEBUG
ENV APP ${APP:-cosmos-sdk}
ENV DEBUG ${DEBUG} VERSION unknown COMMIT unknown LEDGER_ENABLE true
ARG TARGET_PLATFORMS
ENV APP ${APP:-app}
ENV DEBUG ${DEBUG}
ENV VERSION unknown
ENV COMMIT unknown
ENV LEDGER_ENABLE true
ENV TARGET_PLATFORMS ${TARGET_PLATFORMS:-linux/amd64}
ENV BUILD_SCRIPT ${BUILD_SCRIPT:-/sources/.build.sh}

# From https://github.com/CosmWasm/wasmd/blob/master/Dockerfile
# For more details see https://github.com/CosmWasm/wasmvm#builds-of-libwasmvm
ADD https://github.com/CosmWasm/wasmvm/releases/download/v1.0.0-beta7/libwasmvm_muslc.a /usr/local/lib/libwasmvm_muslc.a
RUN chown builder /usr/local/lib/libwasmvm_muslc.a
RUN sha256sum /usr/local/lib/libwasmvm_muslc.a | grep d0152067a5609bfdfb3f0d5d6c0f2760f79d5f2cd7fd8513cafa9932d22eb350

# Drop root privileges.
USER builder:builder
WORKDIR /sources
VOLUME [ "/sources" ]
ENTRYPOINT [ "/sources/build.sh" ]

# Run the application's build.sh.
ENTRYPOINT [ "/bin/bash", "-c", "${BUILD_SCRIPT}" ]
44 changes: 44 additions & 0 deletions contrib/images/rbuilder/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Reproducible Build System

This image and scripts are meant to provide a minimal deterministic
build system for the Osmosis application.

It was created by referencing Tendermint's [implementation](https://github.com/tendermint/images/blob/cf0d1a9f3731e30540bbfa36a36d13e4dcccf5eb/rbuilder/README.md)

# Requirements And Usage

The Osmosis repository must include a`.build.sh` executable file
in the root folder meant to drive the build process.

The build's outputs are produced in the top-level `artifacts` directory.

## Building the Image Locally

```
cd ./contrib/images

make rbuilder
```

This creates the `rbuilder` image. To run a container of this image locally and build the binaries:
```
cd <osmosis root>

make build-reproducible
```

This spins up an `rbuilder` container with a volume installed to the
root of the repository. This way, the builder has access to the `.build.sh`file and is
able to execute it.

## Wasmvm dependency

Currently, only `linux/amd64` is supported. Adding more support is blocked by our dependency on wasmvm.

The support of some platforms is already added in new versions of wasmvm.
Follow the release log for more detaisl when updating our builder:
https://github.com/CosmWasm/wasmvm/releases

Once wasmvm is upgraded, more platforms may be built by changing
`TARGET_PLATFORMS` environment variable in `build-reproducible`
Makefile step.
39 changes: 20 additions & 19 deletions contrib/images/rbuilder/buildlib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,29 +20,29 @@ f_setup_pristine_src_dir() {
go mod download
}

f_build_archs() {
local l_os

l_os=$1

case "${l_os}" in
darwin | windows)
echo 'amd64'
;;
linux)
echo 'amd64 arm64'
;;
*)
echo "unknown OS -- ${l_os}" >&2
return 1
esac
f_exe_file_ext() {
[ $(go env GOOS) = windows ] && printf '%s' '.exe' || printf ''
}

f_binary_file_ext() {
[ $1 = windows ] && printf '%s' '.exe' || printf ''
setup_build_env_for_platform() {
local l_platform=$1

g_old_GOOS="$(go env GOOS)"
g_old_GOARCH="$(go env GOARCH)"
g_old_OS_FILE_EXT="${OS_FILE_EXT}"

go env -w GOOS="${l_platform%%/*}"
go env -w GOARCH="${l_platform##*/}"
OS_FILE_EXT="$(f_exe_file_ext)"
}

restore_build_env() {
go env -w GOOS="${g_old_GOOS}"
go env -w GOARCH="${g_old_GOARCH}"
OS_FILE_EXT="${g_old_OS_FILE_EXT}"
}

f_generate_build_report() {
generate_build_report() {
local l_tempfile

l_tempfile="$(mktemp)"
Expand All @@ -63,6 +63,7 @@ EOF

[ "x${DEBUG}" = "x" ] || set -x

OS_FILE_EXT=''
BASEDIR="$(mktemp -d)"
OUTDIR=$HOME/artifacts
rm -rfv ${OUTDIR}/
Expand Down