Skip to content

Commit

Permalink
feat(.circleci): auto-publish artifacts when new tags are pushed
Browse files Browse the repository at this point in the history
Signed-off-by: Taiga Nakayama <dora@dora-gt.jp>
  • Loading branch information
dora-gt committed Nov 11, 2019
1 parent 0f13e34 commit b0251d4
Show file tree
Hide file tree
Showing 7 changed files with 502 additions and 63 deletions.
311 changes: 249 additions & 62 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,44 +3,42 @@ jobs:
build:
docker:
- image: circleci/rust
environment:
CARGO_HOME: /home/circleci/.cargo
resource_class: medium+
environment:
# setting BASH_ENV to custom file because .bashrc of circleci/rust doesn't work with
# non-interactive shell.
BASH_ENV: /home/circleci/.bash_circleci
CARGO_HOME: /home/circleci/.cargo
steps:
- checkout
- restore_cache:
keys:
- cargo-cache-1-{{ checksum "Cargo.toml" }}-{{ .Branch }}
- cargo-cache-1-{{ checksum "Cargo.toml" }}
- cargo-cache-1-
- run:
name: Install Cargo Extensions
name: Install Dependencies
command: |
# cargo-audit started requiring libcurl3
echo "deb http://security.ubuntu.com/ubuntu xenial-security main" | sudo tee -a /etc/apt/sources.list
sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 3B4FE6ACC0B21F32
# install system dependeicies
sudo apt-get update
sudo apt-get install libcurl3 -y
# get libcurl to a place where it won't get overwritten
sudo cp /usr/lib/x86_64-linux-gnu/libcurl.so.3 /usr/lib
sudo apt-get install curl -y
cargo install --quiet cargo-audit || true # cargo-kcov
rustup component add rustfmt clippy || true
sudo apt-get install -y redis-server redis-tools libssl-dev
# install rust components
cargo install cargo-audit
rustup component add clippy
# install nvm
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
export NVM_DIR="/home/circleci/.nvm"
source $NVM_DIR/nvm.sh
nvm install "v11.15.0"
# install yarn & components (conventional-changelog-cli, ilp-settlement-xrp)
curl -o- -L https://yarnpkg.com/install.sh | bash
export PATH="/home/circleci/.yarn/bin:/home/circleci/.config/yarn/global/node_modules/.bin:$PATH"
yarn global add ganache-cli ilp-settlement-xrp
# env
echo 'export NVM_DIR="/home/circleci/.nvm"' >> ${BASH_ENV}
echo 'source $NVM_DIR/nvm.sh' >> ${BASH_ENV}
echo "export PATH=/home/circleci/.cargo/bin:$PATH" >> ${BASH_ENV}
- run:
name: Install Redis
command: |
sudo apt-get update
sudo apt-get install redis-server
redis-server --version
- run:
name: Install Node.js, Ganache CLI, and XRP Engine
command: |
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
source ~/.nvm/nvm.sh
nvm install node
npm i -g ganache-cli ilp-settlement-xrp
- run:
name: Reduce codegen units
name: Reduce codegen Units
# If we don't include this, the linker runs out of memory when building
# the project on CI. We don't include this normally though because
# it should be able to build with more units on other machines
Expand All @@ -54,43 +52,232 @@ jobs:
# do not run for more than 10 minutes under any circumstances
# (We have had issues with bugs causing the tests to "run"
# for 5 hours, wasting a ton of compute credits)
command: |
source ~/.nvm/nvm.sh
timeout 10m cargo test --all --all-features
command: timeout 10m cargo test --all --all-features
environment:
RUST_BACKTRACE: "1"
- RUST_LOG: "interledger=trace"
- RUST_BACKTRACE: "full"
- run:
name: Check Style
command: |
cargo fmt --all -- --check
cargo clippy --all-targets --all-features -- -D warnings
- run:
name: Audit Dependencies
command: LD_PRELOAD=/usr/lib/libcurl.so.3 cargo audit
# - run:
# name: Install kcov
# command: >-
# kcov --version ||
# (sudo apt-get install cmake g++ pkg-config jq libcurl4-openssl-dev libelf-dev libdw-dev binutils-dev libiberty-dev
# && cargo kcov --print-install-kcov-sh | sh)
# - run:
# name: Generate Coverage Report
# # Enable sudo to avoid the error: "Can't set personality: Operation not permitted"
# # See discussion in https://github.com/travis-ci/travis-ci/issues/9061
# sudo: required
# command:
# cargo kcov --all --verbose
# # -- --verify
# # --exclude-region="kcov-ignore-start:kcov-ignore-end"
# # --exclude-pattern="$CARGO_HOME,**/tests/**"
# # --exclude-line="^(\)*\}*,*)$"
# environment:
# # Configure the redis tests to use a unix socket instead of TCP
# REDISRS_SERVER_TYPE: unix
# - run:
# name: Upload Code Coverage
# command: "bash <(curl -s https://codecov.io/bash)"
- save_cache:
key: cargo-cache-1-{{ checksum "Cargo.toml" }}-{{ .Branch }}
command: cargo audit
update-docker-images:
docker:
- image: circleci/rust
resource_class: medium+
steps:
- setup_remote_docker:
docker_layer_caching: true
- checkout
- run:
# builds release profile images to be released
name: Build Docker Images
command: |
export DOCKER_IMAGE_TAG=$(./.circleci/release/get_docker_image_tag.sh interledger-settlement-engines ${CIRCLE_TAG})
docker/docker-build.sh settlement-engines
environment:
PROFILE: "release"
- run:
name: Push to DockerHub
command: |
echo $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
DOCKER_IMAGE_TAG=$(./.circleci/release/get_docker_image_tag.sh interledger-settlement-engines ${CIRCLE_TAG})
echo "Pushing docker image of tag: interledgerrs/settlement-engines:${DOCKER_IMAGE_TAG}"
docker push interledgerrs/settlement-engines:${DOCKER_IMAGE_TAG}
build-release-binary-linux:
docker:
- image: circleci/rust
resource_class: medium+
steps:
- setup_remote_docker:
docker_layer_caching: true
- checkout
- run:
# Builds release profile binaries
# The artifact is stored in release-builds directory, which is persistent
name: Build binary
command: |
crate_name=$(./.circleci/release/get_crate_name.sh ${CIRCLE_TAG})
printf "Building crate_name: %s, tag: %s\n" "${crate_name}" "${CIRCLE_TAG}"
mkdir -p /tmp/release-builds/x86_64-unknown-linux-musl
./.circleci/release/build_musl_binary.sh ${crate_name} /tmp/release-builds/x86_64-unknown-linux-musl/${crate_name}
- persist_to_workspace:
root: /tmp/release-builds
paths:
- x86_64-unknown-linux-musl/*
- store_artifacts:
path: /tmp/release-builds/x86_64-unknown-linux-musl
destination: release-builds/x86_64-unknown-linux-musl
build-release-binary-darwin:
macos:
xcode: "10.2.0"
resource_class: medium+
steps:
- checkout
- run:
name: Install Rust
command: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.38.0
- run:
# Builds release profile binaries
# The artifact is stored in release-builds directory, which is persistent
name: Build binary
command: |
crate_name=$(./.circleci/release/get_crate_name.sh ${CIRCLE_TAG})
printf "Building crate_name: %s, tag: %s\n" "${crate_name}" "${CIRCLE_TAG}"
cargo build --release --features "ethereum" --package "${crate_name}" --bin "${crate_name}" --target x86_64-apple-darwin
mkdir -p /tmp/release-builds/x86_64-apple-darwin
cp target/x86_64-apple-darwin/release/${crate_name} /tmp/release-builds/x86_64-apple-darwin/${crate_name}
- persist_to_workspace:
root: /tmp/release-builds
paths:
- /home/circleci/.cargo
- x86_64-apple-darwin/*
- store_artifacts:
path: /tmp/release-builds/x86_64-apple-darwin
destination: release-builds/x86_64-apple-darwin
release-binaries:
docker:
- image: circleci/rust
environment:
BASH_ENV: /home/circleci/.bash_circleci
CARGO_HOME: /home/circleci/.cargo
steps:
- attach_workspace:
at: /tmp/release-builds
- checkout
- run:
name: Install Dependencies
command: |
# install system dependeicies
sudo apt-get update
sudo apt-get install -y curl jq
# install nvm
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
export NVM_DIR="/home/circleci/.nvm"
source $NVM_DIR/nvm.sh
nvm install "v11.15.0"
# install yarn & components (conventional-changelog-cli)
curl -o- -L https://yarnpkg.com/install.sh | bash
export PATH="/home/circleci/.yarn/bin:/home/circleci/.config/yarn/global/node_modules/.bin:$PATH"
yarn global add conventional-changelog-cli
# env
echo 'export NVM_DIR="/home/circleci/.nvm"' >> ${BASH_ENV}
echo 'source $NVM_DIR/nvm.sh' >> ${BASH_ENV}
echo "export PATH=/home/circleci/.cargo/bin:$PATH" >> ${BASH_ENV}
- run:
name: Release Binaries
command: |
mkdir -p logs
crate_name=$(./.circleci/release/get_crate_name.sh ${CIRCLE_TAG})
# linux musl
base_dir=/tmp/release-builds/x86_64-unknown-linux-musl
pushd ${base_dir}
musl_zip="${base_dir}/${crate_name}-x86_64-unknown-linux-musl.zip"
musl_tar_gz="${base_dir}/${crate_name}-x86_64-unknown-linux-musl.tar.gz"
zip $(basename ${musl_zip}) ${crate_name}
tar -zcvf $(basename ${musl_tar_gz}) ${crate_name}
popd
# macOS
base_dir=/tmp/release-builds/x86_64-apple-darwin
pushd ${base_dir}
darwin_zip="${base_dir}/${crate_name}-x86_64-apple-darwin.zip"
darwin_tar_gz="${base_dir}/${crate_name}-x86_64-apple-darwin.tar.gz"
zip $(basename ${darwin_zip}) ${crate_name}
tar -zcvf $(basename ${darwin_tar_gz}) ${crate_name}
popd
# Auto-generate a release note
release_note=logs/release_note.txt
conventional-changelog -p angular > ${release_note}
# tag release
./.circleci/release/github_release.sh \
${CIRCLE_TAG} \
${CIRCLE_TAG} \
${release_note} \
${musl_zip} \
${musl_tar_gz} \
${darwin_zip} \
${darwin_tar_gz}
# latest release, only if on the master branch
# tag the current commit as (crate_name)-latest, and release it
if [ "${CIRCLE_TAG}" == "master" ]; then
tag_name="${crate_name}-latest"
LOG_DIR=logs/${tag_name} ./.circleci/release/github_tag.sh \
${tag_name} \
"The latest release of ${crate_name}." \
"$(git rev-parse HEAD)" \
"commit"
./.circleci/release/github_release.sh \
${tag_name} \
${tag_name} \
${release_note} \
${musl_zip} \
${musl_tar_gz} \
${darwin_zip} \
${darwin_tar_gz}
fi
- store_artifacts:
path: logs
destination: release-binaries-logs

workflows:
version: 2
test-and-release:
jobs:
- build:
filters:
tags:
only: # we need this to kick builds when tags are given
- /.*/
- update-docker-images: # updates docker images on DockerHub, only if the branch is master or something is tagged
filters:
branches:
only: # master branch will be tagged as `latest` image
- master
tags:
only: # whatever tagged binary crates
- /^(interledger-settlement-engines-|v).*$/
requires:
- build
- build-release-binary-linux:
filters:
branches:
ignore: # no branch pushes are built
- /.*/
tags:
only: # whatever tagged binary crates
- /^(interledger-settlement-engines-|v).*$/
requires:
- build
- build-release-binary-darwin:
filters:
branches:
ignore: # no branch pushes are built
- /.*/
tags:
only: # whatever tagged binary crates
- /^(interledger-settlement-engines-|v).*$/
requires:
- build
- release-binaries:
filters:
branches:
ignore: # no branch pushes are built
- /.*/
tags:
only: # whatever tagged binary crates
- /^(interledger-settlement-engines-|v).*$/
requires:
- build-release-binary-linux
- build-release-binary-darwin
35 changes: 35 additions & 0 deletions .circleci/release/build_musl_binary.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#!/usr/bin/env bash

# Build musl binary inside a Docker container and copy it to local disk.
# This script requires docker command.
# `crate_name` ($1) is expected to be the same as bin name.
#
# 1) Spin up a new container of "clux/muslrust:stable"
# 2) Copy source code etc. to the container
# 3) Build a musl binary of the specified crate ($1) inside the container
# 4) Copy the built binary to local disk space ($2)

crate_name=$1
artifacts_path=$2
docker_image_name="clux/muslrust:stable"

if [ -z "${crate_name}" ]; then
printf "%s\n" "crate_name is required."
exit 1
fi
if [ -z "${artifacts_path}" ]; then
printf "%s\n" "artifacts_path is required."
exit 1
fi

docker run -dt --name builder "${docker_image_name}"

docker cp ./Cargo.toml builder:/usr/src/Cargo.toml
docker cp ./Cargo.lock builder:/usr/src/Cargo.lock
docker cp ./src builder:/usr/src/src

# "--workdir" requires API version 1.35, but the Docker daemon API version of CircleCI is 1.32
docker exec builder "/bin/bash" "-c" "cd /usr/src && cargo build --release --features \"ethereum\" --package \"${crate_name}\" --bin \"${crate_name}\" --target x86_64-unknown-linux-musl"
docker cp "builder:/usr/src/target/x86_64-unknown-linux-musl/release/${crate_name}" "${artifacts_path}"

docker stop builder
13 changes: 13 additions & 0 deletions .circleci/release/get_crate_name.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/usr/bin/env bash

# Returns a crate name to build from a git tag name.
# The tag name is assumed to be output by `cargo release` or be tagged manually.

# The regex means "(something)-(semantic version)"
if [[ $1 =~ ^(.*)-v([0-9]+)\.([0-9]+)\.([0-9]+)(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?(\+[0-9A-Za-z-]+)?$ ]]; then
echo ${BASH_REMATCH[1]}
elif [[ $1 =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?(\+[0-9A-Za-z-]+)?$ ]]; then
echo "interledger-settlement-engines"
elif [[ $1 =~ ^interledger-settlement-engines-.*$ ]]; then
echo "interledger-settlement-engines"
fi
Loading

0 comments on commit b0251d4

Please sign in to comment.