Update build_and_test.yml #180
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Build, Test, Clippy | |
on: | |
workflow_dispatch: | |
push: | |
branches: | |
- master | |
- 'sdk-v[0-9]+.[0-9]+.[0-9]+-*' | |
tags: | |
- 'v[0-9]+.[0-9]+.[0-9]+*' | |
pull_request: | |
branches: | |
- master | |
- 'sdk-v[0-9]+.[0-9]+.[0-9]+-*' | |
env: | |
CARGO_TERM_COLOR: always | |
LOG_DIR: logs | |
BUILD_CONTAINER_NAME: integritee_worker_enclave_test | |
jobs: | |
cancel_previous_runs: | |
name: Cancel Previous Runs | |
runs-on: ubuntu-latest | |
steps: | |
- uses: styfle/cancel-workflow-action@0.11.0 | |
with: | |
access_token: ${{ secrets.GITHUB_TOKEN }} | |
build-test: | |
runs-on: ${{ matrix.host }} | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- flavor_id: offchain-worker | |
mode: offchain-worker | |
host: ubuntu-latest | |
sgx_mode: SW | |
additional_features: dcap | |
- flavor_id: offchain-worker | |
mode: offchain-worker | |
host: ubuntu-latest | |
sgx_mode: HW | |
additional_features: dcap | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set env | |
run: | | |
fingerprint=$RANDOM | |
echo "FINGERPRINT=$fingerprint" >> $GITHUB_ENV | |
SGX_MODE_LOWERCASE=$(echo "${${{ matrix.sgx_mode }},,}") | |
echo "IMAGE_SUFFIX=$SGX_MODE_LOWERCASE-${{ matrix.flavor_id }}-${{ github.sha }}" >> $GITHUB_ENV | |
if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then | |
echo "DOCKER_DEVICES=--device=/dev/sgx/enclave --device=/dev/sgx/provision" >> $GITHUB_ENV | |
echo "DOCKER_VOLUMES=--volume /var/run/aesmd:/var/run/aesmd --volume /etc/sgx_default_qcnl.conf:/etc/sgx_default_qcnl.conf" >> $GITHUB_ENV | |
else | |
echo "DOCKER_DEVICES=" >> $GITHUB_ENV | |
echo "DOCKER_VOLUMES=" >> $GITHUB_ENV | |
fi | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v2 | |
with: | |
buildkitd-flags: --debug | |
driver: docker-container | |
- name: Build Worker | |
env: | |
DOCKER_BUILDKIT: 1 | |
run: > | |
docker build -t integritee-worker-${{ env.IMAGE_SUFFIX }} | |
--target deployed-worker | |
--build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg FINGERPRINT=${FINGERPRINT} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} --build-arg SGX_MODE=${{ matrix.sgx_mode }} | |
-f build.Dockerfile . | |
- run: docker images --all | |
- name: Test Enclave # cargo test is not supported in the enclave, see: https://github.com/apache/incubator-teaclave-sgx-sdk/issues/232 | |
if: ${{ matrix.sgx_mode != 'HW' }} # Change for OLI, can't run these tests in HW mode on Github Hosted Runners | |
run: docker run --rm ${{ env.DOCKER_DEVICES }} ${{ env.DOCKER_VOLUMES }} integritee-worker-${{ env.IMAGE_SUFFIX }} test --all | |
- name: Export worker image | |
run: | | |
docker image save integritee-worker-${{ env.IMAGE_SUFFIX }} | gzip > integritee-worker-${{ env.IMAGE_SUFFIX }}.tar.gz | |
- name: Upload worker image | |
uses: actions/upload-artifact@v3 | |
with: | |
name: integritee-worker-${{ env.IMAGE_SUFFIX }}.tar.gz | |
path: integritee-worker-${{ env.IMAGE_SUFFIX }}.tar.gz | |
- name: Create Enclave Digest File | |
run: | | |
mrenclave_hex=$(docker run integritee-worker-${{ env.IMAGE_SUFFIX }} mrenclave | grep -oP ':\s*\K[a-fA-F0-9]+') | |
echo "$mrenclave_hex" > mrenclave-${{ env.IMAGE_SUFFIX }}.hex | |
- name: Upload Enclave Digest File | |
uses: actions/upload-artifact@v3 | |
with: | |
name: mrenclave-${{ env.IMAGE_SUFFIX }}.hex | |
path: mrenclave-${{ env.IMAGE_SUFFIX }}.hex | |
- name: Delete images | |
run: | | |
if [[ "$(docker images -q integritee-worker-${{ env.IMAGE_SUFFIX }} 2> /dev/null)" != "" ]]; then | |
docker image rmi --force integritee-worker-${{ env.IMAGE_SUFFIX }} 2>/dev/null | |
fi | |
docker images --all | |
build-client: | |
runs-on: ${{ matrix.host }} | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- flavor_id: offchain-worker | |
mode: offchain-worker | |
host: ubuntu-latest | |
sgx_mode: SW | |
additional_features: dcap | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set env | |
run: | | |
fingerprint=$RANDOM | |
echo "FINGERPRINT=$fingerprint" >> $GITHUB_ENV | |
SGX_MODE_LOWERCASE=$(echo "${${{ matrix.sgx_mode }},,}") | |
echo "IMAGE_SUFFIX=$SGX_MODE_LOWERCASE-${{ matrix.flavor_id }}-${{ github.sha }}" >> $GITHUB_ENV | |
if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then | |
echo "DOCKER_DEVICES=--device=/dev/sgx/enclave --device=/dev/sgx/provision" >> $GITHUB_ENV | |
echo "DOCKER_VOLUMES=--volume /var/run/aesmd:/var/run/aesmd --volume /etc/sgx_default_qcnl.conf:/etc/sgx_default_qcnl.conf" >> $GITHUB_ENV | |
else | |
echo "DOCKER_DEVICES=" >> $GITHUB_ENV | |
echo "DOCKER_VOLUMES=" >> $GITHUB_ENV | |
fi | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v2 | |
with: | |
buildkitd-flags: --debug | |
driver: docker-container | |
- name: Build CLI client | |
env: | |
DOCKER_BUILDKIT: 1 | |
run: > | |
docker build -t integritee-cli-client-${{ env.IMAGE_SUFFIX }} | |
--target deployed-client | |
--build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} | |
-f build.Dockerfile . | |
- run: docker images --all | |
- name: Export client image | |
run: | | |
docker image save integritee-cli-client-${{ env.IMAGE_SUFFIX }} | gzip > integritee-cli-client-${{ env.IMAGE_SUFFIX }}.tar.gz | |
- name: Upload CLI client image | |
uses: actions/upload-artifact@v3 | |
with: | |
name: integritee-cli-client-${{ env.IMAGE_SUFFIX }}.tar.gz | |
path: integritee-cli-client-${{ env.IMAGE_SUFFIX }}.tar.gz | |
- name: Delete images | |
run: | | |
if [[ "$(docker images -q integritee-cli-client-${{ env.IMAGE_SUFFIX }} 2> /dev/null)" != "" ]]; then | |
docker image rmi --force integritee-cli-client-${{ env.IMAGE_SUFFIX }} 2>/dev/null | |
fi | |
docker images --all | |
code-quality: | |
runs-on: ubuntu-latest | |
container: "integritee/integritee-dev:0.2.2" | |
strategy: | |
fail-fast: false | |
matrix: | |
check: [ | |
# Workspace | |
cargo test --release, | |
# Worker | |
# Use release mode as the CI runs out of disk space otherwise. | |
cargo clippy --release -- -D warnings, | |
cargo clippy --release --features evm -- -D warnings, | |
cargo clippy --release --features sidechain -- -D warnings, | |
cargo clippy --release --features teeracle -- -D warnings, | |
cargo clippy --release --features offchain-worker -- -D warnings, | |
# Enclave | |
cd enclave-runtime && cargo clippy -- -D warnings, | |
cd enclave-runtime && cargo clippy --features evm -- -D warnings, | |
cd enclave-runtime && cargo clippy --features sidechain -- -D warnings, | |
cd enclave-runtime && cargo clippy --features teeracle -- -D warnings, | |
cd enclave-runtime && cargo clippy --features offchain-worker -- -D warnings, | |
# Fmt | |
cargo fmt --all -- --check, | |
cd enclave-runtime && cargo fmt --all -- --check, | |
] | |
steps: | |
- uses: actions/checkout@v3 | |
- name: init-rust-target | |
# Enclave is not in the same workspace | |
run: rustup show && cd enclave-runtime && rustup show | |
- uses: Swatinem/rust-cache@v2 | |
with: | |
key: ${{ matrix.check }} | |
- name: ${{ matrix.check }} | |
run: ${{ matrix.check }} | |
toml-fmt: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- name: init rust | |
run: rustup show | |
- name: Install taplo | |
run: cargo install taplo-cli --locked | |
- name: Cargo.toml fmt | |
run: taplo fmt --check | |
- name: Fail-fast; cancel other jobs | |
if: failure() | |
uses: andymckay/cancel-action@0.3 | |
integration-tests: | |
runs-on: ${{ matrix.host }} | |
if: ${{ always() }} | |
needs: [build-test, build-client] | |
env: | |
WORKER_IMAGE_TAG: integritee-worker:dev | |
CLIENT_IMAGE_TAG: integritee-cli:dev | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- test: M6 | |
flavor_id: offchain-worker | |
demo_name: demo-indirect-invocation | |
- test: BestEnergy | |
flavor_id: offchain-worker | |
demo_name: demo-energy-market | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set env | |
run: | | |
version=$RANDOM | |
SGX_MODE_LOWERCASE=$(echo "${${{ matrix.sgx_mode }},,}") | |
echo "IMAGE_SUFFIX=$SGX_MODE_LOWERCASE-${{ matrix.flavor_id }}-${{ github.sha }}" >> $GITHUB_ENV | |
echo "FLAVOR_ID=${{ matrix.flavor_id }}" >> $GITHUB_ENV | |
echo "PROJECT=${{ matrix.flavor_id }}-${{ matrix.demo_name }}" >> $GITHUB_ENV | |
echo "VERSION=dev.$version" >> $GITHUB_ENV | |
echo "WORKER_IMAGE_TAG=integritee-worker:dev.$version" >> $GITHUB_ENV | |
echo "INTEGRITEE_NODE=integritee-node:1.1.3.$version" >> $GITHUB_ENV | |
echo "CLIENT_IMAGE_TAG=integritee-cli:dev.$version" >> $GITHUB_ENV | |
if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then | |
echo "SGX_PROVISION=/dev/sgx/provision" >> $GITHUB_ENV | |
echo "SGX_ENCLAVE=/dev/sgx/enclave" >> $GITHUB_ENV | |
echo "AESMD=/var/run/aesmd" >> $GITHUB_ENV | |
echo "SGX_QCNL=/etc/sgx_default_qcnl.conf" >> $GITHUB_ENV | |
fi | |
echo "LOG_DIR=./logs-$version" >> $GITHUB_ENV | |
- name: Download Worker Image | |
uses: actions/download-artifact@v4 | |
with: | |
name: integritee-worker-${{ env.IMAGE_SUFFIX }}.tar.gz | |
path: . | |
- name: Download CLI client Image | |
uses: actions/download-artifact@v4 | |
with: | |
name: integritee-cli-client-${{ env.IMAGE_SUFFIX }}.tar.gz | |
path: . | |
- name: Load Worker & Client Images | |
env: | |
DOCKER_BUILDKIT: 1 | |
run: | | |
docker image load --input integritee-worker-${{ env.IMAGE_SUFFIX }}.tar.gz | |
docker image load --input integritee-cli-client-${{ env.IMAGE_SUFFIX }}.tar.gz | |
docker images --all | |
## | |
# Before tagging, delete the old "stuck" ones to be sure that the newly created ones are the latest | |
# Without if the docker image rmi throws an error if the image doesn't exist. | |
## | |
- name: Re-name Image Tags | |
run: | | |
if [[ "$(docker images -q ${{ env.WORKER_IMAGE_TAG }} 2> /dev/null)" == "" ]]; then | |
docker image rmi --force ${{ env.WORKER_IMAGE_TAG }} 2>/dev/null | |
fi | |
if [[ "$(docker images -q ${{ env.CLIENT_IMAGE_TAG }} 2> /dev/null)" == "" ]]; then | |
docker image rmi --force ${{ env.CLIENT_IMAGE_TAG }} 2>/dev/null | |
fi | |
docker tag integritee-worker-${{ env.IMAGE_SUFFIX }} ${{ env.WORKER_IMAGE_TAG }} | |
docker tag integritee-cli-client-${{ env.IMAGE_SUFFIX }} ${{ env.CLIENT_IMAGE_TAG }} | |
docker pull integritee/integritee-node:1.1.3 | |
docker tag integritee/integritee-node:1.1.3 ${{ env.INTEGRITEE_NODE }} | |
docker images --all | |
- name: Integration Test ${{ matrix.test }}-${{ matrix.flavor_id }} | |
timeout-minutes: 30 | |
run: | | |
cd docker | |
docker compose -f docker-compose.yml -f ${{ matrix.demo_name }}.yml up ${{ matrix.demo_name }} --no-build --exit-code-from ${{ matrix.demo_name }} | |
- name: Stop docker containers | |
if: always() | |
run: | | |
cd docker | |
docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < ${{ matrix.demo_name }}.yml) -p ${PROJECT} stop | |
- name: Integration Test ${{ matrix.test }}-${{ matrix.flavor_id }} | |
timeout-minutes: 30 | |
run: | | |
cd docker | |
docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < ${{ matrix.demo_name }}.yml) -p ${PROJECT} up ${{ matrix.demo_name }} --no-build --exit-code-from ${{ matrix.demo_name }} --remove-orphans | |
- name: Collect Docker Logs | |
continue-on-error: true | |
if: always() | |
uses: jwalton/gh-docker-logs@v2 | |
with: | |
images: '${{ env.WORKER_IMAGE_TAG }},${{ env.CLIENT_IMAGE_TAG }},${{ env.INTEGRITEE_NODE }}' | |
tail: all | |
dest: ${{ env.LOG_DIR }} | |
- name: Upload logs | |
if: always() | |
uses: actions/upload-artifact@v3 | |
with: | |
name: logs-${{ matrix.test }}-${{ matrix.flavor_id }} | |
path: ${{ env.LOG_DIR }} | |
- name: Stop docker containers | |
if: always() | |
continue-on-error: true | |
run: | | |
cd docker | |
docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < ${{ matrix.demo_name }}.yml) -p ${PROJECT} stop | |
- name: Delete images | |
run: | | |
if [[ "$(docker images -q integritee-worker-${{ env.IMAGE_SUFFIX }} 2> /dev/null)" != "" ]]; then | |
docker image rmi --force integritee-worker-${{ env.IMAGE_SUFFIX }} 2>/dev/null | |
fi | |
if [[ "$(docker images -q integritee-cli-client-${{ env.IMAGE_SUFFIX }} 2> /dev/null)" != "" ]]; then | |
docker image rmi --force integritee-cli-client-${{ env.IMAGE_SUFFIX }} 2>/dev/null | |
fi | |
if [[ "$(docker images -q ${{ env.WORKER_IMAGE_TAG }} 2> /dev/null)" != "" ]]; then | |
docker image rmi --force ${{ env.WORKER_IMAGE_TAG }} 2>/dev/null | |
fi | |
if [[ "$(docker images -q ${{ env.CLIENT_IMAGE_TAG }} 2> /dev/null)" != "" ]]; then | |
docker image rmi --force ${{ env.CLIENT_IMAGE_TAG }} 2>/dev/null | |
fi | |
if [[ "$(docker images -q ${{ env.INTEGRITEE_NODE }} 2> /dev/null)" != "" ]]; then | |
docker image rmi --force ${{ env.INTEGRITEE_NODE }} 2>/dev/null | |
fi | |
docker images --all | |
release-build: | |
runs-on: integritee-builder-sgx | |
name: Release Build of teeracle | |
if: startsWith(github.ref, 'refs/tags/') | |
needs: [ build-test, integration-tests ] | |
strategy: | |
fail-fast: false | |
matrix: | |
include: | |
- flavor_id: teeracle | |
mode: teeracle | |
sgx_mode: HW | |
additional_features: dcap | |
- flavor_id: sidechain | |
mode: sidechain | |
sgx_mode: HW | |
additional_features: dcap | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Add masks | |
run: | | |
echo "::add-mask::$VAULT_TOKEN" | |
echo "::add-mask::$PRIVKEY_B64" | |
echo "::add-mask::$PRIVKEY_PASS" | |
- name: Set env | |
run: | | |
fingerprint=$RANDOM | |
echo "FINGERPRINT=$fingerprint" >> $GITHUB_ENV | |
SGX_MODE_LOWERCASE=$(echo "${${{ matrix.sgx_mode }},,}") | |
echo "IMAGE_SUFFIX=$SGX_MODE_LOWERCASE-${{ matrix.flavor_id }}-${{ github.sha }}" >> $GITHUB_ENV | |
if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then | |
echo "DOCKER_DEVICES=--device=/dev/sgx/enclave --device=/dev/sgx/provision" >> $GITHUB_ENV | |
echo "DOCKER_VOLUMES=--volume /var/run/aesmd:/var/run/aesmd --volume /etc/sgx_default_qcnl.conf:/etc/sgx_default_qcnl.conf" >> $GITHUB_ENV | |
else | |
echo "DOCKER_DEVICES=" >> $GITHUB_ENV | |
echo "DOCKER_VOLUMES=" >> $GITHUB_ENV | |
fi | |
echo "VAULT_TOKEN=$VAULT_TOKEN" >> "$GITHUB_ENV" | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v2 | |
with: | |
buildkitd-flags: --debug | |
driver: docker-container | |
- name: Import secrets | |
uses: hashicorp/vault-action@v2 | |
id: import-secrets | |
with: | |
url: ${{ secrets.VAULT_URL }} | |
tlsSkipVerify: false | |
token: ${{ env.VAULT_TOKEN }} | |
exportEnv: false | |
secrets: | | |
${{ secrets.VAULT_PATH }} intel_sgx_pem_base64 | PRIVKEY_B64 ; | |
${{ secrets.VAULT_PATH }} password | PRIVKEY_PASS | |
- name: Get secrets | |
env: | |
PRIVKEY_B64: ${{ steps.import-secrets.outputs.PRIVKEY_B64 }} | |
PRIVKEY_PASS: ${{ steps.import-secrets.outputs.PRIVKEY_PASS }} | |
run: | | |
echo $PRIVKEY_B64 | base64 --ignore-garbage --decode > enclave-runtime/intel_sgx.pem | |
echo $PRIVKEY_PASS > enclave-runtime/passfile.txt | |
- name: Build Worker & Run Cargo Test | |
env: | |
DOCKER_BUILDKIT: 1 | |
run: > | |
docker build -t integritee/${{ matrix.flavor_id }}:${{ github.ref_name }} | |
--target deployed-worker | |
--build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg SGX_COMMERCIAL_KEY=enclave-runtime/intel_sgx.pem --build-arg SGX_PASSFILE=enclave-runtime/passfile.txt --build-arg SGX_PRODUCTION=1 --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} --build-arg SGX_MODE=${{ matrix.sgx_mode }} | |
-f build.Dockerfile . | |
- name: Save released teeracle | |
run: | | |
docker image save integritee/${{ matrix.flavor_id }}:${{ github.ref_name }} | gzip > integritee-worker-${{ matrix.flavor_id }}-${{ github.ref_name }}.tar.gz | |
docker images --all | |
- name: Upload teeracle image | |
uses: actions/upload-artifact@v3 | |
with: | |
name: integritee-worker-${{ matrix.flavor_id }}-${{ github.ref_name }}.tar.gz | |
path: integritee-worker-${{ matrix.flavor_id }}-${{ github.ref_name }}.tar.gz | |
- name: Delete images | |
run: | | |
if [[ "$(docker images -q integritee/${{ matrix.flavor_id }}:${{ github.ref_name }} 2> /dev/null)" != "" ]]; then | |
docker image rmi --force integritee/${{ matrix.flavor_id }}:${{ github.ref_name }} 2>/dev/null | |
fi | |
docker images --all | |
release: | |
runs-on: ubuntu-latest | |
name: Draft Release | |
if: startsWith(github.ref, 'refs/tags/') | |
needs: [ build-test, integration-tests, release-build ] | |
outputs: | |
release_url: ${{ steps.create-release.outputs.html_url }} | |
asset_upload_url: ${{ steps.create-release.outputs.upload_url }} | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Download Worker Image | |
uses: actions/download-artifact@v3 | |
with: | |
name: integritee-worker-teeracle-${{ github.ref_name }}.tar.gz | |
path: . | |
- name: Download Worker Image | |
uses: actions/download-artifact@v3 | |
with: | |
name: integritee-worker-sidechain-${{ github.ref_name }}.tar.gz | |
path: . | |
# | |
# Temporary comment out until we decide what to release | |
# | |
# - name: Download Integritee Client | |
# uses: actions/download-artifact@v3 | |
# with: | |
# name: integritee-client-sidechain-${{ github.sha }} | |
# path: integritee-client-tmp | |
# - name: Download Enclave Signed | |
# uses: actions/download-artifact@v3 | |
# with: | |
# name: enclave-signed-sidechain-${{ github.sha }} | |
# path: enclave-signed-tmp | |
# - name: Move service binaries | |
# run: mv integritee-worker-tmp/integritee-service ./integritee-demo-validateer | |
# - name: Move service client binaries | |
# run: mv integritee-client-tmp/integritee-cli ./integritee-client | |
# - name: Move service client binaries | |
# run: mv enclave-signed-tmp/enclave.signed.so ./enclave.signed.so | |
- name: Changelog | |
uses: scottbrenner/generate-changelog-action@master | |
id: Changelog | |
- name: Display structure of downloaded files | |
run: ls -R | |
working-directory: . | |
- name: Release | |
id: create-release | |
uses: softprops/action-gh-release@v1 | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
with: | |
body: | | |
${{ steps.Changelog.outputs.changelog }} | |
draft: true | |
name: Docker ${{ github.ref_name }} | |
files: | | |
integritee-worker-teeracle-${{ github.ref_name }}.tar.gz | |
integritee-worker-sidechain-${{ github.ref_name }}.tar.gz | |
integritee-client | |
integritee-demo-validateer | |
enclave.signed.so |