Skip to content

Update build_and_test.yml #180

Update build_and_test.yml

Update build_and_test.yml #180

name: Build, Test, Clippy
on:
workflow_dispatch:
push:
branches:
- master
- 'sdk-v[0-9]+.[0-9]+.[0-9]+-*'
tags:
- 'v[0-9]+.[0-9]+.[0-9]+*'
pull_request:
branches:
- master
- 'sdk-v[0-9]+.[0-9]+.[0-9]+-*'
env:
CARGO_TERM_COLOR: always
LOG_DIR: logs
BUILD_CONTAINER_NAME: integritee_worker_enclave_test
jobs:
cancel_previous_runs:
name: Cancel Previous Runs
runs-on: ubuntu-latest
steps:
- uses: styfle/cancel-workflow-action@0.11.0
with:
access_token: ${{ secrets.GITHUB_TOKEN }}
build-test:
runs-on: ${{ matrix.host }}
strategy:
fail-fast: false
matrix:
include:
- flavor_id: offchain-worker
mode: offchain-worker
host: ubuntu-latest
sgx_mode: SW
additional_features: dcap
- flavor_id: offchain-worker
mode: offchain-worker
host: ubuntu-latest
sgx_mode: HW
additional_features: dcap
steps:
- uses: actions/checkout@v3
- name: Set env
run: |
fingerprint=$RANDOM
echo "FINGERPRINT=$fingerprint" >> $GITHUB_ENV
SGX_MODE_LOWERCASE=$(echo "${${{ matrix.sgx_mode }},,}")
echo "IMAGE_SUFFIX=$SGX_MODE_LOWERCASE-${{ matrix.flavor_id }}-${{ github.sha }}" >> $GITHUB_ENV
if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then
echo "DOCKER_DEVICES=--device=/dev/sgx/enclave --device=/dev/sgx/provision" >> $GITHUB_ENV
echo "DOCKER_VOLUMES=--volume /var/run/aesmd:/var/run/aesmd --volume /etc/sgx_default_qcnl.conf:/etc/sgx_default_qcnl.conf" >> $GITHUB_ENV
else
echo "DOCKER_DEVICES=" >> $GITHUB_ENV
echo "DOCKER_VOLUMES=" >> $GITHUB_ENV
fi
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
buildkitd-flags: --debug
driver: docker-container
- name: Build Worker
env:
DOCKER_BUILDKIT: 1
run: >
docker build -t integritee-worker-${{ env.IMAGE_SUFFIX }}
--target deployed-worker
--build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg FINGERPRINT=${FINGERPRINT} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} --build-arg SGX_MODE=${{ matrix.sgx_mode }}
-f build.Dockerfile .
- run: docker images --all
- name: Test Enclave # cargo test is not supported in the enclave, see: https://github.com/apache/incubator-teaclave-sgx-sdk/issues/232
if: ${{ matrix.sgx_mode != 'HW' }} # Change for OLI, can't run these tests in HW mode on Github Hosted Runners
run: docker run --rm ${{ env.DOCKER_DEVICES }} ${{ env.DOCKER_VOLUMES }} integritee-worker-${{ env.IMAGE_SUFFIX }} test --all
- name: Export worker image
run: |
docker image save integritee-worker-${{ env.IMAGE_SUFFIX }} | gzip > integritee-worker-${{ env.IMAGE_SUFFIX }}.tar.gz
- name: Upload worker image
uses: actions/upload-artifact@v3
with:
name: integritee-worker-${{ env.IMAGE_SUFFIX }}.tar.gz
path: integritee-worker-${{ env.IMAGE_SUFFIX }}.tar.gz
- name: Create Enclave Digest File
run: |
mrenclave_hex=$(docker run integritee-worker-${{ env.IMAGE_SUFFIX }} mrenclave | grep -oP ':\s*\K[a-fA-F0-9]+')
echo "$mrenclave_hex" > mrenclave-${{ env.IMAGE_SUFFIX }}.hex
- name: Upload Enclave Digest File
uses: actions/upload-artifact@v3
with:
name: mrenclave-${{ env.IMAGE_SUFFIX }}.hex
path: mrenclave-${{ env.IMAGE_SUFFIX }}.hex
- name: Delete images
run: |
if [[ "$(docker images -q integritee-worker-${{ env.IMAGE_SUFFIX }} 2> /dev/null)" != "" ]]; then
docker image rmi --force integritee-worker-${{ env.IMAGE_SUFFIX }} 2>/dev/null
fi
docker images --all
build-client:
runs-on: ${{ matrix.host }}
strategy:
fail-fast: false
matrix:
include:
- flavor_id: offchain-worker
mode: offchain-worker
host: ubuntu-latest
sgx_mode: SW
additional_features: dcap
steps:
- uses: actions/checkout@v3
- name: Set env
run: |
fingerprint=$RANDOM
echo "FINGERPRINT=$fingerprint" >> $GITHUB_ENV
SGX_MODE_LOWERCASE=$(echo "${${{ matrix.sgx_mode }},,}")
echo "IMAGE_SUFFIX=$SGX_MODE_LOWERCASE-${{ matrix.flavor_id }}-${{ github.sha }}" >> $GITHUB_ENV
if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then
echo "DOCKER_DEVICES=--device=/dev/sgx/enclave --device=/dev/sgx/provision" >> $GITHUB_ENV
echo "DOCKER_VOLUMES=--volume /var/run/aesmd:/var/run/aesmd --volume /etc/sgx_default_qcnl.conf:/etc/sgx_default_qcnl.conf" >> $GITHUB_ENV
else
echo "DOCKER_DEVICES=" >> $GITHUB_ENV
echo "DOCKER_VOLUMES=" >> $GITHUB_ENV
fi
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
buildkitd-flags: --debug
driver: docker-container
- name: Build CLI client
env:
DOCKER_BUILDKIT: 1
run: >
docker build -t integritee-cli-client-${{ env.IMAGE_SUFFIX }}
--target deployed-client
--build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }}
-f build.Dockerfile .
- run: docker images --all
- name: Export client image
run: |
docker image save integritee-cli-client-${{ env.IMAGE_SUFFIX }} | gzip > integritee-cli-client-${{ env.IMAGE_SUFFIX }}.tar.gz
- name: Upload CLI client image
uses: actions/upload-artifact@v3
with:
name: integritee-cli-client-${{ env.IMAGE_SUFFIX }}.tar.gz
path: integritee-cli-client-${{ env.IMAGE_SUFFIX }}.tar.gz
- name: Delete images
run: |
if [[ "$(docker images -q integritee-cli-client-${{ env.IMAGE_SUFFIX }} 2> /dev/null)" != "" ]]; then
docker image rmi --force integritee-cli-client-${{ env.IMAGE_SUFFIX }} 2>/dev/null
fi
docker images --all
code-quality:
runs-on: ubuntu-latest
container: "integritee/integritee-dev:0.2.2"
strategy:
fail-fast: false
matrix:
check: [
# Workspace
cargo test --release,
# Worker
# Use release mode as the CI runs out of disk space otherwise.
cargo clippy --release -- -D warnings,
cargo clippy --release --features evm -- -D warnings,
cargo clippy --release --features sidechain -- -D warnings,
cargo clippy --release --features teeracle -- -D warnings,
cargo clippy --release --features offchain-worker -- -D warnings,
# Enclave
cd enclave-runtime && cargo clippy -- -D warnings,
cd enclave-runtime && cargo clippy --features evm -- -D warnings,
cd enclave-runtime && cargo clippy --features sidechain -- -D warnings,
cd enclave-runtime && cargo clippy --features teeracle -- -D warnings,
cd enclave-runtime && cargo clippy --features offchain-worker -- -D warnings,
# Fmt
cargo fmt --all -- --check,
cd enclave-runtime && cargo fmt --all -- --check,
]
steps:
- uses: actions/checkout@v3
- name: init-rust-target
# Enclave is not in the same workspace
run: rustup show && cd enclave-runtime && rustup show
- uses: Swatinem/rust-cache@v2
with:
key: ${{ matrix.check }}
- name: ${{ matrix.check }}
run: ${{ matrix.check }}
toml-fmt:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: init rust
run: rustup show
- name: Install taplo
run: cargo install taplo-cli --locked
- name: Cargo.toml fmt
run: taplo fmt --check
- name: Fail-fast; cancel other jobs
if: failure()
uses: andymckay/cancel-action@0.3
integration-tests:
runs-on: ${{ matrix.host }}
if: ${{ always() }}
needs: [build-test, build-client]
env:
WORKER_IMAGE_TAG: integritee-worker:dev
CLIENT_IMAGE_TAG: integritee-cli:dev
strategy:
fail-fast: false
matrix:
include:
- test: M6
flavor_id: offchain-worker
demo_name: demo-indirect-invocation
- test: BestEnergy
flavor_id: offchain-worker
demo_name: demo-energy-market
steps:
- uses: actions/checkout@v3
- name: Set env
run: |
version=$RANDOM
SGX_MODE_LOWERCASE=$(echo "${${{ matrix.sgx_mode }},,}")
echo "IMAGE_SUFFIX=$SGX_MODE_LOWERCASE-${{ matrix.flavor_id }}-${{ github.sha }}" >> $GITHUB_ENV
echo "FLAVOR_ID=${{ matrix.flavor_id }}" >> $GITHUB_ENV
echo "PROJECT=${{ matrix.flavor_id }}-${{ matrix.demo_name }}" >> $GITHUB_ENV
echo "VERSION=dev.$version" >> $GITHUB_ENV
echo "WORKER_IMAGE_TAG=integritee-worker:dev.$version" >> $GITHUB_ENV
echo "INTEGRITEE_NODE=integritee-node:1.1.3.$version" >> $GITHUB_ENV
echo "CLIENT_IMAGE_TAG=integritee-cli:dev.$version" >> $GITHUB_ENV
if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then
echo "SGX_PROVISION=/dev/sgx/provision" >> $GITHUB_ENV
echo "SGX_ENCLAVE=/dev/sgx/enclave" >> $GITHUB_ENV
echo "AESMD=/var/run/aesmd" >> $GITHUB_ENV
echo "SGX_QCNL=/etc/sgx_default_qcnl.conf" >> $GITHUB_ENV
fi
echo "LOG_DIR=./logs-$version" >> $GITHUB_ENV
- name: Download Worker Image
uses: actions/download-artifact@v4
with:
name: integritee-worker-${{ env.IMAGE_SUFFIX }}.tar.gz
path: .
- name: Download CLI client Image
uses: actions/download-artifact@v4
with:
name: integritee-cli-client-${{ env.IMAGE_SUFFIX }}.tar.gz
path: .
- name: Load Worker & Client Images
env:
DOCKER_BUILDKIT: 1
run: |
docker image load --input integritee-worker-${{ env.IMAGE_SUFFIX }}.tar.gz
docker image load --input integritee-cli-client-${{ env.IMAGE_SUFFIX }}.tar.gz
docker images --all
##
# Before tagging, delete the old "stuck" ones to be sure that the newly created ones are the latest
# Without if the docker image rmi throws an error if the image doesn't exist.
##
- name: Re-name Image Tags
run: |
if [[ "$(docker images -q ${{ env.WORKER_IMAGE_TAG }} 2> /dev/null)" == "" ]]; then
docker image rmi --force ${{ env.WORKER_IMAGE_TAG }} 2>/dev/null
fi
if [[ "$(docker images -q ${{ env.CLIENT_IMAGE_TAG }} 2> /dev/null)" == "" ]]; then
docker image rmi --force ${{ env.CLIENT_IMAGE_TAG }} 2>/dev/null
fi
docker tag integritee-worker-${{ env.IMAGE_SUFFIX }} ${{ env.WORKER_IMAGE_TAG }}
docker tag integritee-cli-client-${{ env.IMAGE_SUFFIX }} ${{ env.CLIENT_IMAGE_TAG }}
docker pull integritee/integritee-node:1.1.3
docker tag integritee/integritee-node:1.1.3 ${{ env.INTEGRITEE_NODE }}
docker images --all
- name: Integration Test ${{ matrix.test }}-${{ matrix.flavor_id }}
timeout-minutes: 30
run: |
cd docker
docker compose -f docker-compose.yml -f ${{ matrix.demo_name }}.yml up ${{ matrix.demo_name }} --no-build --exit-code-from ${{ matrix.demo_name }}
- name: Stop docker containers
if: always()
run: |
cd docker
docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < ${{ matrix.demo_name }}.yml) -p ${PROJECT} stop
- name: Integration Test ${{ matrix.test }}-${{ matrix.flavor_id }}
timeout-minutes: 30
run: |
cd docker
docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < ${{ matrix.demo_name }}.yml) -p ${PROJECT} up ${{ matrix.demo_name }} --no-build --exit-code-from ${{ matrix.demo_name }} --remove-orphans
- name: Collect Docker Logs
continue-on-error: true
if: always()
uses: jwalton/gh-docker-logs@v2
with:
images: '${{ env.WORKER_IMAGE_TAG }},${{ env.CLIENT_IMAGE_TAG }},${{ env.INTEGRITEE_NODE }}'
tail: all
dest: ${{ env.LOG_DIR }}
- name: Upload logs
if: always()
uses: actions/upload-artifact@v3
with:
name: logs-${{ matrix.test }}-${{ matrix.flavor_id }}
path: ${{ env.LOG_DIR }}
- name: Stop docker containers
if: always()
continue-on-error: true
run: |
cd docker
docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < ${{ matrix.demo_name }}.yml) -p ${PROJECT} stop
- name: Delete images
run: |
if [[ "$(docker images -q integritee-worker-${{ env.IMAGE_SUFFIX }} 2> /dev/null)" != "" ]]; then
docker image rmi --force integritee-worker-${{ env.IMAGE_SUFFIX }} 2>/dev/null
fi
if [[ "$(docker images -q integritee-cli-client-${{ env.IMAGE_SUFFIX }} 2> /dev/null)" != "" ]]; then
docker image rmi --force integritee-cli-client-${{ env.IMAGE_SUFFIX }} 2>/dev/null
fi
if [[ "$(docker images -q ${{ env.WORKER_IMAGE_TAG }} 2> /dev/null)" != "" ]]; then
docker image rmi --force ${{ env.WORKER_IMAGE_TAG }} 2>/dev/null
fi
if [[ "$(docker images -q ${{ env.CLIENT_IMAGE_TAG }} 2> /dev/null)" != "" ]]; then
docker image rmi --force ${{ env.CLIENT_IMAGE_TAG }} 2>/dev/null
fi
if [[ "$(docker images -q ${{ env.INTEGRITEE_NODE }} 2> /dev/null)" != "" ]]; then
docker image rmi --force ${{ env.INTEGRITEE_NODE }} 2>/dev/null
fi
docker images --all
release-build:
runs-on: integritee-builder-sgx
name: Release Build of teeracle
if: startsWith(github.ref, 'refs/tags/')
needs: [ build-test, integration-tests ]
strategy:
fail-fast: false
matrix:
include:
- flavor_id: teeracle
mode: teeracle
sgx_mode: HW
additional_features: dcap
- flavor_id: sidechain
mode: sidechain
sgx_mode: HW
additional_features: dcap
steps:
- uses: actions/checkout@v3
- name: Add masks
run: |
echo "::add-mask::$VAULT_TOKEN"
echo "::add-mask::$PRIVKEY_B64"
echo "::add-mask::$PRIVKEY_PASS"
- name: Set env
run: |
fingerprint=$RANDOM
echo "FINGERPRINT=$fingerprint" >> $GITHUB_ENV
SGX_MODE_LOWERCASE=$(echo "${${{ matrix.sgx_mode }},,}")
echo "IMAGE_SUFFIX=$SGX_MODE_LOWERCASE-${{ matrix.flavor_id }}-${{ github.sha }}" >> $GITHUB_ENV
if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then
echo "DOCKER_DEVICES=--device=/dev/sgx/enclave --device=/dev/sgx/provision" >> $GITHUB_ENV
echo "DOCKER_VOLUMES=--volume /var/run/aesmd:/var/run/aesmd --volume /etc/sgx_default_qcnl.conf:/etc/sgx_default_qcnl.conf" >> $GITHUB_ENV
else
echo "DOCKER_DEVICES=" >> $GITHUB_ENV
echo "DOCKER_VOLUMES=" >> $GITHUB_ENV
fi
echo "VAULT_TOKEN=$VAULT_TOKEN" >> "$GITHUB_ENV"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
buildkitd-flags: --debug
driver: docker-container
- name: Import secrets
uses: hashicorp/vault-action@v2
id: import-secrets
with:
url: ${{ secrets.VAULT_URL }}
tlsSkipVerify: false
token: ${{ env.VAULT_TOKEN }}
exportEnv: false
secrets: |
${{ secrets.VAULT_PATH }} intel_sgx_pem_base64 | PRIVKEY_B64 ;
${{ secrets.VAULT_PATH }} password | PRIVKEY_PASS
- name: Get secrets
env:
PRIVKEY_B64: ${{ steps.import-secrets.outputs.PRIVKEY_B64 }}
PRIVKEY_PASS: ${{ steps.import-secrets.outputs.PRIVKEY_PASS }}
run: |
echo $PRIVKEY_B64 | base64 --ignore-garbage --decode > enclave-runtime/intel_sgx.pem
echo $PRIVKEY_PASS > enclave-runtime/passfile.txt
- name: Build Worker & Run Cargo Test
env:
DOCKER_BUILDKIT: 1
run: >
docker build -t integritee/${{ matrix.flavor_id }}:${{ github.ref_name }}
--target deployed-worker
--build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg SGX_COMMERCIAL_KEY=enclave-runtime/intel_sgx.pem --build-arg SGX_PASSFILE=enclave-runtime/passfile.txt --build-arg SGX_PRODUCTION=1 --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} --build-arg SGX_MODE=${{ matrix.sgx_mode }}
-f build.Dockerfile .
- name: Save released teeracle
run: |
docker image save integritee/${{ matrix.flavor_id }}:${{ github.ref_name }} | gzip > integritee-worker-${{ matrix.flavor_id }}-${{ github.ref_name }}.tar.gz
docker images --all
- name: Upload teeracle image
uses: actions/upload-artifact@v3
with:
name: integritee-worker-${{ matrix.flavor_id }}-${{ github.ref_name }}.tar.gz
path: integritee-worker-${{ matrix.flavor_id }}-${{ github.ref_name }}.tar.gz
- name: Delete images
run: |
if [[ "$(docker images -q integritee/${{ matrix.flavor_id }}:${{ github.ref_name }} 2> /dev/null)" != "" ]]; then
docker image rmi --force integritee/${{ matrix.flavor_id }}:${{ github.ref_name }} 2>/dev/null
fi
docker images --all
release:
runs-on: ubuntu-latest
name: Draft Release
if: startsWith(github.ref, 'refs/tags/')
needs: [ build-test, integration-tests, release-build ]
outputs:
release_url: ${{ steps.create-release.outputs.html_url }}
asset_upload_url: ${{ steps.create-release.outputs.upload_url }}
steps:
- uses: actions/checkout@v3
- name: Download Worker Image
uses: actions/download-artifact@v3
with:
name: integritee-worker-teeracle-${{ github.ref_name }}.tar.gz
path: .
- name: Download Worker Image
uses: actions/download-artifact@v3
with:
name: integritee-worker-sidechain-${{ github.ref_name }}.tar.gz
path: .
#
# Temporary comment out until we decide what to release
#
# - name: Download Integritee Client
# uses: actions/download-artifact@v3
# with:
# name: integritee-client-sidechain-${{ github.sha }}
# path: integritee-client-tmp
# - name: Download Enclave Signed
# uses: actions/download-artifact@v3
# with:
# name: enclave-signed-sidechain-${{ github.sha }}
# path: enclave-signed-tmp
# - name: Move service binaries
# run: mv integritee-worker-tmp/integritee-service ./integritee-demo-validateer
# - name: Move service client binaries
# run: mv integritee-client-tmp/integritee-cli ./integritee-client
# - name: Move service client binaries
# run: mv enclave-signed-tmp/enclave.signed.so ./enclave.signed.so
- name: Changelog
uses: scottbrenner/generate-changelog-action@master
id: Changelog
- name: Display structure of downloaded files
run: ls -R
working-directory: .
- name: Release
id: create-release
uses: softprops/action-gh-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
body: |
${{ steps.Changelog.outputs.changelog }}
draft: true
name: Docker ${{ github.ref_name }}
files: |
integritee-worker-teeracle-${{ github.ref_name }}.tar.gz
integritee-worker-sidechain-${{ github.ref_name }}.tar.gz
integritee-client
integritee-demo-validateer
enclave.signed.so