From eb3f9a592f51a12332a9b22819959c1b53d3dc45 Mon Sep 17 00:00:00 2001 From: mosonyi Date: Wed, 12 Apr 2023 22:48:03 +0200 Subject: [PATCH] Sgx runner (#1222) * Added self hosted runner #386 --------- Co-authored-by: Szilard Parrag Co-authored-by: Christian Langenbacher --- .github/workflows/build_and_test.yml | 123 ++++++++++++++++--- .github/workflows/delete-release.yml | 2 +- .github/workflows/label-checker.yml | 2 +- .github/workflows/publish-docker-release.yml | 2 +- .github/workflows/publish-draft-release.yml | 2 +- build.Dockerfile | 50 ++++---- docker/README.md | 32 ++--- docker/demo-direct-call.yml | 16 ++- docker/demo-indirect-invocation.yml | 16 ++- docker/demo-sidechain.yml | 17 ++- docker/demo-smart-contract.yml | 16 ++- docker/demo-teeracle-generic.yml | 29 +++-- docker/demo-teeracle.yml | 29 +++-- docker/docker-compose.yml | 45 ++++--- docker/fork-inducer.yml | 23 ++-- docker/sidechain-benchmark.yml | 16 ++- 16 files changed, 289 insertions(+), 131 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 6e81353183..cc1641130d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -21,31 +21,49 @@ env: jobs: cancel_previous_runs: name: Cancel Previous Runs - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: styfle/cancel-workflow-action@0.11.0 with: access_token: ${{ secrets.GITHUB_TOKEN }} build-test: - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.host }} strategy: fail-fast: false matrix: include: - flavor_id: sidechain mode: sidechain + host: integritee-builder-sgx + sgx_mode: HW - flavor_id: offchain-worker mode: offchain-worker + host: integritee-builder-sgx + sgx_mode: HW - flavor_id: teeracle mode: teeracle + host: integritee-builder-sgx + sgx_mode: HW - flavor_id: sidechain-evm mode: sidechain additional_features: evm + host: integritee-builder-sgx + sgx_mode: HW steps: - uses: actions/checkout@v3 + - name: Set env + run: | + if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then + echo "DOCKER_DEVICES=--device=/dev/sgx/enclave --device=/dev/sgx/provision" >> $GITHUB_ENV + echo "DOCKER_VOLUMES=--volume /var/run/aesmd:/var/run/aesmd" >> $GITHUB_ENV + else + echo "DOCKER_DEVICES=" >> $GITHUB_ENV + echo "DOCKER_VOLUMES=" >> $GITHUB_ENV + fi + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 with: @@ -58,7 +76,7 @@ jobs: run: > docker build -t integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }} --target deployed-worker - --build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} + --build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} --build-arg SGX_MODE=${{ matrix.sgx_mode }} -f build.Dockerfile . - name: Build CLI client @@ -73,7 +91,7 @@ jobs: - run: docker images --all - name: Test Enclave # cargo test is not supported in the enclave, see: https://github.com/apache/incubator-teaclave-sgx-sdk/issues/232 - run: docker run --name ${{ env.BUILD_CONTAINER_NAME }} integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }} test --all + run: docker run ${{ env.DOCKER_DEVICES }} ${{ env.DOCKER_VOLUMES }} integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }} test --all - name: Export worker image(s) run: | @@ -93,8 +111,8 @@ jobs: path: integritee-cli-client-${{ matrix.flavor_id }}-${{ github.sha }}.tar.gz clippy: - runs-on: ubuntu-22.04 - container: "integritee/integritee-dev:0.1.13" + runs-on: ubuntu-latest + container: "integritee/integritee-dev:0.2.1" steps: - uses: actions/checkout@v3 - name: init rust @@ -131,7 +149,7 @@ jobs: uses: andymckay/cancel-action@0.3 fmt: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: init rust @@ -152,7 +170,7 @@ jobs: uses: andymckay/cancel-action@0.3 integration-tests: - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.host }} if: ${{ always() }} needs: build-test env: @@ -168,34 +186,63 @@ jobs: - test: M6 flavor_id: sidechain demo_name: demo-indirect-invocation + host: test-runner-sgx + sgx_mode: HW - test: M8 flavor_id: sidechain demo_name: demo-direct-call + host: test-runner-sgx + sgx_mode: HW - test: Sidechain flavor_id: sidechain demo_name: demo-sidechain + host: test-runner-sgx + sgx_mode: HW - test: M6 flavor_id: offchain-worker demo_name: demo-indirect-invocation + host: test-runner-sgx + sgx_mode: HW - test: Teeracle flavor_id: teeracle demo_name: demo-teeracle + host: test-runner-sgx + sgx_mode: HW - test: Teeracle flavor_id: teeracle demo_name: demo-teeracle-generic + host: test-runner-sgx + sgx_mode: HW - test: Benchmark flavor_id: sidechain demo_name: sidechain-benchmark + host: test-runner-sgx + sgx_mode: HW - test: EVM flavor_id: sidechain-evm demo_name: demo-smart-contract + host: test-runner-sgx + sgx_mode: HW steps: - uses: actions/checkout@v3 - name: Set env run: | + version=$RANDOM echo "FLAVOR_ID=${{ matrix.flavor_id }}" >> $GITHUB_ENV + echo "PROJECT=${{ matrix.flavor_id }}-${{ matrix.demo_name }}" >> $GITHUB_ENV + echo "VERSION=dev.$version" >> $GITHUB_ENV + echo "WORKER_IMAGE_TAG=integritee-worker:dev.$version" >> $GITHUB_ENV + echo "INTEGRITEE_NODE=integritee-node-dev:1.0.33.$version" >> $GITHUB_ENV + echo "CLIENT_IMAGE_TAG=integritee-cli:dev.$version" >> $GITHUB_ENV + if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then + echo "SGX_PROVISION=/dev/sgx/provision" >> $GITHUB_ENV + echo "SGX_ENCLAVE=/dev/sgx/enclave" >> $GITHUB_ENV + echo "AESMD=/var/run/aesmd" >> $GITHUB_ENV + fi + + echo "LOG_DIR=./logs-$version" >> $GITHUB_ENV - name: Download Worker Image uses: actions/download-artifact@v3 @@ -217,42 +264,86 @@ jobs: docker image load --input integritee-cli-client-${{ matrix.flavor_id }}-${{ github.sha }}.tar.gz docker images --all + ## + # Before tagging, delete the old "stuck" ones to be sure that the newly created ones are the latest + # Without if the docker image rmi throws an error if the image doesn't exist. + ## - name: Re-name Image Tags run: | + if [[ "$(docker images -q ${{ env.WORKER_IMAGE_TAG }} 2> /dev/null)" == "" ]]; then + docker image rmi --force ${{ env.WORKER_IMAGE_TAG }} 2>/dev/null + fi + if [[ "$(docker images -q ${{ env.CLIENT_IMAGE_TAG }} 2> /dev/null)" == "" ]]; then + docker image rmi --force ${{ env.CLIENT_IMAGE_TAG }} 2>/dev/null + fi docker tag integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }} ${{ env.WORKER_IMAGE_TAG }} docker tag integritee-cli-client-${{ matrix.flavor_id }}-${{ github.sha }} ${{ env.CLIENT_IMAGE_TAG }} + docker tag integritee/integritee-node-dev:1.0.33 ${{ env.INTEGRITEE_NODE }} docker images --all - - name: Integration Test ${{ matrix.test }}-${{ matrix.flavor_id }} + ## + # Stop any stucked/running compose projects + ## + - name: Stop docker containers + if: always() + continue-on-error: true run: | cd docker - docker compose -f docker-compose.yml -f ${{ matrix.demo_name }}.yml up ${{ matrix.demo_name }} --no-build --exit-code-from ${{ matrix.demo_name }} + docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < ${{ matrix.demo_name }}.yml) -p ${PROJECT} stop - - name: Stop docker containers + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Integration Test ${{ matrix.test }}-${{ matrix.flavor_id }} run: | cd docker - docker compose -f docker-compose.yml -f ${{ matrix.demo_name }}.yml stop + docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < ${{ matrix.demo_name }}.yml) -p ${PROJECT} up ${{ matrix.demo_name }} --no-build --exit-code-from ${{ matrix.demo_name }} --remove-orphans + - name: Collect Docker Logs continue-on-error: true if: always() uses: jwalton/gh-docker-logs@v2 with: - #images: '${{ env.WORKER_IMAGE_TAG }},${{ env.CLIENT_IMAGE_TAG }}' + images: '${{ env.WORKER_IMAGE_TAG }},${{ env.CLIENT_IMAGE_TAG }},${{ env.INTEGRITEE_NODE }}' tail: all - dest: ./${{ env.LOG_DIR }} + dest: ${{ env.LOG_DIR }} - name: Upload logs if: always() uses: actions/upload-artifact@v3 with: name: logs-${{ matrix.test }}-${{ matrix.flavor_id }} - path: ./${{ env.LOG_DIR }} + path: ${{ env.LOG_DIR }} + + - name: Stop docker containers + if: always() + continue-on-error: true + run: | + cd docker + docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < ${{ matrix.demo_name }}.yml) -p ${PROJECT} stop + + - name: Delete images + run: | + if [[ "$(docker images -q ${{ env.WORKER_IMAGE_TAG }} 2> /dev/null)" != "" ]]; then + docker image rmi --force ${{ env.WORKER_IMAGE_TAG }} 2>/dev/null + fi + if [[ "$(docker images -q ${{ env.CLIENT_IMAGE_TAG }} 2> /dev/null)" != "" ]]; then + docker image rmi --force ${{ env.CLIENT_IMAGE_TAG }} 2>/dev/null + fi + if [[ "$(docker images -q ${{ env.INTEGRITEE_NODE }} 2> /dev/null)" != "" ]]; then + docker image rmi --force ${{ env.INTEGRITEE_NODE }} 2>/dev/null + fi + docker images --all + release: + runs-on: ubuntu-latest name: Draft Release if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-22.04 needs: [build-test, integration-tests] outputs: release_url: ${{ steps.create-release.outputs.html_url }} diff --git a/.github/workflows/delete-release.yml b/.github/workflows/delete-release.yml index 71853e764b..b1d0e13750 100644 --- a/.github/workflows/delete-release.yml +++ b/.github/workflows/delete-release.yml @@ -7,7 +7,7 @@ on: jobs: purge-image: name: Delete image from ghcr.io - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: matrix: binary: ["integritee-client", "integritee-demo-validateer"] diff --git a/.github/workflows/label-checker.yml b/.github/workflows/label-checker.yml index fe264a6bcd..c645658ca3 100644 --- a/.github/workflows/label-checker.yml +++ b/.github/workflows/label-checker.yml @@ -8,7 +8,7 @@ on: jobs: check_for_matching_labels: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest if: github.base_ref == 'master' && github.event.pull_request.draft == false steps: - name: Label check diff --git a/.github/workflows/publish-docker-release.yml b/.github/workflows/publish-docker-release.yml index 1e4cf47d6a..a73674b256 100644 --- a/.github/workflows/publish-docker-release.yml +++ b/.github/workflows/publish-docker-release.yml @@ -8,7 +8,7 @@ on: jobs: main: name: Push Integritee Services to Dockerhub - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: matrix: binary: ["integritee-demo-validateer", "integritee-client"] diff --git a/.github/workflows/publish-draft-release.yml b/.github/workflows/publish-draft-release.yml index 337d07dcd1..11ac50ea33 100644 --- a/.github/workflows/publish-draft-release.yml +++ b/.github/workflows/publish-draft-release.yml @@ -8,7 +8,7 @@ on: jobs: publish-draft-release: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout sources uses: actions/checkout@v3 diff --git a/build.Dockerfile b/build.Dockerfile index 26c755826b..466c02ebe6 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -17,18 +17,21 @@ ### Builder Stage ################################################## -FROM integritee/integritee-dev:0.1.13 AS builder +FROM integritee/integritee-dev:0.2.1 AS builder LABEL maintainer="zoltan@integritee.network" # set environment variables ENV SGX_SDK /opt/sgxsdk -ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/root/.cargo/bin" +ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/opt/rust/bin//bin" ENV PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${SGX_SDK}/pkgconfig" ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/sdk_libs" ENV CARGO_NET_GIT_FETCH_WITH_CLI true -ENV SGX_MODE SW -ENV HOME=/root/work +# Default SGX MODE is software mode +ARG SGX_MODE=SW +ENV SGX_MODE=$SGX_MODE + +ENV HOME=/home/ubuntu/work ARG WORKER_MODE_ARG ENV WORKER_MODE=$WORKER_MODE_ARG @@ -49,24 +52,26 @@ RUN cargo test --release # A builder stage that uses sccache to speed up local builds with docker # Installation and setup of sccache should be moved to the integritee-dev image, so we don't # always need to compile and install sccache on CI (where we have no caching so far). -FROM integritee/integritee-dev:0.1.13 AS cached-builder +FROM integritee/integritee-dev:0.2.1 AS cached-builder LABEL maintainer="zoltan@integritee.network" # set environment variables ENV SGX_SDK /opt/sgxsdk -ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/root/.cargo/bin" +ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/opt/rust/bin/bin" ENV PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${SGX_SDK}/pkgconfig" ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/sdk_libs" ENV CARGO_NET_GIT_FETCH_WITH_CLI true -ENV SGX_MODE SW -ENV HOME=/root/work +# Default SGX MODE is software mode +ARG SGX_MODE=SW +ENV SGX_MODE=$SGX_MODE + +ENV HOME=/home/ubuntu/work -RUN rustup default stable && cargo install sccache --root /usr/local/cargo -ENV PATH "$PATH:/usr/local/cargo/bin" +RUN rustup default stable && cargo install sccache ENV SCCACHE_CACHE_SIZE="3G" ENV SCCACHE_DIR=$HOME/.cache/sccache -ENV RUSTC_WRAPPER="/usr/local/cargo/bin/sccache" +ENV RUSTC_WRAPPER="/opt/rust/bin/sccache" ARG WORKER_MODE_ARG ENV WORKER_MODE=$WORKER_MODE_ARG @@ -74,17 +79,15 @@ ENV WORKER_MODE=$WORKER_MODE_ARG WORKDIR $HOME/worker COPY . . -RUN --mount=type=cache,id=cargo,target=/root/work/.cache/sccache make && sccache --show-stats +RUN --mount=type=cache,id=cargo,target=${HOME}/.cache/sccache make && sccache --show-stats -RUN --mount=type=cache,id=cargo,target=/root/work/.cache/sccache cargo test --release && sccache --show-stats +RUN --mount=type=cache,id=cargo,target=${HOME}/.cache/sccache cargo test --release && sccache --show-stats ### Base Runner Stage -################################################## -FROM ubuntu:22.04 AS runner - -RUN apt update && apt install -y libssl-dev iproute2 curl - +### The runner needs the aesmd service for the `SGX_MODE=HW`. +###################################################### +FROM oasisprotocol/aesmd:master AS runner ### Deployed CLI client ################################################## @@ -97,7 +100,7 @@ ARG LOG_DIR=/usr/local/log ENV SCRIPT_DIR ${SCRIPT_DIR} ENV LOG_DIR ${LOG_DIR} -COPY --from=builder /root/work/worker/bin/integritee-cli /usr/local/bin +COPY --from=builder /home/ubuntu/work/worker/bin/integritee-cli /usr/local/bin COPY ./cli/*.sh /usr/local/worker-cli/ RUN chmod +x /usr/local/bin/integritee-cli ${SCRIPT_DIR}/*.sh @@ -114,13 +117,10 @@ ENTRYPOINT ["/usr/local/bin/integritee-cli"] FROM runner AS deployed-worker LABEL maintainer="zoltan@integritee.network" -ENV SGX_SDK /opt/sgxsdk -ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/lib64" - WORKDIR /usr/local/bin -COPY --from=builder /opt/sgxsdk/lib64 /opt/sgxsdk/lib64 -COPY --from=builder /root/work/worker/bin/* ./ +COPY --from=builder /opt/sgxsdk /opt/sgxsdk +COPY --from=builder /home/ubuntu/work/worker/bin/* ./ COPY --from=builder /lib/x86_64-linux-gnu/libsgx* /lib/x86_64-linux-gnu/ COPY --from=builder /lib/x86_64-linux-gnu/libdcap* /lib/x86_64-linux-gnu/ @@ -129,6 +129,8 @@ RUN chmod +x /usr/local/bin/integritee-service RUN ls -al /usr/local/bin # checks +ENV SGX_SDK /opt/sgxsdk +ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:$SGX_SDK/sdk_libs RUN ldd /usr/local/bin/integritee-service && \ /usr/local/bin/integritee-service --version diff --git a/docker/README.md b/docker/README.md index a823a52603..4bf3308c25 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,19 +5,21 @@ * Make sure you have installed Docker (version >= `2.0.0`) with [Docker Compose](https://docs.docker.com/compose/install/). On Windows, this can be Docker Desktop with WSL 2 integration. * In case you also build the worker directly, without docker (e.g. on a dev machine, running `make`), you should run `make clean` before running the docker build. Otherwise, it can occasionally lead to build errors. * The node image version that is loaded in the `docker-compose.yml`, (e.g. `image: "integritee/integritee-node-dev:1.0.32"`) needs to be compatible with the worker you're trying to build. +* Set export VERSION=dev +* `envsubst` should be installed, it is needed to replace the $VERSION in yaml files as docker compose doesn't support variables on service names. ## Building the Docker containers Run ``` -COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose build +COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f <(envsubst < docker-compose.yml) build ``` in this folder to build the worker image. This will build the worker from source and tag it in an image called `integritee-worker:dev`. ## Running the docker setup ``` -docker compose up +docker compose -f <(envsubst < docker-compose.yml) up ``` Starts all services (node and workers), using the `integritee-worker:dev` images you've built in the previous step. @@ -26,71 +28,71 @@ Starts all services (node and workers), using the `integritee-worker:dev` images ### Demo indirect invocation (M6) Build ``` -COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f docker-compose.yml -f demo-indirect-invocation.yml build --build-arg WORKER_MODE_ARG=offchain-worker +COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < demo-indirect-invocation.yml) build --build-arg WORKER_MODE_ARG=offchain-worker ``` Run ``` -docker compose -f docker-compose.yml -f demo-indirect-invocation.yml up demo-indirect-invocation --exit-code-from demo-indirect-invocation +FLAVOR_ID=offchain-worker docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < demo-indirect-invocation.yml) up demo-indirect-invocation --exit-code-from demo-indirect-invocation ``` ### Demo direct call (M8) Build ``` -COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f docker-compose.yml -f demo-direct-call.yml build --build-arg WORKER_MODE_ARG=sidechain +COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < demo-direct-call.yml) build --build-arg WORKER_MODE_ARG=sidechain ``` Run ``` -docker compose -f docker-compose.yml -f demo-direct-call.yml up demo-direct-call --exit-code-from demo-direct-call +docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < demo-direct-call.yml) up demo-direct-call --exit-code-from demo-direct-call ``` ### Demo sidechain Build ``` -COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f docker-compose.yml -f demo-sidechain.yml build --build-arg WORKER_MODE_ARG=sidechain +COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < demo-sidechain.yml) build --build-arg WORKER_MODE_ARG=sidechain ``` Run ``` -docker compose -f docker-compose.yml -f demo-sidechain.yml up demo-sidechain --exit-code-from demo-sidechain +docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < demo-sidechain.yml) up demo-sidechain --exit-code-from demo-sidechain ``` ### Demo Teeracle Build ``` -COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f docker-compose.yml -f demo-teeracle.yml build --build-arg WORKER_MODE_ARG=teeracle +COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < demo-teeracle.yml) build --build-arg WORKER_MODE_ARG=teeracle ``` Run ``` -docker compose -f docker-compose.yml -f demo-teeracle.yml up demo-teeracle --exit-code-from demo-teeracle +docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < demo-teeracle.yml) up demo-teeracle --exit-code-from demo-teeracle ``` ## Run the benchmarks Build with ``` -COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f docker-compose.yml -f sidechain-benchmark.yml build +COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < sidechain-benchmark.yml) build ``` and then run with ``` -docker compose -f docker-compose.yml -f sidechain-benchmark.yml up sidechain-benchmark --exit-code-from sidechain-benchmark +docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < sidechain-benchmark.yml) up sidechain-benchmark --exit-code-from sidechain-benchmark ``` ## Run the fork simulator The fork simulation uses `pumba` which in turn uses the Linux traffic control (TC). This is only available on Linux hosts, not on Windows with WSL unfortunately. Build the docker compose setup with ``` -COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f docker-compose.yml -f fork-inducer.yml -f demo-sidechain.yml build --build-arg WORKER_MODE_ARG=sidechain +COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < fork-inducer.yml) -f <(envsubst < demo-sidechain.yml) build --build-arg WORKER_MODE_ARG=sidechain ``` This requires the docker BuildKit (docker version >= 18.09) and support for it in docker compose (version >= 1.25.0) Run the 2-worker setup with a fork inducer (pumba) that delays the traffic on worker 2 ``` -docker compose -f docker-compose.yml -f fork-inducer.yml -f integration-test.yml up --exit-code-from demo-sidechain +docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < fork-inducer.yml) -f <(envsubst < integration-test.yml) up --exit-code-from demo-sidechain ``` This should show that the integration test fails, because we had an unhandled fork in the sidechain. Clean up the containers after each run with: ``` -docker compose -f docker-compose.yml -f fork-inducer.yml -f demo-sidechain.yml down +docker compose -f <(envsubst < docker-compose.yml) -f <(envsubst < fork-inducer.yml) -f <(envsubst < demo-sidechain.yml) down ``` We need these different compose files to separate the services that we're using. E.g. we want the integration test and fork simulator to be optional. The same could be solved using `profiles` - but that requires a more up-to-date version of `docker compose`. diff --git a/docker/demo-direct-call.yml b/docker/demo-direct-call.yml index a4b91e557e..abf5e59552 100644 --- a/docker/demo-direct-call.yml +++ b/docker/demo-direct-call.yml @@ -1,17 +1,21 @@ services: demo-direct-call: - image: integritee-cli:dev - container_name: integritee-direct-call-demo + image: integritee-cli:${VERSION:-dev} + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" build: - context: .. + context: ${PWD}/.. dockerfile: build.Dockerfile target: deployed-client depends_on: - integritee-node: + integritee-node-${VERSION}: condition: service_healthy - integritee-worker-1: + integritee-worker-1-${VERSION}: condition: service_healthy - integritee-worker-2: + integritee-worker-2-${VERSION}: condition: service_healthy networks: - integritee-test-network diff --git a/docker/demo-indirect-invocation.yml b/docker/demo-indirect-invocation.yml index c8caae8af3..e7eb1a0c20 100644 --- a/docker/demo-indirect-invocation.yml +++ b/docker/demo-indirect-invocation.yml @@ -1,17 +1,21 @@ services: demo-indirect-invocation: - image: integritee-cli:dev - container_name: integritee-indirect-invocation-demo + image: integritee-cli:${VERSION:-dev} + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" build: - context: .. + context: ${PWD}/.. dockerfile: build.Dockerfile target: deployed-client depends_on: - integritee-node: + integritee-node-${VERSION}: condition: service_healthy - integritee-worker-1: + integritee-worker-1-${VERSION}: condition: service_healthy - integritee-worker-2: + integritee-worker-2-${VERSION}: condition: service_healthy environment: - RUST_LOG=warn,ws=warn,itc_rpc_client=warn diff --git a/docker/demo-sidechain.yml b/docker/demo-sidechain.yml index b39b41f800..ad7779d9bf 100644 --- a/docker/demo-sidechain.yml +++ b/docker/demo-sidechain.yml @@ -1,17 +1,22 @@ services: demo-sidechain: - image: integritee-cli:dev - container_name: integritee-sidechain-demo + image: integritee-cli:${VERSION:-dev} + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" + build: - context: .. + context: ${PWD}/.. dockerfile: build.Dockerfile target: deployed-client depends_on: - integritee-node: + integritee-node-${VERSION}: condition: service_healthy - integritee-worker-1: + integritee-worker-1-${VERSION}: condition: service_healthy - integritee-worker-2: + integritee-worker-2-${VERSION}: condition: service_healthy networks: - integritee-test-network diff --git a/docker/demo-smart-contract.yml b/docker/demo-smart-contract.yml index 9c41b17001..7f1500bd71 100644 --- a/docker/demo-smart-contract.yml +++ b/docker/demo-smart-contract.yml @@ -1,17 +1,21 @@ services: demo-smart-contract: - image: integritee-cli:dev - container_name: integritee-smart-contract-demo + image: integritee-cli:${VERSION:-dev} + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" build: - context: .. + context: ${PWD}/.. dockerfile: build.Dockerfile target: deployed-client depends_on: - integritee-node: + integritee-node-${VERSION}: condition: service_healthy - integritee-worker-1: + integritee-worker-1-${VERSION}: condition: service_healthy - integritee-worker-2: + integritee-worker-2-${VERSION}: condition: service_healthy environment: - RUST_LOG=warn,ws=warn,itc_rpc_client=warn diff --git a/docker/demo-teeracle-generic.yml b/docker/demo-teeracle-generic.yml index 785b591f52..43a65d8411 100644 --- a/docker/demo-teeracle-generic.yml +++ b/docker/demo-teeracle-generic.yml @@ -3,15 +3,20 @@ # The demo is parameterized with the interval that the teeracle uses to query its sources. # Set the `TEERACLE_INTERVAL_SECONDS` variable when invoking, e.g. `TEERACLE_INTERVAL_SECONDS=4 docker compose -f docker-compose.yml -f demo-teeracle-generic.yml up --exit-code-from demo-teeracle-generic` services: - integritee-teeracle-worker: - image: integritee-worker:dev - container_name: integritee-teeracle-worker + integritee-teeracle-worker-${VERSION}: + image: integritee-worker:${VERSION:-dev} + hostname: integritee-teeracle-worker + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" build: - context: .. + context: ${PWD}/.. dockerfile: build.Dockerfile target: deployed-worker depends_on: - integritee-node: + integritee-node-${VERSION}: condition: service_healthy environment: - RUST_LOG=warn,ws=warn,sp_io=warn,substrate_api_client=warn,jsonrpsee_ws_client=warn,jsonrpsee_ws_server=warn,enclave_runtime=warn,integritee_service=info,integritee_service::teeracle=debug,ita_stf=warn,ita_oracle=debug @@ -28,16 +33,20 @@ services: run --dev --skip-ra --teeracle-interval ${TEERACLE_INTERVAL_SECONDS}s" restart: always demo-teeracle-generic: - image: integritee-cli:dev - container_name: integritee-teeracle-demo + image: integritee-cli:${VERSION:-dev} + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" build: - context: .. + context: ${PWD}/.. dockerfile: build.Dockerfile target: deployed-client depends_on: - integritee-node: + integritee-node-${VERSION}: condition: service_healthy - integritee-teeracle-worker: + integritee-teeracle-worker-${VERSION}: condition: service_healthy environment: - RUST_LOG=warn,sp_io=warn,integritee_cli::exchange_oracle=debug diff --git a/docker/demo-teeracle.yml b/docker/demo-teeracle.yml index 4b199eb862..580281c483 100644 --- a/docker/demo-teeracle.yml +++ b/docker/demo-teeracle.yml @@ -5,15 +5,20 @@ # This setup requires an API key for CoinMarketCap # Add the API key to the environment variable `COINMARKETCAP_KEY`, with `export COINMARKETCAP_KEY=` services: - integritee-teeracle-worker: - image: integritee-worker:dev - container_name: integritee-teeracle-worker + integritee-teeracle-worker-${VERSION}: + image: integritee-worker:${VERSION:-dev} + hostname: integritee-teeracle-worker + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" build: - context: .. + context: ${PWD}/.. dockerfile: build.Dockerfile target: deployed-worker depends_on: - integritee-node: + integritee-node-${VERSION}: condition: service_healthy environment: - RUST_LOG=warn,ws=warn,sp_io=warn,substrate_api_client=warn,jsonrpsee_ws_client=warn,jsonrpsee_ws_server=warn,enclave_runtime=warn,integritee_service=info,integritee_service::teeracle=debug,ita_stf=warn,ita_exchange_oracle=debug @@ -31,16 +36,20 @@ services: run --dev --skip-ra --teeracle-interval ${TEERACLE_INTERVAL_SECONDS}s" restart: always demo-teeracle: - image: integritee-cli:dev - container_name: integritee-teeracle-demo + image: integritee-cli:${VERSION:-dev} + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" build: - context: .. + context: ${PWD}/.. dockerfile: build.Dockerfile target: deployed-client depends_on: - integritee-node: + integritee-node-${VERSION}: condition: service_healthy - integritee-teeracle-worker: + integritee-teeracle-worker-${VERSION}: condition: service_healthy environment: - RUST_LOG=warn,sp_io=warn,integritee_cli::exchange_oracle=debug diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index af48c65803..01e9d59aa7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,7 +1,12 @@ services: - integritee-node: - image: "integritee/integritee-node-dev:1.0.33" - container_name: integritee-node + integritee-node-${VERSION}: + image: "${INTEGRITEE_NODE:-integritee/integritee-node-dev:1.0.33}" + hostname: integritee-node + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" networks: - integritee-test-network healthcheck: @@ -12,16 +17,21 @@ services: command: --dev --rpc-methods unsafe --ws-external --rpc-external --ws-port 9912 #logging: #driver: local - integritee-worker-1: - image: integritee-worker:dev - container_name: integritee-worker-1 + integritee-worker-1-${VERSION}: + image: integritee-worker:${VERSION:-dev} + hostname: integritee-worker-1 build: - context: .. + context: ${PWD}/.. dockerfile: build.Dockerfile target: deployed-worker depends_on: - integritee-node: + integritee-node-${VERSION}: condition: service_healthy + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" environment: - RUST_LOG=warn,ws=warn,sp_io=warn,substrate_api_client=warn,jsonrpsee_ws_client=warn,jsonrpsee_ws_server=warn,enclave_runtime=warn,integritee_service=warn,ita_stf=warn networks: @@ -36,18 +46,23 @@ services: -u ws://integritee-node -U ws://integritee-worker-1 -P 2011 -w 2101 -p 9912 -h 4645 run --dev --skip-ra" restart: "no" - integritee-worker-2: - image: integritee-worker:dev - container_name: integritee-worker-2 + integritee-worker-2-${VERSION}: + image: integritee-worker:${VERSION:-dev} + hostname: integritee-worker-2 build: - context: .. + context: ${PWD}/.. dockerfile: build.Dockerfile target: deployed-worker depends_on: - integritee-node: + integritee-node-${VERSION}: condition: service_healthy - integritee-worker-1: + integritee-worker-1-${VERSION}: condition: service_healthy + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" environment: - RUST_LOG=warn,ws=warn,sp_io=warn,substrate_api_client=warn,jsonrpsee_ws_client=warn,jsonrpsee_ws_server=warn,enclave_runtime=warn,integritee_service=warn,ita_stf=warn networks: @@ -64,4 +79,4 @@ services: restart: "no" networks: integritee-test-network: - driver: bridge + driver: bridge \ No newline at end of file diff --git a/docker/fork-inducer.yml b/docker/fork-inducer.yml index 639fb2c759..47d00ac5ba 100644 --- a/docker/fork-inducer.yml +++ b/docker/fork-inducer.yml @@ -1,25 +1,34 @@ services: worker-ping: - image: worker-ping:dev + image: worker-ping:${VERSION:-dev} + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" build: context: . dockerfile: ping.Dockerfile - depends_on: [ 'integritee-node', 'integritee-worker-1', 'integritee-worker-2' ] + depends_on: [ 'integritee-node-${VERSION}', 'integritee-worker-1-${VERSION}', 'integritee-worker-2-${VERSION}' ] networks: - integritee-test-network - #entrypoint: "ping integritee-worker-2 | while read pong; do echo \"$$(date): $$pong\"; done" entrypoint: "ping integritee-worker-2" pumba-network-delay: - image: integritee-fork-producer:dev + image: integritee-fork-producer:${VERSION:-dev} + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" build: context: . dockerfile: fork.Dockerfile depends_on: - integritee-node: + integritee-node-${VERSION}: condition: service_healthy - integritee-worker-1: + integritee-worker-1-${VERSION}: condition: service_healthy - integritee-worker-2: + integritee-worker-2-${VERSION}: condition: service_healthy networks: - integritee-test-network diff --git a/docker/sidechain-benchmark.yml b/docker/sidechain-benchmark.yml index eca0659fb3..7cf5ca8c4c 100644 --- a/docker/sidechain-benchmark.yml +++ b/docker/sidechain-benchmark.yml @@ -1,17 +1,21 @@ services: sidechain-benchmark: - image: integritee-cli:dev - container_name: integritee-benchmark + image: integritee-cli:${VERSION:-dev} + devices: + - "${SGX_PROVISION:-/dev/null}:/dev/sgx/provision" + - "${SGX_ENCLAVE:-/dev/null}:/dev/sgx/enclave" + volumes: + - "${AESMD:-/dev/null}:/var/run/aesmd" build: - context: .. + context: ${PWD}/.. dockerfile: build.Dockerfile target: deployed-client depends_on: - integritee-node: + integritee-node-${VERSION}: condition: service_healthy - integritee-worker-1: + integritee-worker-1-${VERSION}: condition: service_healthy - integritee-worker-2: + integritee-worker-2-${VERSION}: condition: service_healthy networks: - integritee-test-network