Sgx runner

.github/workflows/build_and_test.yml

@@ -5,12 +5,14 @@ on:
   push:
     branches:
       - master
+      - sgx_runner
       - 'sdk-v[0-9]+.[0-9]+.[0-9]+-*'
     tags:
       - '[0-9]+.[0-9]+.[0-9]+'
   pull_request:
     branches:
       - master
+      - sgx_runner
       - 'sdk-v[0-9]+.[0-9]+.[0-9]+-*'
 
 env:
@@ -21,14 +23,14 @@ env:
 jobs:
   cancel_previous_runs:
     name: Cancel Previous Runs
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: styfle/cancel-workflow-action@0.11.0
         with:
           access_token: ${{ secrets.GITHUB_TOKEN }}
 
   build-test:
-    runs-on: ubuntu-22.04
+    runs-on: self-hosted
     strategy:
       fail-fast: false
       matrix:
@@ -58,7 +60,7 @@ jobs:
         run: >
           docker build -t integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }}
           --target deployed-worker
-          --build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }}
+          --build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} --build-arg SGX_MODE=HW
           -f build.Dockerfile .
 
       - name: Build CLI client
@@ -67,13 +69,13 @@ jobs:
         run: >
           docker build -t integritee-cli-client-${{ matrix.flavor_id }}-${{ github.sha }}
           --target deployed-client
-          --build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }}
+          --build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} --build-arg SGX_MODE=HW
           -f build.Dockerfile .
 
       - run: docker images --all
 
       - name: Test Enclave # cargo test is not supported in the enclave, see: https://github.com/apache/incubator-teaclave-sgx-sdk/issues/232
-        run: docker run --name ${{ env.BUILD_CONTAINER_NAME }} integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }} test --all
+        run: docker run --device=/dev/sgx/enclave --device=/dev/sgx/provision -v /var/run/aesmd:/var/run/aesmd integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }} test --all
 
       - name: Export worker image(s)
         run: |
@@ -94,7 +96,7 @@ jobs:
 
   clippy:
     runs-on: ubuntu-22.04
-    container: "integritee/integritee-dev:0.1.13"
+    container: "integritee/integritee-dev:0.2"
     steps:
       - uses: actions/checkout@v3
       - name: init rust
@@ -152,14 +154,14 @@ jobs:
         uses: andymckay/cancel-action@0.2
 
   integration-tests:
-    runs-on: ubuntu-22.04
+    runs-on: self-hosted
     if: ${{ always() }}
     needs: build-test
     env:
       WORKER_IMAGE_TAG: integritee-worker:dev
       CLIENT_IMAGE_TAG: integritee-cli:dev
       COINMARKETCAP_KEY: ${{ secrets.COINMARKETCAP_KEY }}
-      TEERACLE_INTERVAL_SECONDS: 4
+      TEERACLE_INTERVAL_SECONDS: 10
 
     strategy:
       fail-fast: false
@@ -219,19 +221,25 @@ jobs:
 
       - name: Re-name Image Tags
         run: |
+          if [[ "$(docker images -q ${{ env.WORKER_IMAGE_TAG }} 2> /dev/null)" == "" ]]; then
+             docker image rmi --force ${{ env.WORKER_IMAGE_TAG }} 2>/dev/null
+          fi
+          if [[ "$(docker images -q ${{ env.CLIENT_IMAGE_TAG }} 2> /dev/null)" == "" ]]; then
+             docker image rmi --force ${{ env.CLIENT_IMAGE_TAG }} 2>/dev/null
+          fi
           docker tag integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }} ${{ env.WORKER_IMAGE_TAG }}
           docker tag integritee-cli-client-${{ matrix.flavor_id }}-${{ github.sha }} ${{ env.CLIENT_IMAGE_TAG }}
           docker images --all
 
-      - name: Integration Test ${{ matrix.test }}-${{ matrix.flavor_id }}
+      - name: Stop docker containers
         run: |
           cd docker
-          docker compose -f docker-compose.yml -f ${{ matrix.demo_name }}.yml up ${{ matrix.demo_name }} --no-build --exit-code-from ${{ matrix.demo_name }}
+          docker compose -f docker-compose.yml -f ${{ matrix.demo_name }}.yml down
 
-      - name: Stop docker containers
+      - name: Integration Test ${{ matrix.test }}-${{ matrix.flavor_id }}
         run: |
           cd docker
-          docker compose -f docker-compose.yml -f ${{ matrix.demo_name }}.yml stop
+          docker compose -f docker-compose.yml -f ${{ matrix.demo_name }}.yml up ${{ matrix.demo_name }} --no-build --exit-code-from ${{ matrix.demo_name }} --remove-orphans
 
       - name: Collect Docker Logs
         continue-on-error: true
@@ -249,10 +257,17 @@ jobs:
           name: logs-${{ matrix.test }}-${{ matrix.flavor_id }}
           path: ./${{ env.LOG_DIR }}
 
+      - name: Stop docker containers
+        if: always()
+        run: |
+          cd docker
+          docker compose -f docker-compose.yml -f ${{ matrix.demo_name }}.yml down
+
+
   release:
+    runs-on: self-hosted
     name: Draft Release
     if: startsWith(github.ref, 'refs/tags/')
-    runs-on: ubuntu-22.04
     needs: [build-test, integration-tests]
     outputs:
       release_url: ${{ steps.create-release.outputs.html_url }}

.github/workflows/delete-release.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   purge-image:
     name: Delete image from ghcr.io
-    runs-on: ubuntu-22.04
+    runs-on: self-hosted
     strategy:
       matrix:
         binary: ["integritee-client", "integritee-demo-validateer"]

.github/workflows/label-checker.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
     check_for_matching_labels:
-      runs-on: ubuntu-22.04
+      runs-on: self-hosted
       if: github.base_ref == 'master' && github.event.pull_request.draft == false
       steps:
         - name: Label check

.github/workflows/publish-docker-release.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   main:
     name: Push Integritee Services to Dockerhub
-    runs-on: ubuntu-22.04
+    runs-on: self-hosted
     strategy:
       matrix:
         binary: ["integritee-demo-validateer", "integritee-client"]

.github/workflows/publish-draft-release.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   publish-draft-release:
-    runs-on: ubuntu-22.04
+    runs-on: self-hosted
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3

build.Dockerfile

@@ -17,18 +17,21 @@
 
 ### Builder Stage
 ##################################################
-FROM integritee/integritee-dev:0.1.13 AS builder
+FROM integritee/integritee-dev:0.2.1 AS builder
 LABEL maintainer="zoltan@integritee.network"
 
 # set environment variables
 ENV SGX_SDK /opt/sgxsdk
-ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/root/.cargo/bin"
+ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/opt/rust/bin//bin"
 ENV PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${SGX_SDK}/pkgconfig"
 ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/sdk_libs"
 ENV CARGO_NET_GIT_FETCH_WITH_CLI true
-ENV SGX_MODE SW
 
-ENV HOME=/root/work
+# Default SGX MODE is software mode
+ARG SGX_MODE=SW
+ENV SGX_MODE=$SGX_MODE
+
+ENV HOME=/home/ubuntu/work
 
 ARG WORKER_MODE_ARG
 ENV WORKER_MODE=$WORKER_MODE_ARG
@@ -49,42 +52,41 @@ RUN cargo test --release
 # A builder stage that uses sccache to speed up local builds with docker
 # Installation and setup of sccache should be moved to the integritee-dev image, so we don't
 # always need to compile and install sccache on CI (where we have no caching so far).
-FROM integritee/integritee-dev:0.1.13 AS cached-builder
+FROM integritee/integritee-dev:0.2.1 AS cached-builder
 LABEL maintainer="zoltan@integritee.network"
 
 # set environment variables
 ENV SGX_SDK /opt/sgxsdk
-ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/root/.cargo/bin"
+ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/opt/rust/bin/bin"
 ENV PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${SGX_SDK}/pkgconfig"
 ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/sdk_libs"
 ENV CARGO_NET_GIT_FETCH_WITH_CLI true
-ENV SGX_MODE SW
 
-ENV HOME=/root/work
+# Default SGX MODE is software mode
+ARG SGX_MODE=SW
+ENV SGX_MODE=$SGX_MODE
+
+ENV HOME=/home/ubuntu/work
 
-RUN rustup default stable && cargo install sccache --root /usr/local/cargo
-ENV PATH "$PATH:/usr/local/cargo/bin"
+RUN rustup default stable && cargo install sccache
 ENV SCCACHE_CACHE_SIZE="3G"
 ENV SCCACHE_DIR=$HOME/.cache/sccache
-ENV RUSTC_WRAPPER="/usr/local/cargo/bin/sccache"
+ENV RUSTC_WRAPPER="/opt/rust/bin/sccache"
 
 ARG WORKER_MODE_ARG
 ENV WORKER_MODE=$WORKER_MODE_ARG
 
 WORKDIR $HOME/worker
 COPY . .
 
-RUN --mount=type=cache,id=cargo,target=/root/work/.cache/sccache make && sccache --show-stats
+RUN --mount=type=cache,id=cargo,target=${HOME}/.cache/sccache make && sccache --show-stats
 
-RUN --mount=type=cache,id=cargo,target=/root/work/.cache/sccache cargo test --release && sccache --show-stats
+RUN --mount=type=cache,id=cargo,target=${HOME}/.cache/sccache cargo test --release && sccache --show-stats
 
 
 ### Base Runner Stage
 ##################################################
-FROM ubuntu:22.04 AS runner
-
-RUN apt update && apt install -y libssl-dev iproute2 curl
-
+FROM oasisprotocol/aesmd:master AS runner
 
 ### Deployed CLI client
 ##################################################
@@ -97,7 +99,7 @@ ARG LOG_DIR=/usr/local/log
 ENV SCRIPT_DIR ${SCRIPT_DIR}
 ENV LOG_DIR ${LOG_DIR}
 
-COPY --from=builder /root/work/worker/bin/integritee-cli /usr/local/bin
+COPY --from=builder /home/ubuntu/work/worker/bin/integritee-cli /usr/local/bin
 COPY ./cli/*.sh /usr/local/worker-cli/
 
 RUN chmod +x /usr/local/bin/integritee-cli ${SCRIPT_DIR}/*.sh
@@ -114,13 +116,10 @@ ENTRYPOINT ["/usr/local/bin/integritee-cli"]
 FROM runner AS deployed-worker
 LABEL maintainer="zoltan@integritee.network"
 
-ENV SGX_SDK /opt/sgxsdk
-ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/lib64"
-
 WORKDIR /usr/local/bin
 
 COPY --from=builder /opt/sgxsdk/lib64 /opt/sgxsdk/lib64
-COPY --from=builder /root/work/worker/bin/* ./
+COPY --from=builder /home/ubuntu/work/worker/bin/* ./
 COPY --from=builder /lib/x86_64-linux-gnu/libsgx* /lib/x86_64-linux-gnu/
 COPY --from=builder /lib/x86_64-linux-gnu/libdcap* /lib/x86_64-linux-gnu/
 

docker/demo-direct-call.yml

@@ -2,6 +2,11 @@ services:
   demo-direct-call:
     image: integritee-cli:dev
     container_name: integritee-direct-call-demo
+    devices:
+      - "/dev/sgx/provision:/dev/sgx/provision"
+      - "/dev/sgx/enclave:/dev/sgx/enclave"
+    volumes:
+      - "/var/run/aesmd:/var/run/aesmd"
     build:
       context: ..
       dockerfile: build.Dockerfile

docker/demo-indirect-invocation.yml

@@ -2,6 +2,11 @@ services:
   demo-indirect-invocation:
     image: integritee-cli:dev
     container_name: integritee-indirect-invocation-demo
+    devices:
+      - "/dev/sgx/provision:/dev/sgx/provision"
+      - "/dev/sgx/enclave:/dev/sgx/enclave"
+    volumes:
+      - "/var/run/aesmd:/var/run/aesmd"
     build:
       context: ..
       dockerfile: build.Dockerfile

docker/demo-sidechain.yml

@@ -2,6 +2,11 @@ services:
   demo-sidechain:
     image: integritee-cli:dev
     container_name: integritee-sidechain-demo
+    devices:
+      - "/dev/sgx/provision:/dev/sgx/provision"
+      - "/dev/sgx/enclave:/dev/sgx/enclave"
+    volumes:
+      - "/var/run/aesmd:/var/run/aesmd"
     build:
       context: ..
       dockerfile: build.Dockerfile

docker/demo-smart-contract.yml

@@ -2,6 +2,11 @@ services:
   demo-smart-contract:
     image: integritee-cli:dev
     container_name: integritee-smart-contract-demo
+    devices:
+      - "/dev/sgx/provision:/dev/sgx/provision"
+      - "/dev/sgx/enclave:/dev/sgx/enclave"
+    volumes:
+      - "/var/run/aesmd:/var/run/aesmd"
     build:
       context: ..
       dockerfile: build.Dockerfile

docker/demo-teeracle-generic.yml

@@ -6,6 +6,11 @@ services:
   integritee-teeracle-worker:
     image: integritee-worker:dev
     container_name: integritee-teeracle-worker
+    devices:
+      - "/dev/sgx/provision:/dev/sgx/provision"
+      - "/dev/sgx/enclave:/dev/sgx/enclave"
+    volumes:
+      - "/var/run/aesmd:/var/run/aesmd"
     build:
       context: ..
       dockerfile: build.Dockerfile
@@ -29,7 +34,12 @@ services:
     restart: always
   demo-teeracle-generic:
     image: integritee-cli:dev
-    container_name: integritee-teeracle-demo
+    container_name: integritee-teeracle-generic-demo
+    devices:
+      - "/dev/sgx/provision:/dev/sgx/provision"
+      - "/dev/sgx/enclave:/dev/sgx/enclave"
+    volumes:
+      - "/var/run/aesmd:/var/run/aesmd"
     build:
       context: ..
       dockerfile: build.Dockerfile

docker/demo-teeracle.yml

@@ -8,6 +8,11 @@ services:
   integritee-teeracle-worker:
     image: integritee-worker:dev
     container_name: integritee-teeracle-worker
+    devices:
+      - "/dev/sgx/provision:/dev/sgx/provision"
+      - "/dev/sgx/enclave:/dev/sgx/enclave"
+    volumes:
+      - "/var/run/aesmd:/var/run/aesmd"
     build:
       context: ..
       dockerfile: build.Dockerfile
@@ -33,6 +38,11 @@ services:
   demo-teeracle:
     image: integritee-cli:dev
     container_name: integritee-teeracle-demo
+    devices:
+      - "/dev/sgx/provision:/dev/sgx/provision"
+      - "/dev/sgx/enclave:/dev/sgx/enclave"
+    volumes:
+      - "/var/run/aesmd:/var/run/aesmd"
     build:
       context: ..
       dockerfile: build.Dockerfile

docker/docker-compose.yml

@@ -15,6 +15,11 @@ services:
   integritee-worker-1:
     image: integritee-worker:dev
     container_name: integritee-worker-1
+    devices:
+      - "/dev/sgx/provision:/dev/sgx/provision"
+      - "/dev/sgx/enclave:/dev/sgx/enclave"
+    volumes:
+      - "/var/run/aesmd:/var/run/aesmd"
     build:
       context: ..
       dockerfile: build.Dockerfile
@@ -39,6 +44,11 @@ services:
   integritee-worker-2:
     image: integritee-worker:dev
     container_name: integritee-worker-2
+    devices:
+      - "/dev/sgx/provision:/dev/sgx/provision"
+      - "/dev/sgx/enclave:/dev/sgx/enclave"
+    volumes:
+      - "/var/run/aesmd:/var/run/aesmd"
     build:
       context: ..
       dockerfile: build.Dockerfile