Replace python with docker

.github/workflows/build_and_test.yml

@@ -11,10 +11,7 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
-  WORKER_BIN: integritee-service
-  CLIENT_BIN: integritee-cli
-  ENCLAVE_BIN: enclave.signed.so
-  LOG_DIR: log
+  LOG_DIR: logs
   BUILD_CONTAINER_NAME: integritee_worker_enclave_test
 
 jobs:
@@ -46,10 +43,11 @@ jobs:
           path: litentry-parachain.tar
 
   build-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
+      fail-fast: false
       matrix:
-        mode: [ sidechain, offchain-worker ]
+        mode: [ sidechain, offchain-worker, teeracle ]
 
     steps:
       - uses: actions/checkout@v3
@@ -58,57 +56,45 @@ jobs:
         uses: docker/setup-buildx-action@v2
         with:
           buildkitd-flags: --debug
+          driver: docker-container
 
-      - name: Build Enclave Test Image
+      - name: Build Worker & Run Cargo Test
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -t integritee-worker-enclave-test-${{ matrix.mode }}-${{ github.sha }} --target enclave-test --build-arg WORKER_MODE_ARG=${{ matrix.mode }} -f build.Dockerfile .
+        run: >
+          docker build -t integritee-worker-${{ matrix.mode }}-${{ github.sha }} 
+          --target deployed-worker --build-arg WORKER_MODE_ARG=${{ matrix.mode }} 
+          -f build.Dockerfile .
 
-      - name: Test Enclave # cargo test is not supported, see: https://github.com/apache/incubator-teaclave-sgx-sdk/issues/232
-        run: docker run --name ${{ env.BUILD_CONTAINER_NAME }} integritee-worker-enclave-test-${{ matrix.mode }}-${{ github.sha }}
-
-      - name: Build Cargo Test Image
+      - name: Build CLI client
         env:
           DOCKER_BUILDKIT: 1
-        run:  docker build -t integritee-worker-ctest-${{ matrix.mode }}-${{ github.sha }} --target cargo-test --build-arg WORKER_MODE_ARG=${{ matrix.mode }} -f build.Dockerfile .
+        run: >
+          docker build -t integritee-cli-client-${{ matrix.mode }}-${{ github.sha }} 
+          --target deployed-client --build-arg WORKER_MODE_ARG=${{ matrix.mode }} 
+          -f build.Dockerfile .
 
-      - name: Run Cargo Test
-        run: docker run --rm integritee-worker-ctest-${{ matrix.mode }}-${{ github.sha }}
+      - run: docker images --all
 
-      - name: Build Deployable Image
-        env:
-          DOCKER_BUILDKIT: 1
-        run: docker build --output=type=tar,dest=/tmp/integritee-worker.tar --target=deployed-worker --build-arg WORKER_MODE_ARG=${{ matrix.mode }} -f build.Dockerfile .
-
-      - name: Copy artifacts from container
-        run: |
-          docker cp ${{ env.BUILD_CONTAINER_NAME }}:/root/work/worker/bin/${{ env.WORKER_BIN }} .
-          docker cp ${{ env.BUILD_CONTAINER_NAME }}:/root/work/worker/bin/${{ env.CLIENT_BIN }} .
-          docker cp ${{ env.BUILD_CONTAINER_NAME }}:/root/work/worker/bin/${{ env.ENCLAVE_BIN }} .
-
-      - name: Upload worker
-        uses: actions/upload-artifact@v2
-        with:
-          name: integritee-worker-${{ matrix.mode }}-${{ github.sha }}
-          path: ${{ env.WORKER_BIN }}
+      - name: Test Enclave # cargo test is not supported in the enclave, see: https://github.com/apache/incubator-teaclave-sgx-sdk/issues/232
+        run: docker run --name ${{ env.BUILD_CONTAINER_NAME }} integritee-worker-${{ matrix.mode }}-${{ github.sha }} test --all
 
-      - name: Upload client
-        uses: actions/upload-artifact@v2
-        with:
-          name: integritee-client-${{ matrix.mode }}-${{ github.sha }}
-          path: ${{ env.CLIENT_BIN }}
+      - name: Export worker image(s)
+        run: |
+          docker image save integritee-worker-${{ matrix.mode }}-${{ github.sha }} | gzip > integritee-worker-${{ matrix.mode }}-${{ github.sha }}.tar.gz
+          docker image save integritee-cli-client-${{ matrix.mode }}-${{ github.sha }} | gzip > integritee-cli-client-${{ matrix.mode }}-${{ github.sha }}.tar.gz
 
-      - name: Upload enclave
+      - name: Upload worker image
         uses: actions/upload-artifact@v2
         with:
-          name: enclave-signed-${{ matrix.mode }}-${{ github.sha }}
-          path: ${{ env.ENCLAVE_BIN }}
+          name: integritee-worker-${{ matrix.mode }}-${{ github.sha }}.tar.gz
+          path: integritee-worker-${{ matrix.mode }}-${{ github.sha }}.tar.gz
 
-      - name: Upload deployable image
+      - name: Upload CLI client image
         uses: actions/upload-artifact@v2
         with:
-          name: integritee-worker-${{ matrix.mode }}-image-${{ github.sha }}
-          path: /tmp/integritee-worker.tar
+          name: integritee-cli-client-${{ matrix.mode }}-${{ github.sha }}.tar.gz
+          path: integritee-cli-client-${{ matrix.mode }}-${{ github.sha }}.tar.gz
 
   clippy:
     runs-on: ubuntu-latest
@@ -145,41 +131,45 @@ jobs:
         uses: andymckay/cancel-action@0.2
 
   integration-tests:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     needs:
       - build-parachain-docker
       - build-test
+    env:
+      WORKER_IMAGE_TAG: integritee-worker:dev
+      CLIENT_IMAGE_TAG: integritee-cli:dev
+      COINMARKETCAP_KEY: ${{ secrets.COINMARKETCAP_KEY }}
+      TEERACLE_INTERVAL_SECONDS: 4
+
     strategy:
       fail-fast: false
       matrix:
         include:
           - test: M6
             mode: sidechain
-            demo_name: m6_demo_shielding_unshielding
-            demo_script: m6.sh
+            demo_name: demo-indirect-invocation
           - test: M8
             mode: sidechain
-            demo_name: m8_demo_direct_call
-            demo_script: m8.sh
+            demo_name: demo-direct-call
           - test: Sidechain
             mode: sidechain
-            demo_name: sidechain_demo
-            demo_script: sidechain.sh
+            demo_name: demo-sidechain
           - test: M6
             mode: offchain-worker
-            demo_name: m6_demo_shielding_unshielding
-            demo_script: m6.sh
+            demo_name: demo-indirect-invocation
+          - test: Teeracle
+            mode: teeracle
+            demo_name: demo-teeracle
+          - test: Benchmark
+            mode: sidechain
+            demo_name: sidechain-benchmark
 
-    env:
-      BIN_DIR: bin
     steps:
       - uses: actions/checkout@v3
 
-      # acutally just need the PSW,
-      # but I simply run the script here
-      - name: Setup
+      - name: Pull polkadot image
         run: |
-          . ./scripts/litentry/ubuntu_setup.sh
+          docker pull parity/polkadot:latest
 
       - uses: actions/download-artifact@v3
         with:
@@ -189,137 +179,58 @@ jobs:
         run: |
           docker load -i litentry-parachain.tar
 
-      - uses: actions/setup-python@v3
-        with:
-          python-version: '3.7'
-
-      - name: Download Worker
+      - name: Download Worker Image
         uses: actions/download-artifact@v2
         with:
-          name: integritee-worker-${{ matrix.mode }}-${{ github.sha }}
-          path: ${{ env.BIN_DIR }}
+          name: integritee-worker-${{ matrix.mode }}-${{ github.sha }}.tar.gz
+          path: .
 
-      - name: Download Client
+      - name: Download CLI client Image
         uses: actions/download-artifact@v2
         with:
-          name: integritee-client-${{ matrix.mode }}-${{ github.sha }}
-          path: ${{ env.BIN_DIR }}
+          name: integritee-cli-client-${{ matrix.mode }}-${{ github.sha }}.tar.gz
+          path: .
 
-      - name: Download Enclave
-        uses: actions/download-artifact@v2
-        with:
-          name: enclave-signed-${{ matrix.mode }}-${{ github.sha }}
-          path: ${{ env.BIN_DIR }}
-
-        # If you want to debug ci and you don't want to build the binaries, you can replace the downloads above with
-        # the actions below to download binaries from an earlier run.
-#      - name: Download Worker
-#        uses: dawidd6/action-download-artifact@v2
-#        with:
-#          github_token: ${{secrets.GITHUB_TOKEN}}
-#          workflow: build_and_test.yml
-#          run_id: 1033249727
-#          name: integritee-worker-da8d4b442d3f2b09dbafb097d4d7a1bce409d518
-#          path: ${{ env.BIN_DIR }}
-#
-#      - name: Download Client
-#        uses: dawidd6/action-download-artifact@v2
-#        with:
-#          github_token: ${{secrets.GITHUB_TOKEN}}
-#          workflow: build_and_test.yml
-#          run_id: 1033249727
-#          name: integritee-client-da8d4b442d3f2b09dbafb097d4d7a1bce409d518
-#          path: ${{ env.BIN_DIR }}
-#
-#      - name: Download Enclave
-#        uses: dawidd6/action-download-artifact@v2
-#        with:
-#          github_token: ${{secrets.GITHUB_TOKEN}}
-#          workflow: build_and_test.yml
-#          run_id: 1033249727
-#          name: enclave-signed-da8d4b442d3f2b09dbafb097d4d7a1bce409d518
-#          path: ${{ env.BIN_DIR }}
-
-      # - name: Download integritee-node
-      #   uses: dawidd6/action-download-artifact@v2
-      #   with:
-      #     github_token: ${{secrets.GITHUB_TOKEN}}
-      #     workflow: ci.yml
-      #     name: integritee-node-dev-5da191f98425a3217df413e89126e8c6f7efcb8a
-      #     # in fact this action should download the latest artifact, but sometimes fails. Then we need to
-      #     # set the `run_id` to force a download of an updated binary.
-      #     run_id: 2307398354
-      #     path: node
-      #     repo: integritee-network/integritee-node
-
-      - name: Prepare working directory
+      - name: Load Worker & Client Images
+        env:
+          DOCKER_BUILDKIT: 1
         run: |
-          mkdir -p ${{ env.LOG_DIR}}
-          cd ${{ env.BIN_DIR }}
-          chmod +x ${{ env.WORKER_BIN }}
-          chmod +x ${{ env.CLIENT_BIN }}
-          chmod +x ${{ env.ENCLAVE_BIN }}
+          docker image load --input integritee-worker-${{ matrix.mode }}-${{ github.sha }}.tar.gz
+          docker image load --input integritee-cli-client-${{ matrix.mode }}-${{ github.sha }}.tar.gz
+          docker images --all
 
-      - name: "Setup Keys"
-        env:
-          KEY: ${{ secrets.IAS_PRIMARY_KEY }}
-          SPID: ${{ secrets.IAS_SPID }}
-          TLS_CERTIFICATE: ${{ secrets.TLS_WS_SERVER_CERTIFICATE }}
-          TLS_PRIVATE_KEY: ${{ secrets.TLS_WS_SERVER_PRIVATE_KEY }}
+      - name: Re-name Image Tags
         run: |
-          cd ${{ env.BIN_DIR }}
-          echo "${{ env.KEY }}" > key.txt
-          echo "${{ env.SPID }}" > spid.txt
-          echo "${{ env.TLS_CERTIFICATE }}" > end.fullchain
-          echo "${{ env.TLS_PRIVATE_KEY }}" > end.rsa
-          chmod 644 end.fullchain
-          chmod 644 end.rsa
-
-      - name: Run local setup
-        # * `set -eo pipefail` is needed to return an error even if piped to `tee`.
-        shell: bash --noprofile --norc -eo pipefail {0}
+          docker tag integritee-worker-${{ matrix.mode }}-${{ github.sha }} ${{ env.WORKER_IMAGE_TAG }}
+          docker tag integritee-cli-client-${{ matrix.mode }}-${{ github.sha }} ${{ env.CLIENT_IMAGE_TAG }} 
+          docker images --all
+
+      - name: Build litentry parachain docker images
         run: |
-          touch ${{ env.LOG_DIR }}/local-setup.log
-          ./local-setup/launch.py local-setup/github-action-config.json 2>&1 | tee -i ${{ env.LOG_DIR }}/local-setup.log &
-          cnt=$(jq ' .workers | length' local-setup/github-action-config.json)
-          for i in $(seq 1 60); do
-            sleep 10
-            found=true
-            for j in $(seq 1 $cnt); do
-              if [ ! -f ${{ env.LOG_DIR }}/worker${j}.log ] || ! grep -Fq 'finalized parentchain blocks' ${{ env.LOG_DIR }}/worker${j}.log; then
-                found=false
-              fi
-            done
-            if [ "$found" = "true" ]; then
-              break
-            fi
-          done
-          echo "all workers synced to parent chain"
-          sleep 30
-
-      - name: ${{ matrix.demo_name }}-${{ matrix.mode }}
-        timeout-minutes: 25
-        # * the change the symbolic link which points to the target/release... folder.
-        # * need overwrite default shell to bash to get access to the `source` cmd.
-        shell: bash --noprofile --norc -eo pipefail {0}
+          cd docker
+          docker-compose -f litentry-parachain.build.yml build
+
+      - name: Integration Test ${{ matrix.test }}-${{ matrix.mode }}
+        timeout-minutes: 30
         run: |
-          source ./scripts/init_env.sh && ./scripts/${{ matrix.demo_script }}
+          cd docker
+          docker compose -f docker-compose.yml -f ${{ matrix.demo_name }}.yml up ${{ matrix.demo_name }} --no-build --exit-code-from ${{ matrix.demo_name }}
 
-      - name: Upload logs
+      - name: Collect Docker Logs
         continue-on-error: true
         if: always()
-        uses: actions/upload-artifact@v2
+        uses: jwalton/gh-docker-logs@v2.2.0
         with:
-          name: ${{ matrix.test }}-${{ matrix.mode }}_logs
-          path: ${{ env.LOG_DIR }}
+          #images: '${{ env.WORKER_IMAGE_TAG }},${{ env.CLIENT_IMAGE_TAG }}'
+          tail: all
+          dest: ./${{ env.LOG_DIR }}
 
-      # won't trigger the graceful handler as we send SIGKILL
-      - name: Stop litentry parachain and cleanup
-        continue-on-error: true
+      - name: Upload logs
         if: always()
-        run: |
-          ./scripts/litentry/stop_parachain.sh || true
-          ./scripts/litentry/cleanup.sh
+        uses: actions/upload-artifact@v2
+        with:
+          name: logs-${{ matrix.test }}-${{ matrix.mode }}
+          path: ./${{ env.LOG_DIR }}
 
   # Only push docker image when tests are passed on dev branch
   push-docker-image: