From 7bdbb190742b52bbcdf07ea93a99e4214ee777cc Mon Sep 17 00:00:00 2001 From: Zoltan Mosonyi Date: Sun, 16 Apr 2023 21:12:08 +0200 Subject: [PATCH 1/9] First trial to use docker cache --- .dockerignore | 1 + build.Dockerfile | 91 ++++++++++++++++++++++++++++++++---------------- 2 files changed, 62 insertions(+), 30 deletions(-) diff --git a/.dockerignore b/.dockerignore index f82d6e8b32..e77752005c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,6 +8,7 @@ docs/ local-setup/ scripts/ target/ +enclave-runtime/target/ tmp/ *.Dockerfile Dockerfile \ No newline at end of file diff --git a/build.Dockerfile b/build.Dockerfile index 466c02ebe6..5384603920 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:experimental # Copyright 2021 Integritee AG # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,34 +18,34 @@ ### Builder Stage ################################################## -FROM integritee/integritee-dev:0.2.1 AS builder -LABEL maintainer="zoltan@integritee.network" +# FROM integritee/integritee-dev:0.2.1 AS builder +# LABEL maintainer="zoltan@integritee.network" -# set environment variables -ENV SGX_SDK /opt/sgxsdk -ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/opt/rust/bin//bin" -ENV PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${SGX_SDK}/pkgconfig" -ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/sdk_libs" -ENV CARGO_NET_GIT_FETCH_WITH_CLI true +# # set environment variables +# ENV SGX_SDK /opt/sgxsdk +# ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/opt/rust/bin" +# ENV PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${SGX_SDK}/pkgconfig" +# ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/sdk_libs" +# ENV CARGO_NET_GIT_FETCH_WITH_CLI true -# Default SGX MODE is software mode -ARG SGX_MODE=SW -ENV SGX_MODE=$SGX_MODE +# # Default SGX MODE is software mode +# ARG SGX_MODE=SW +# ENV SGX_MODE=$SGX_MODE -ENV HOME=/home/ubuntu/work +# ENV HOME=/home/ubuntu/work -ARG WORKER_MODE_ARG -ENV WORKER_MODE=$WORKER_MODE_ARG +# ARG WORKER_MODE_ARG +# ENV WORKER_MODE=$WORKER_MODE_ARG -ARG ADDITIONAL_FEATURES_ARG -ENV ADDITIONAL_FEATURES=$ADDITIONAL_FEATURES_ARG +# ARG ADDITIONAL_FEATURES_ARG +# ENV ADDITIONAL_FEATURES=$ADDITIONAL_FEATURES_ARG -WORKDIR $HOME/worker -COPY . . +# WORKDIR $HOME/worker +# COPY . . -RUN make +# RUN make -RUN cargo test --release +# RUN cargo test --release ### Cached Builder Stage (WIP) @@ -52,12 +53,12 @@ RUN cargo test --release # A builder stage that uses sccache to speed up local builds with docker # Installation and setup of sccache should be moved to the integritee-dev image, so we don't # always need to compile and install sccache on CI (where we have no caching so far). -FROM integritee/integritee-dev:0.2.1 AS cached-builder +FROM integritee/integritee-dev:0.2.1 AS builder LABEL maintainer="zoltan@integritee.network" # set environment variables ENV SGX_SDK /opt/sgxsdk -ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/opt/rust/bin/bin" +ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/opt/rust/bin" ENV PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${SGX_SDK}/pkgconfig" ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/sdk_libs" ENV CARGO_NET_GIT_FETCH_WITH_CLI true @@ -66,23 +67,53 @@ ENV CARGO_NET_GIT_FETCH_WITH_CLI true ARG SGX_MODE=SW ENV SGX_MODE=$SGX_MODE -ENV HOME=/home/ubuntu/work +ENV WORKHOME=/home/ubuntu/work +ENV HOME=/home/ubuntu + +RUN rustup default stable +#&& cargo install sccache -RUN rustup default stable && cargo install sccache -ENV SCCACHE_CACHE_SIZE="3G" -ENV SCCACHE_DIR=$HOME/.cache/sccache -ENV RUSTC_WRAPPER="/opt/rust/bin/sccache" +#ENV SCCACHE_CACHE_SIZE="3G" +#ENV SCCACHE_DIR=$HOME/.cache/sccache +#ENV RUSTC_WRAPPER="/opt/rust/bin/sccache" ARG WORKER_MODE_ARG ENV WORKER_MODE=$WORKER_MODE_ARG -WORKDIR $HOME/worker +WORKDIR $WORKHOME/worker + +# ADD src src +# ADD Cargo.lock . +# ADD Cargo.toml . + COPY . . +#RUN --mount=type=cache,id=cargo-target,target=/home/ubuntu/work/worker/target ls -la target + + +RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ + --mount=type=cache,id=cargo-target,target=/home/ubuntu/work/worker/target \ + --mount=type=cache,id=cargo-enclave-target,target=/home/ubuntu/work/worker/enclave-runtime/target \ + ls -la /opt/rust/registry && ls -la /home/ubuntu/work/worker/target && ls -la /home/ubuntu/work/worker/enclave-runtime/target + +#RUN mkdir -p $HOME/.cache/sccache +# RUN sccache --start-server +RUN echo 11 +#RUN --mount=type=cache,id=cargo,target=${SCCACHE_DIR} --mount=type=cache,id=cargo-target,target=${WORKHOME}/worker/target \ +# --mount=type=cache,id=cargo-git,target=/opt/rust/git/db --mount=type=cache,id=cargo-registry,target=/opt/rust/registry make && cargo test --release && sccache --show-stats -RUN --mount=type=cache,id=cargo,target=${HOME}/.cache/sccache make && sccache --show-stats +RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ + --mount=type=cache,id=cargo-git,target=/opt/rust/git/db \ + --mount=type=cache,id=cargo-target,target=/home/ubuntu/work/worker/target \ + --mount=type=cache,id=cargo-enclave-target,target=/home/ubuntu/work/worker/enclave-runtime/target \ + make -RUN --mount=type=cache,id=cargo,target=${HOME}/.cache/sccache cargo test --release && sccache --show-stats +RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ + --mount=type=cache,id=cargo-target,target=/home/ubuntu/work/worker/target \ + --mount=type=cache,id=cargo-enclave-target,target=/home/ubuntu/work/worker/enclave-runtime/target \ + ls -la /opt/rust/registry && ls -la /home/ubuntu/work/worker/target && ls -la /home/ubuntu/work/worker/enclave-runtime/target +# RUN --mount=type=cache,id=cargo,target=${SCCACHE_DIR} cargo test --release && sccache --show-stats +# RUN sccache --stop-server ### Base Runner Stage ### The runner needs the aesmd service for the `SGX_MODE=HW`. From 658ea4dba25a139d80ad327e1dc77a822fd5af44 Mon Sep 17 00:00:00 2001 From: Zoltan Mosonyi Date: Sun, 16 Apr 2023 21:29:34 +0200 Subject: [PATCH 2/9] Added back cargo test release --- build.Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build.Dockerfile b/build.Dockerfile index 5384603920..78bff83880 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -107,6 +107,12 @@ RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ --mount=type=cache,id=cargo-enclave-target,target=/home/ubuntu/work/worker/enclave-runtime/target \ make +RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ + --mount=type=cache,id=cargo-git,target=/opt/rust/git/db \ + --mount=type=cache,id=cargo-target,target=/home/ubuntu/work/worker/target \ + --mount=type=cache,id=cargo-enclave-target,target=/home/ubuntu/work/worker/enclave-runtime/target \ + cargo test --release + RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ --mount=type=cache,id=cargo-target,target=/home/ubuntu/work/worker/target \ --mount=type=cache,id=cargo-enclave-target,target=/home/ubuntu/work/worker/enclave-runtime/target \ From 485f7e8ce25da1024a58d7a0e5bb7f283c70ba84 Mon Sep 17 00:00:00 2001 From: Zoltan Mosonyi Date: Mon, 17 Apr 2023 07:47:07 +0200 Subject: [PATCH 3/9] Remove debugging commands --- build.Dockerfile | 72 ++---------------------------------------------- 1 file changed, 2 insertions(+), 70 deletions(-) diff --git a/build.Dockerfile b/build.Dockerfile index 78bff83880..36df677399 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -16,39 +16,7 @@ # This is a multi-stage docker file, where the first stage is used # for building and the second deploys the built application. -### Builder Stage -################################################## -# FROM integritee/integritee-dev:0.2.1 AS builder -# LABEL maintainer="zoltan@integritee.network" - -# # set environment variables -# ENV SGX_SDK /opt/sgxsdk -# ENV PATH "$PATH:${SGX_SDK}/bin:${SGX_SDK}/bin/x64:/opt/rust/bin" -# ENV PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${SGX_SDK}/pkgconfig" -# ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/sdk_libs" -# ENV CARGO_NET_GIT_FETCH_WITH_CLI true - -# # Default SGX MODE is software mode -# ARG SGX_MODE=SW -# ENV SGX_MODE=$SGX_MODE - -# ENV HOME=/home/ubuntu/work - -# ARG WORKER_MODE_ARG -# ENV WORKER_MODE=$WORKER_MODE_ARG - -# ARG ADDITIONAL_FEATURES_ARG -# ENV ADDITIONAL_FEATURES=$ADDITIONAL_FEATURES_ARG - -# WORKDIR $HOME/worker -# COPY . . - -# RUN make - -# RUN cargo test --release - - -### Cached Builder Stage (WIP) +### Cached Builder Stage ################################################## # A builder stage that uses sccache to speed up local builds with docker # Installation and setup of sccache should be moved to the integritee-dev image, so we don't @@ -71,55 +39,19 @@ ENV WORKHOME=/home/ubuntu/work ENV HOME=/home/ubuntu RUN rustup default stable -#&& cargo install sccache - -#ENV SCCACHE_CACHE_SIZE="3G" -#ENV SCCACHE_DIR=$HOME/.cache/sccache -#ENV RUSTC_WRAPPER="/opt/rust/bin/sccache" ARG WORKER_MODE_ARG ENV WORKER_MODE=$WORKER_MODE_ARG WORKDIR $WORKHOME/worker -# ADD src src -# ADD Cargo.lock . -# ADD Cargo.toml . - COPY . . -#RUN --mount=type=cache,id=cargo-target,target=/home/ubuntu/work/worker/target ls -la target - - -RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ - --mount=type=cache,id=cargo-target,target=/home/ubuntu/work/worker/target \ - --mount=type=cache,id=cargo-enclave-target,target=/home/ubuntu/work/worker/enclave-runtime/target \ - ls -la /opt/rust/registry && ls -la /home/ubuntu/work/worker/target && ls -la /home/ubuntu/work/worker/enclave-runtime/target - -#RUN mkdir -p $HOME/.cache/sccache -# RUN sccache --start-server -RUN echo 11 -#RUN --mount=type=cache,id=cargo,target=${SCCACHE_DIR} --mount=type=cache,id=cargo-target,target=${WORKHOME}/worker/target \ -# --mount=type=cache,id=cargo-git,target=/opt/rust/git/db --mount=type=cache,id=cargo-registry,target=/opt/rust/registry make && cargo test --release && sccache --show-stats - -RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ - --mount=type=cache,id=cargo-git,target=/opt/rust/git/db \ - --mount=type=cache,id=cargo-target,target=/home/ubuntu/work/worker/target \ - --mount=type=cache,id=cargo-enclave-target,target=/home/ubuntu/work/worker/enclave-runtime/target \ - make RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ --mount=type=cache,id=cargo-git,target=/opt/rust/git/db \ --mount=type=cache,id=cargo-target,target=/home/ubuntu/work/worker/target \ --mount=type=cache,id=cargo-enclave-target,target=/home/ubuntu/work/worker/enclave-runtime/target \ - cargo test --release - -RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ - --mount=type=cache,id=cargo-target,target=/home/ubuntu/work/worker/target \ - --mount=type=cache,id=cargo-enclave-target,target=/home/ubuntu/work/worker/enclave-runtime/target \ - ls -la /opt/rust/registry && ls -la /home/ubuntu/work/worker/target && ls -la /home/ubuntu/work/worker/enclave-runtime/target - -# RUN --mount=type=cache,id=cargo,target=${SCCACHE_DIR} cargo test --release && sccache --show-stats -# RUN sccache --stop-server + make && cargo test --release ### Base Runner Stage ### The runner needs the aesmd service for the `SGX_MODE=HW`. From 6d336ebe759f6f205dd97573ea4f3864e4765f67 Mon Sep 17 00:00:00 2001 From: Zoltan Mosonyi Date: Mon, 17 Apr 2023 11:27:56 +0200 Subject: [PATCH 4/9] Add sccache to try --- build.Dockerfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/build.Dockerfile b/build.Dockerfile index 36df677399..8a657da5d3 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -39,6 +39,11 @@ ENV WORKHOME=/home/ubuntu/work ENV HOME=/home/ubuntu RUN rustup default stable +RUN cargo install sccache + +ENV SCCACHE_CACHE_SIZE="20G" +ENV SCCACHE_DIR=$HOME/.cache/sccache +ENV RUSTC_WRAPPER="/opt/rust/bin/sccache" ARG WORKER_MODE_ARG ENV WORKER_MODE=$WORKER_MODE_ARG @@ -49,10 +54,11 @@ COPY . . RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ --mount=type=cache,id=cargo-git,target=/opt/rust/git/db \ - --mount=type=cache,id=cargo-target,target=/home/ubuntu/work/worker/target \ - --mount=type=cache,id=cargo-enclave-target,target=/home/ubuntu/work/worker/enclave-runtime/target \ + --mount=type=cache,id=cargo-sccache-${WORKER_MODE},target=/home/ubuntu/.cache/sccache \ make && cargo test --release +# --mount=type=cache,id=cargo-enclave-target-${WORKER_MODE},target=/home/ubuntu/work/worker/enclave-runtime/target \ + ### Base Runner Stage ### The runner needs the aesmd service for the `SGX_MODE=HW`. ###################################################### From ab1988da4c5be09dd9a1c482e6f8ba9caad20316 Mon Sep 17 00:00:00 2001 From: Zoltan Mosonyi Date: Mon, 17 Apr 2023 12:21:11 +0200 Subject: [PATCH 5/9] Added sccache status --- build.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.Dockerfile b/build.Dockerfile index 8a657da5d3..da07bad325 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -55,7 +55,7 @@ COPY . . RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ --mount=type=cache,id=cargo-git,target=/opt/rust/git/db \ --mount=type=cache,id=cargo-sccache-${WORKER_MODE},target=/home/ubuntu/.cache/sccache \ - make && cargo test --release + make && cargo test --release && sccache --show-stats # --mount=type=cache,id=cargo-enclave-target-${WORKER_MODE},target=/home/ubuntu/work/worker/enclave-runtime/target \ From 8e1789d9146c955012a023dfddf4504c45f7de05 Mon Sep 17 00:00:00 2001 From: Zoltan Mosonyi Date: Mon, 17 Apr 2023 13:18:43 +0200 Subject: [PATCH 6/9] Be sure that build has unique id --- build.Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.Dockerfile b/build.Dockerfile index da07bad325..16874b399c 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -46,7 +46,9 @@ ENV SCCACHE_DIR=$HOME/.cache/sccache ENV RUSTC_WRAPPER="/opt/rust/bin/sccache" ARG WORKER_MODE_ARG +ARG ADDITIONAL_FEATURES_ARG ENV WORKER_MODE=$WORKER_MODE_ARG +ENV ADDITIONAL_FEATURES=$ADDITIONAL_FEATURES_ARG WORKDIR $WORKHOME/worker @@ -54,7 +56,7 @@ COPY . . RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ --mount=type=cache,id=cargo-git,target=/opt/rust/git/db \ - --mount=type=cache,id=cargo-sccache-${WORKER_MODE},target=/home/ubuntu/.cache/sccache \ + --mount=type=cache,id=cargo-sccache-${WORKER_MODE}${ADDITIONAL_FEATURES},target=/home/ubuntu/.cache/sccache \ make && cargo test --release && sccache --show-stats # --mount=type=cache,id=cargo-enclave-target-${WORKER_MODE},target=/home/ubuntu/work/worker/enclave-runtime/target \ From 8059e8664ac66f66ab753737fb996c80df9fdc99 Mon Sep 17 00:00:00 2001 From: Zoltan Mosonyi Date: Mon, 17 Apr 2023 16:02:26 +0200 Subject: [PATCH 7/9] Add force run that line --- .github/workflows/build_and_test.yml | 4 +++- build.Dockerfile | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index cc1641130d..629303ee8d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -56,6 +56,8 @@ jobs: - name: Set env run: | + fingerprint=$RANDOM + echo "FINGERPRINT=$fingerprint" >> $GITHUB_ENV if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then echo "DOCKER_DEVICES=--device=/dev/sgx/enclave --device=/dev/sgx/provision" >> $GITHUB_ENV echo "DOCKER_VOLUMES=--volume /var/run/aesmd:/var/run/aesmd" >> $GITHUB_ENV @@ -76,7 +78,7 @@ jobs: run: > docker build -t integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }} --target deployed-worker - --build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} --build-arg SGX_MODE=${{ matrix.sgx_mode }} + --build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg FINGERPRINT=${FINGERPRINT} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} --build-arg SGX_MODE=${{ matrix.sgx_mode }} -f build.Dockerfile . - name: Build CLI client diff --git a/build.Dockerfile b/build.Dockerfile index 16874b399c..5e1ad69ae2 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -50,6 +50,8 @@ ARG ADDITIONAL_FEATURES_ARG ENV WORKER_MODE=$WORKER_MODE_ARG ENV ADDITIONAL_FEATURES=$ADDITIONAL_FEATURES_ARG +ARG FINGERPRINT=none + WORKDIR $WORKHOME/worker COPY . . @@ -57,7 +59,7 @@ COPY . . RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ --mount=type=cache,id=cargo-git,target=/opt/rust/git/db \ --mount=type=cache,id=cargo-sccache-${WORKER_MODE}${ADDITIONAL_FEATURES},target=/home/ubuntu/.cache/sccache \ - make && cargo test --release && sccache --show-stats + echo ${FINGERPRINT} && make && cargo test --release && sccache --show-stats # --mount=type=cache,id=cargo-enclave-target-${WORKER_MODE},target=/home/ubuntu/work/worker/enclave-runtime/target \ From 872a39e4f63780cb8e4cba3b9e6db707dbf112b7 Mon Sep 17 00:00:00 2001 From: Zoltan Mosonyi Date: Wed, 19 Apr 2023 16:08:38 +0200 Subject: [PATCH 8/9] Remove commented out line --- build.Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.Dockerfile b/build.Dockerfile index 5e1ad69ae2..214af47795 100644 --- a/build.Dockerfile +++ b/build.Dockerfile @@ -61,8 +61,6 @@ RUN --mount=type=cache,id=cargo-registry,target=/opt/rust/registry \ --mount=type=cache,id=cargo-sccache-${WORKER_MODE}${ADDITIONAL_FEATURES},target=/home/ubuntu/.cache/sccache \ echo ${FINGERPRINT} && make && cargo test --release && sccache --show-stats -# --mount=type=cache,id=cargo-enclave-target-${WORKER_MODE},target=/home/ubuntu/work/worker/enclave-runtime/target \ - ### Base Runner Stage ### The runner needs the aesmd service for the `SGX_MODE=HW`. ###################################################### From 1d4331694e8c160083632fea55a627f072f3ed82 Mon Sep 17 00:00:00 2001 From: Zoltan Mosonyi Date: Wed, 19 Apr 2023 22:51:53 +0200 Subject: [PATCH 9/9] Pull dev image if not exist --- .github/workflows/build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 56fed8fff9..14f856713c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -280,6 +280,7 @@ jobs: fi docker tag integritee-worker-${{ matrix.flavor_id }}-${{ github.sha }} ${{ env.WORKER_IMAGE_TAG }} docker tag integritee-cli-client-${{ matrix.flavor_id }}-${{ github.sha }} ${{ env.CLIENT_IMAGE_TAG }} + docker pull integritee/integritee-node-dev:1.0.33 docker tag integritee/integritee-node-dev:1.0.33 ${{ env.INTEGRITEE_NODE }} docker images --all