Set up a smoke test pass that check if Substrate/Polkadot build using dist sccache

.github/workflows/ci.yml

@@ -80,6 +80,108 @@ jobs:
 
       - name: Run tests
         run: cargo test --locked --all-targets --verbose ${{ matrix.extra_args }}
+  smoke:
+    name: smoke test ${{ matrix.rustc || 'stable' }} ${{ matrix.target_repository }}
+    needs: [build]
+    runs-on: ubuntu-18.04
+    continue-on-error: ${{ matrix.allow_failure || false }}
+    strategy:
+      matrix:
+        include:
+          - target_repository: paritytech/polkadot
+            # Pin a well-known nightly version to avoid upstream breakage
+            rustc: nightly-2021-05-10
+            target: wasm32-unknown-unknown
+            allow_failure: true
+          - target_repository: paritytech/substrate
+            rustc: nightly-2021-05-10
+            target: wasm32-unknown-unknown
+            allow_failure: true
+          - target_repository: serde-rs/serde
+          - target_repository: serde-rs/json
+    env:
+      RUST_BACKTRACE: 1
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v1
+
+      - name: Install rust
+        uses: ./.github/actions/rust-toolchain
+        with:
+          toolchain: ${{ matrix.rustc }}
+          target: ${{ matrix.target }}
+
+      - name: Fetch current git tag/sha
+        id: id
+        shell: bash
+        run: echo "::set-output name=id::${ID#refs/tags/}"
+        env:
+          ID: ${{ startsWith(github.ref, 'refs/tags/') && github.ref || github.sha }}
+
+      # To cut down on the workflow execution time, we re-use pre-compiled binaries
+      # here and simply vendor the `sccache-dist` in the container via
+      # `SCCACHE_IN_DOCKERFILE: vendor` rather than build separately it via Dockerfile
+      - name: Download artifacts (sccache)
+        uses: actions/download-artifact@v2
+        with:
+          name: sccache-${{ steps.id.outputs.id }}-x86_64-unknown-linux-musl
+          path: artifacts/
+      - name: Download artifacts (sccache-dist)
+        uses: actions/download-artifact@v2
+        with:
+          name: sccache-dist-${{ steps.id.outputs.id }}-x86_64-unknown-linux-musl
+          path: artifacts/
+      - name: Set up pre-built artifact binaries
+        run: |
+          cp artifacts/sccache ~/.cargo/bin/sccache
+          cp artifacts/sccache-dist systemd/sccache-dist
+          chmod +x ~/.cargo/bin/sccache systemd/sccache-dist
+          ls -la ~/.cargo/bin/sccache systemd/sccache-dist
+
+      - name: Set up local distributed sccache stack
+        env:
+          RUST_LOG: sccache=trace
+          SCCACHE_IN_DOCKERFILE: vendor
+        run: docker-compose --file systemd/docker-compose.yaml up -d
+      # TODO: Wait until the stack is up/ready
+      - run: docker ps -a
+
+      - name: Clone the project repository being tested
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ matrix.target_repository }}
+          path: target_repo
+
+      - name: Double-check that sccache is not running
+        run: sccache --stop-server || true
+      - name: Start the local sccache server daemon
+        run: sccache --start-server
+        # This is the main server that's going to send the compilation requests
+        # so logging is more important for this than for subsequent (client)
+        # sccache invocations
+        env:
+          SCCACHE_CONF: ${{ github.workspace }}/systemd/config/client.conf
+          SCCACHE_LOG: sccache=trace
+          SCCACHE_ERROR_LOG: ${{ github.workspace }}/sccache.err.log
+      - name: Verify that the local distributed sccache stack is running
+        run: test $(sccache --dist-status | jq ".SchedulerStatus[1].num_servers") -gt 0
+
+      - name: Run a smoke check build
+        env:
+          RUSTC_WRAPPER: sccache
+        working-directory: target_repo
+        run: cargo check --verbose ${{ matrix.extra_args }}
+      - run: sccache --show-stats
+      - name: Verify that the distribution compilation *did* take place
+        run:
+          test $(sccache --show-stats --stats-format json | jq ".stats.dist_compiles | to_entries[].value") -gt 0
+
+      - name: Dump Compose logs
+        run: docker-compose --file systemd/docker-compose.yaml logs
+        if: failure()
+      - name: Dump local sccache daemon server logs
+        run: cat ${{ github.workspace }}/sccache.err.log
+        if: failure()
 
   build:
     name: build ${{ matrix.binary || 'sccache' }} ${{ matrix.target }}
@@ -90,9 +192,10 @@ jobs:
         include:
           - os: ubuntu-20.04
             target: x86_64-unknown-linux-musl
+            extra_args: --features="dist-server"
           - os: ubuntu-20.04
             binary: sccache-dist
-            extra_args: --no-default-features --features="dist-server"
+            extra_args: --features="dist-server"
             target: x86_64-unknown-linux-musl
           - os: ubuntu-20.04
             target: aarch64-unknown-linux-musl
@@ -113,6 +216,14 @@ jobs:
       - name: Clone repository
         uses: actions/checkout@v1
 
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-build-v2-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }}
+
       - name: Install rust
         uses: ./.github/actions/rust-toolchain
         with:
@@ -163,7 +274,7 @@ jobs:
   release:
     name: release
     runs-on: ubuntu-latest
-    needs: [build, lint, test]
+    needs: [build, lint, test, smoke]
     if: ${{ startsWith(github.ref, 'refs/tags/') }}
     steps:
       - name: Clone repository

systemd/Dockerfile.build.sccache-dist

@@ -0,0 +1,22 @@
+FROM ubuntu:18.04 as bwrap-build
+RUN apt-get update && \
+    apt-get install -y wget xz-utils gcc libcap-dev make && \
+    apt-get clean
+RUN wget -q -O - https://github.com/projectatomic/bubblewrap/releases/download/v0.3.1/bubblewrap-0.3.1.tar.xz | \
+    tar -xJ
+RUN cd /bubblewrap-0.3.1 && \
+    ./configure --disable-man && \
+    make
+
+FROM rust:1.52-buster as sccache-build
+RUN git clone https://github.com/paritytech/sccache.git --depth=1 && \
+    cd sccache && \
+    cargo build --bin sccache-dist --release --features="dist-server"
+
+FROM ubuntu:18.04
+RUN apt-get update && \
+    apt-get install libcap2 libssl1.1 && \
+    apt-get clean
+COPY --from=bwrap-build /bubblewrap-0.3.1/bwrap /usr/bin/bwrap
+COPY --from=sccache-build /sccache/target/release/sccache-dist /usr/bin/sccache-dist
+CMD [ "sccache-dist" ]

systemd/Dockerfile.vendor.sccache-dist

@@ -0,0 +1,21 @@
+# A version of `Dockerfile.build.sccache-dist` that skips the build stage and
+# instead copies over the locally pre-built `sccache-dist` binary.
+# TODO: Switch to BuildKit with Docker/Compose support when available, which
+# allows us to unify these into a single Dockerfile
+FROM ubuntu:18.04 as bwrap-build
+RUN apt-get update && \
+    apt-get install -y wget xz-utils gcc libcap-dev make && \
+    apt-get clean
+RUN wget -q -O - https://github.com/projectatomic/bubblewrap/releases/download/v0.3.1/bubblewrap-0.3.1.tar.xz | \
+    tar -xJ
+RUN cd /bubblewrap-0.3.1 && \
+    ./configure --disable-man && \
+    make
+
+FROM ubuntu:18.04
+RUN apt-get update && \
+    apt-get install libcap2 libssl1.1 && \
+    apt-get clean
+COPY --from=bwrap-build /bubblewrap-0.3.1/bwrap /usr/bin/bwrap
+ADD sccache-dist /usr/bin/sccache-dist
+CMD [ "sccache-dist" ]

systemd/config/client.conf

@@ -0,0 +1,14 @@
+[dist]
+# The URL used to connect to the scheduler (should use https, given an ideal
+# setup of a HTTPS server in front of the scheduler)
+scheduler_url = "http://172.25.0.10:10600"
+# Used for mapping local toolchains to remote cross-compile toolchains. Empty in
+# this example where the client and build server are both Linux.
+toolchains = []
+# Size of the local toolchain cache, in bytes (5GB here, 10GB if unspecified).
+toolchain_cache_size = 5368709120
+
+[dist.auth]
+type = "token"
+# This should match the `client_auth` section of the scheduler config.
+token = "a concrete secret that's shared client and scheduler"

systemd/docker-compose.yaml

@@ -0,0 +1,82 @@
+# An example setup for a distributed sccache cluster that uses
+# - a single scheduler (172.25.0.10)
+# - a single build server (172.25.0.11)
+# both served from a local network.
+# To test this locally, make sure to set up `dist.scheduler_url` and `dist.auth`
+# correctly, as used by the `sccache` binary (the "client") and verify that the
+# connection works by running `sccache --dist-status`.
+# It's worth noting that the security is virtually none using this exact setup.
+# To correctly set everything up (e.g. set up auth across scheduler <-> server
+# and client <-> scheduler, use HTTPS for the scheduler) refer to the
+# `docs/Distributed.md` file.
+version: "3.6"
+
+services:
+    scheduler:
+        build:
+            context: .
+            dockerfile: Dockerfile.${SCCACHE_IN_DOCKERFILE:-build}.sccache-dist
+        environment:
+            RUST_LOG:
+            SCCACHE_NO_DAEMON: 1
+            SCCACHE_CONFIG_TOML: |
+                public_addr = "172.25.0.10:10600"
+                [server_auth]
+                type = "DANGEROUSLY_INSECURE"
+                [client_auth]
+                type = "token"
+                token = "a concrete secret that's shared client and scheduler"
+        ports:
+            - 10600
+        networks:
+            dist_network:
+                ipv4_address: 172.25.0.10
+        command: /bin/sh -c "echo \"$$SCCACHE_CONFIG_TOML\" > config.toml; sccache-dist scheduler --config config.toml"
+    server:
+        # XXX: For the time being, due to the usage of bubblewrap and overlayfs
+        # we are required to run this in the privileged mode (as we do in the
+        # integration tests). Please keep that in mind when running the container.
+        # TODO: In the future we'd like to run the build sandbox in an unprivileged
+        # mode.
+        privileged: true
+        build:
+            context: .
+            dockerfile: Dockerfile.${SCCACHE_IN_DOCKERFILE:-build}.sccache-dist
+        environment:
+            RUST_LOG:
+            SCCACHE_NO_DAEMON: 1
+            SCCACHE_CONFIG_TOML: |
+                # A public IP address and port that clients will use to connect to this builder.
+                public_addr = "172.25.0.11:10501"
+                # The URL used to connect to the scheduler (should use https, given an ideal
+                # setup of a HTTPS server in front of the scheduler)
+                scheduler_url = "http://172.25.0.10:10600"
+                # The maximum size of the toolchain cache, in bytes.
+                # If unspecified the default is 10GB.
+                # toolchain_cache_size = 10737418240
+                cache_dir="/sccache-dirs/cache/"
+
+                [builder]
+                type = "overlay"
+                build_dir = "/sccache-dirs/builder/"
+                # The path to the bubblewrap version 0.3.0+ `bwrap` binary.
+                bwrap_path = "/usr/bin/bwrap"
+
+                [scheduler_auth]
+                type = "DANGEROUSLY_INSECURE"
+        ports:
+            - 10501
+        networks:
+            dist_network:
+                ipv4_address: 172.25.0.11
+        tmpfs:
+            - /sccache-dirs
+        command: /bin/sh -c "echo \"$$SCCACHE_CONFIG_TOML\" > config.toml; sccache-dist server --config config.toml"
+# We need to set static IPs for the services as scheduler/server always use it
+# to authorize themselves (even when using the DANGEROUSLY_INSECURE scheme)
+networks:
+    dist_network:
+        ipam:
+            driver: default
+            config:
+                - subnet: 172.25.0.0/24