Merge branch 'master' into andre/companion-9448

* master: (138 commits) Allow an Offset to Lease Periods (#3980) Bump quote from 1.0.9 to 1.0.10 (#4013) Companion for #9834 (Transaction Priority) (#3901) chore: update `builder` image (#3884) Free disputed cores before processing bitfields (#4008) Make candidate validation timeouts configurable (#4001) Add extrinsic ordering filtering (#3631) chore: ci list files that spellcheck finds (#3992) Use background tasks properly in candidate-validation (#4002) Fix unoccupied bitfields (#4004) Bump syn from 1.0.77 to 1.0.78 (#4006) Bump jsonrpsee-ws-client from 0.3.0 to 0.3.1 (#3931) fix clock drift for assignments issued before the block (#3851) Remove unoccupied bit check (#3999) bump substrate (#4000) change genesis authority set for wococo-local, revert rococo-local (#3998) ignore irrelevant approvals in logs (#3859) avoid expect, on free availability core (#3994) preserve finalized block in active leaves (#3997) some tweaks to rococo-local (#3996) ...
paritytech · Oct 5, 2021 · 9e0e469 · 9e0e469
2 parents a0d7b5e + 2a3fca3
commit 9e0e469
Show file tree

Hide file tree

Showing 377 changed files with 15,213 additions and 5,885 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -15,5 +15,6 @@ updates:
       - dependency-name: "beefy-*"
       - dependency-name: "try-runtime-*"
       - dependency-name: "test-runner"
+      - dependency-name: "generate-bags"
     schedule:
       interval: "daily"
diff --git a/.github/workflows/extrinsic-ordering-check-from-bin.yml b/.github/workflows/extrinsic-ordering-check-from-bin.yml
@@ -6,15 +6,15 @@ on:
     inputs:
       reference_url:
         description: The WebSocket url of the reference node
-        default: wss://rpc.polkadot.io
+        default: wss://kusama-rpc.polkadot.io
         required: true
       binary_url:
         description: A url to a Linux binary for the node containing the runtime to test
-        default: https://releases.parity.io/polkadot/x86_64-debian:stretch/v0.9.9-rc1/polkadot
+        default: https://releases.parity.io/polkadot/x86_64-debian:stretch/v0.9.10/polkadot
         required: true
       chain:
         description: The name of the chain under test. Usually, you would pass a local chain
-        default: polkadot-local
+        default: kusama-local
         required: true
 
 jobs:
@@ -27,6 +27,8 @@ jobs:
       REF_URL: ${{github.event.inputs.reference_url}}
 
     steps:
+      - uses: actions/checkout@v2
+
       - name: Fetch binary
         run: |
           echo Fetching $BIN_URL
@@ -46,17 +48,26 @@ jobs:
           echo "Date: $(date)" >> output.txt
           echo "Reference: $REF_URL" >> output.txt
           echo "Target version: $VERSION" >> output.txt
-          echo "-------------------------------------------" >> output.txt
+          echo "Chain: $CHAIN" >> output.txt
+          echo "----------------------------------------------------------------------" >> output.txt
+
+      - name: Pull polkadot-js-tools image
+        run: docker pull jacogr/polkadot-js-tools
 
       - name: Compare the metadata
         run: |
-          CMD="docker run --network host jacogr/polkadot-js-tools metadata $REF_URL ws://localhost:9944"
+          CMD="docker run --pull always --network host jacogr/polkadot-js-tools metadata $REF_URL ws://localhost:9944"
           echo -e "Running:\n$CMD"
           $CMD >> output.txt
           sed -z -i 's/\n\n/\n/g' output.txt
+          cat output.txt | egrep -n -i ''
+          SUMMARY=$(./scripts/github/extrinsic-ordering-filter.sh output.txt)
+          echo -e $SUMMARY
+          echo -e $SUMMARY >> output.txt
 
       - name: Show result
-        run: cat output.txt
+        run: |
+          cat output.txt
 
       - name: Stop our local node
         run: pkill polkadot

diff --git a/.github/workflows/honggfuzz.yml b/.github/workflows/honggfuzz.yml
@@ -13,6 +13,13 @@ jobs:
         with:
           fetch-depth: 1
 
+      - name: Cache Seed
+        id: cache-seed-round-trip
+        uses: actions/cache@v2
+        with:
+          path: erasure-coding/fuzzer/hfuzz_workspace
+          key: ${{ runner.os }}-erasure-coding
+
       - name: Install minimal stable Rust
         uses: actions-rs/toolchain@v1
         with:
@@ -52,6 +59,13 @@ jobs:
         with:
           fetch-depth: 1
 
+      - name: Cache Seed
+        id: cache-seed-reconstruct
+        uses: actions/cache@v2
+        with:
+          path: erasure-coding/fuzzer/hfuzz_workspace
+          key: ${{ runner.os }}-erasure-coding
+
       - name: Install minimal stable Rust
         uses: actions-rs/toolchain@v1
         with:

diff --git a/.github/workflows/publish-docker-release.yml b/.github/workflows/publish-docker-release.yml
@@ -29,14 +29,15 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           push: true
-          file: scripts/docker/release.Dockerfile
+          file: scripts/docker/polkadot_injected_release.Dockerfile
           tags: |
             parity/polkadot:latest
             parity/polkadot:${{ github.event.release.tag_name }}
           build-args: |
             POLKADOT_VERSION=${{ github.event.release.tag_name }}
             VCS_REF=${{ github.ref }}
             BUILD_DATE=${{ github.event.release.published_at }}
+            GPG_KEYSERVER="hkps://keys.mailvelope.com"
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
       - name: Image digest

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -28,6 +28,9 @@ variables:
   CI_SERVER_NAME:                  "GitLab CI"
   DOCKER_OS:                       "debian:stretch"
   ARCH:                            "x86_64"
+  VAULT_SERVER_URL:                "https://vault.parity-mgmt-vault.parity.io"
+  VAULT_AUTH_PATH:                 "gitlab-parity-io-jwt"
+  VAULT_AUTH_ROLE:                 "cicd_gitlab_parity_${CI_PROJECT_NAME}"
 
 default:
   cache:                           {}
@@ -84,13 +87,63 @@ default:
       when: never
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
 
+#### Vault secrets
+.vault-secrets:                    &vault-secrets
+  secrets:
+    AWS_ACCESS_KEY_ID:
+      vault:                       cicd/gitlab/$CI_PROJECT_PATH/AWS_ACCESS_KEY_ID@kv
+      file:                        false
+    AWS_SECRET_ACCESS_KEY:
+      vault:                       cicd/gitlab/$CI_PROJECT_PATH/AWS_SECRET_ACCESS_KEY@kv
+      file:                        false
+    DOCKER_HUB_USER:
+      vault:                       cicd/gitlab/parity/DOCKER_HUB_USER@kv
+      file:                        false
+    DOCKER_HUB_PASS:
+      vault:                       cicd/gitlab/parity/DOCKER_HUB_PASS@kv
+      file:                        false
+    GITHUB_PR_TOKEN:
+      vault:                       cicd/gitlab/parity/GITHUB_PR_TOKEN@kv
+      file:                        false
+    GITHUB_USER:
+      vault:                       cicd/gitlab/$CI_PROJECT_PATH/GITHUB_USER@kv
+      file:                        false
+    GITHUB_RELEASE_TOKEN:
+      vault:                       cicd/gitlab/$CI_PROJECT_PATH/GITHUB_RELEASE_TOKEN@kv
+      file:                        false
+    GITHUB_TOKEN:
+      vault:                       cicd/gitlab/$CI_PROJECT_PATH/GITHUB_TOKEN@kv
+      file:                        false
+    MATRIX_ACCESS_TOKEN:
+      vault:                       cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ACCESS_TOKEN@kv
+      file:                        false
+    MATRIX_ROOM_ID:
+      vault:                       cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ROOM_ID@kv
+      file:                        false
+    PARITYPR_USER:
+      vault:                       cicd/gitlab/$CI_PROJECT_PATH/PARITYPR_USER@kv
+      file:                        false
+    PARITYPR_PASS:
+      vault:                       cicd/gitlab/$CI_PROJECT_PATH/PARITYPR_PASS@kv
+      file:                        false
+    PIPELINE_TOKEN:
+      vault:                       cicd/gitlab/$CI_PROJECT_PATH/PIPELINE_TOKEN@kv
+      file:                        false
+    REL_MAN_ROOM_ID:
+      vault:                       cicd/gitlab/$CI_PROJECT_PATH/REL_MAN_ROOM_ID@kv
+      file:                        false
+    SSH_PRIVATE_KEY:
+      vault:                       cicd/gitlab/$CI_PROJECT_PATH/SSH_PRIVATE_KEY@kv
+      file:                        false
+
 #### stage:                        test
 
 check-runtime:
   stage:                           test
   image:                           paritytech/tools:latest
   <<:                              *kubernetes-env
   <<:                              *rules-pr-only
+  <<:                              *vault-secrets
   variables:
     GITLAB_API:                    "https://gitlab.parity.io/api/v4"
     GITHUB_API_PROJECT:            "parity%2Finfrastructure%2Fgithub-api"
@@ -120,6 +173,7 @@ test-deterministic-wasm:
   <<:                              *rules-test
   <<:                              *docker-env
   <<:                              *compiler-info
+  <<:                              *vault-secrets
   script:
     - ./scripts/gitlab/test_deterministic_wasm.sh
 
@@ -128,6 +182,7 @@ test-build-linux-stable:
   <<:                              *docker-env
   <<:                              *compiler-info
   <<:                              *collect-artifacts
+  <<:                              *vault-secrets
   variables:
     RUST_TOOLCHAIN: stable
     # Enable debug assertions since we are running optimized builds for testing
@@ -155,18 +210,30 @@ test-build-linux-stable:
     - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
     - echo -n ${VERSION} > ./artifacts/VERSION
     - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
-    - cp -r scripts/docker/* ./artifacts
+    - cp -r scripts/* ./artifacts
 
 check-runtime-benchmarks:
   stage:                           test
   <<:                              *rules-test
   <<:                              *docker-env
   <<:                              *compiler-info
+  <<:                              *vault-secrets
   script:
     # Check that the node will compile with `runtime-benchmarks` feature flag.
     - ./scripts/gitlab/check_runtime_benchmarks.sh
     - sccache -s
 
+check-no-default-features:
+  stage:                           test
+  <<:                              *rules-test
+  <<:                              *docker-env
+  <<:                              *compiler-info
+  <<:                              *vault-secrets
+  script:
+    # Check that polkadot-cli will compile no default features.
+    - ./scripts/gitlab/check_no_default_features.sh
+    - sccache -s
+
 spellcheck:
   stage:                           test
   <<:                              *docker-env
@@ -175,8 +242,9 @@ spellcheck:
     - cargo spellcheck --version
     # compare with the commit parent to the PR, given it's from a default branch
     - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH}
+    - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH}))
     - time cargo spellcheck check -vvv --cfg=scripts/gitlab/spellcheck.toml --checkers hunspell --code 1
-        -r $(git diff --name-only ${CI_COMMIT_SHA} $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH}))
+        $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH}))
   allow_failure:                   true
 
 build-adder-collator:
@@ -198,15 +266,17 @@ build-adder-collator:
     - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
     - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
     - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
-    - cp -r scripts/docker/* ./artifacts
+    - cp -r scripts/* ./artifacts
 
 #### stage:                        build
 
 check-transaction-versions:
-  image:                           node:15
+  # image must be ubuntu:20.04 based to match the linkers, this image has npm installed
+  image:                           paritytech/contracts-ci-linux:production
   stage:                           build
   <<:                              *rules-test
   <<:                              *docker-env
+  <<:                              *vault-secrets
   needs:
     - job:                         test-build-linux-stable
       artifacts:                   true
@@ -251,6 +321,7 @@ build-rustdoc:
 
 .build-push-image:                 &build-push-image
   <<:                              *kubernetes-env
+  <<:                              *vault-secrets
   image:                           quay.io/buildah/stable
   variables:                       &image-variables
     GIT_STRATEGY:                  none
@@ -281,15 +352,15 @@ build-rustdoc:
     - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
     - buildah push --format=v2s2 "$IMAGE_NAME:$EXTRATAG"
   after_script:
-    - buildah logout "$IMAGE_NAME"
+    - buildah logout --all
 
 publish-polkadot-image:
   stage:                           build
   <<:                              *build-push-image
   variables:
     <<:                            *image-variables
-    # scripts/docker/Dockerfile
-    DOCKERFILE:                    Dockerfile
+    # scripts/dockerfiles/polkadot_injected_debug.Dockerfile
+    DOCKERFILE:                    dockerfiles/polkadot_injected_debug.Dockerfile
     IMAGE_NAME:                    docker.io/paritypr/synth-wave
   rules:
     # Don't run on releases - this is handled by the Github Action here:
@@ -303,8 +374,8 @@ publish-polkadot-image:
       variables:
         <<:                        *image-variables
         IMAGE_NAME:                docker.io/parity/rococo
-        DOCKER_USER:               ${Docker_Hub_User_Parity}
-        DOCKER_PASS:               ${Docker_Hub_Pass_Parity}
+        DOCKER_USER:               ${DOCKER_HUB_USER}
+        DOCKER_PASS:               ${DOCKER_HUB_PASS}
   needs:
     - job:                         test-build-linux-stable
       artifacts:                   true
@@ -324,8 +395,8 @@ publish-adder-collator-image:
   <<:                              *build-push-image
   variables:
     <<:                            *image-variables
-    # scripts/docker/collator.Dockerfile
-    DOCKERFILE:                    collator.Dockerfile
+    # scripts/dockerfiles/collator_injected.Dockerfile
+    DOCKERFILE:                    dockerfiles/collator_injected.Dockerfile
     IMAGE_NAME:                    docker.io/paritypr/colander
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -336,7 +407,7 @@ publish-adder-collator-image:
     - job:                         build-adder-collator
       artifacts:                   true
   after_script:
-    - buildah logout "$IMAGE_NAME"
+    - buildah logout --all
     # pass artifacts to the simnet-tests job
     - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env
     - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env
@@ -380,6 +451,7 @@ publish-s3-release:                &publish-s3
     - job:                         test-build-linux-stable
       artifacts:                   true
   <<:                              *kubernetes-env
+  <<:                              *vault-secrets
   image:                           paritytech/awscli:latest
   variables:
     GIT_STRATEGY:                  none
@@ -415,6 +487,7 @@ publish-s3-release:                &publish-s3
 publish-rustdoc:
   stage:                           publish
   <<:                              *kubernetes-env
+  <<:                              *vault-secrets
   image:                           paritytech/tools:latest
   variables:
     GIT_DEPTH:                     100
@@ -504,4 +577,4 @@ simnet-tests:
   allow_failure:                   true
   retry: 2
   tags:
-    - parity-simnet
+    - polkadot-simnet