diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d150d96deedc..cc13966a2ba1d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -188,13 +188,13 @@ default: file: false AWX_TOKEN: vault: cicd/gitlab/$CI_PROJECT_PATH/AWX_TOKEN@kv - file: false + file: false CRATES_TOKEN: vault: cicd/gitlab/$CI_PROJECT_PATH/CRATES_TOKEN@kv file: false DOCKER_CHAOS_TOKEN: vault: cicd/gitlab/$CI_PROJECT_PATH/DOCKER_CHAOS_TOKEN@kv - file: false + file: false DOCKER_CHAOS_USER: vault: cicd/gitlab/$CI_PROJECT_PATH/DOCKER_CHAOS_USER@kv file: false @@ -544,9 +544,6 @@ check-polkadot-companion-build: <<: *docker-env <<: *test-refs-no-trigger <<: *vault-secrets - needs: - - job: test-linux-stable-int - artifacts: false script: - ./.maintain/gitlab/check_polkadot_companion_build.sh after_script: @@ -850,7 +847,7 @@ deploy-prometheus-alerting-rules: # Runs "quick" and "long" tests on nightly schedule and on commit / merge to master # A "quick" test is a smoke test where basic check-expect tests run by # checking values from metrics exposed by the app. -# A "long" test is the load testing where we send 50K transactions into the +# A "long" test is the load testing where we send 50K transactions into the # network and check if all completed successfully simnet-tests: stage: deploy diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index 72bfaf7151522..6cdf3f16ad299 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash # # check if a pr is compatible with polkadot companion pr or master if not # available @@ -9,7 +9,8 @@ # polkadot companion: paritytech/polkadot#567 # -set -e +set -eu -o pipefail +shopt -s inherit_errexit github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls" # use github api v3 in order to access the data without authentication @@ -40,18 +41,18 @@ EOT # Set the user name and email to make merging work git config --global user.name 'CI system' git config --global user.email '<>' +git config --global pull.rebase false # Merge master into our branch before building Polkadot to make sure we don't miss # any commits that are required by Polkadot. -git fetch --depth 100 origin -git merge origin/master +git pull origin master -# Clone the current Polkadot master branch into ./polkadot. -# NOTE: we need to pull enough commits to be able to find a common -# ancestor for successfully performing merges below. -git clone --depth 20 https://github.com/paritytech/polkadot.git +substrate_dir="$PWD" +polkadot_dir="$substrate_dir/polkadot" -cd polkadot +git clone --depth 1 https://github.com/paritytech/polkadot.git "$polkadot_dir" + +cd "$polkadot_dir" # either it's a pull request then check for a companion otherwise use # polkadot:master @@ -76,7 +77,8 @@ then boldprint "companion pr specified/detected: #${pr_companion}" git fetch origin refs/pull/${pr_companion}/head:pr/${pr_companion} git checkout pr/${pr_companion} - git merge origin/master + # we want this because `bot merge` will include master into the pr's branch before merging + git pull origin master else boldprint "no companion branch found - building polkadot:master" fi @@ -85,15 +87,131 @@ else boldprint "this is not a pull request - building polkadot:master" fi +our_crates=() +our_crates_source="git+https://github.com/paritytech/substrate" +discover_our_crates() { + local found + + cd "$substrate_dir" + + # workaround for early exits not being detected in command substitution + # https://unix.stackexchange.com/questions/541969/nested-command-substitution-does-not-stop-a-script-on-a-failure-even-if-e-and-s + local last_line + + while IFS= read -r crate; do + last_line="$crate" + # for avoiding duplicate entries + for our_crate in "${our_crates[@]}"; do + if [ "$crate" == "$our_crate" ]; then + found=true + break + fi + done + if [ "${found:-}" ]; then + unset found + else + our_crates+=("$crate") + fi + # dependencies with {"source": null} are the ones we own, hence the getpath($p)==null in the jq + # script below + done < <(cargo metadata --quiet --format-version=1 | jq -r ' + . as $in | + paths | + select(.[-1]=="source" and . as $p | $in | getpath($p)==null) as $path | + del($path[-1]) as $path | + $in | getpath($path + ["name"]) + ') + if [ -z "${last_line+_}" ]; then + echo "No lines were read for cargo metadata of $PWD (some error probably occurred)" + exit 1 + fi +} +discover_our_crates + +match_their_crates() { + local target_dir="$1" + shift + + cd "$target_dir" + + local target_name="$(basename "$target_dir")" + local crates_not_found=() + local found + + # workaround for early exits not being detected in command substitution + # https://unix.stackexchange.com/questions/541969/nested-command-substitution-does-not-stop-a-script-on-a-failure-even-if-e-and-s + local last_line + + # output will be consumed in the format: + # crate + # source + # crate + # ... + local next="crate" + while IFS= read -r line; do + last_line="$line" + case "$next" in + crate) + next="source" + crate="$line" + ;; + source) + next="crate" + if [ "$line" == "$our_crates_source" ] || [[ "$line" == "$our_crates_source?"* ]]; then + for our_crate in "${our_crates[@]}"; do + if [ "$our_crate" == "$crate" ]; then + found=true + break + fi + done + if [ "${found:-}" ]; then + unset found + else + # for avoiding duplicate entries + for crate_not_found in "${crates_not_found[@]}"; do + if [ "$crate_not_found" == "$crate" ]; then + found=true + break + fi + done + if [ "${found:-}" ]; then + unset found + else + crates_not_found+=("$crate") + fi + fi + fi + ;; + *) + echo "ERROR: Unknown state $next" + exit 1 + ;; + esac + done < <(cargo metadata --quiet --format-version=1 | jq -r ' + . as $in | + paths(select(type=="string")) | + select(.[-1]=="source") as $source_path | + del($source_path[-1]) as $path | + [$in | getpath($path + ["name"]), getpath($path + ["source"])] | + .[] + ') + if [ -z "${last_line+_}" ]; then + echo "No lines were read for cargo metadata of $PWD (some error probably occurred)" + exit 1 + fi + + if [ "${crates_not_found[@]}" ]; then + echo -e "Errors during crate matching\n" + printf "Failed to detect our crate \"%s\" referenced in $target_name\n" "${crates_not_found[@]}" + echo -e "\nNote: this error generally happens if you have deleted or renamed a crate and did not update it in $target_name. Consider opening a companion pull request on $target_name and referencing it in this pull request's description like:\n$target_name companion: [your companion PR here]" + exit 1 + fi +} +match_their_crates "$polkadot_dir" + # Patch all Substrate crates in Polkadot -diener patch --crates-to-patch ../ --substrate --path Cargo.toml - -# We need to update specifically our patched Substrate crates so that other -# crates that depend on them (e.g. Polkadot, BEEFY) use this unified version -# NOTE: There's no way to only update patched crates, so we use a heuristic -# of updating a crucial Substrate crate (`sp-core`) to minimize the impact of -# updating unrelated dependencies -cargo update -p sp-core +cd "$polkadot_dir" +diener patch --crates-to-patch "$substrate_dir" --substrate --path Cargo.toml # Test Polkadot pr or master branch with this Substrate commit. time cargo test --workspace --release --verbose --features=runtime-benchmarks