From d8a74fe7290b012eb47e5c5f6519db0dd4d69a0b Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 28 Nov 2022 14:55:35 +0100 Subject: [PATCH 01/11] [DNM] debug implementer guide job --- .gitlab-ci.yml | 53 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8f70394cd5c..7eca8c2cca0f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -180,27 +180,42 @@ default: after_script: - buildah logout --all -include: - # weights jobs - - scripts/ci/gitlab/pipeline/weights.yml - # check jobs - - scripts/ci/gitlab/pipeline/check.yml - # test jobs - - scripts/ci/gitlab/pipeline/test.yml - # build jobs - - scripts/ci/gitlab/pipeline/build.yml - # short-benchmarks jobs - - scripts/ci/gitlab/pipeline/short-benchmarks.yml - # publish jobs - - scripts/ci/gitlab/pipeline/publish.yml - # zombienet jobs - - scripts/ci/gitlab/pipeline/zombienet.yml - +# include: +# # weights jobs +# - scripts/ci/gitlab/pipeline/weights.yml +# # check jobs +# - scripts/ci/gitlab/pipeline/check.yml +# # test jobs +# - scripts/ci/gitlab/pipeline/test.yml +# # build jobs +# - scripts/ci/gitlab/pipeline/build.yml +# # short-benchmarks jobs +# - scripts/ci/gitlab/pipeline/short-benchmarks.yml +# # publish jobs +# - scripts/ci/gitlab/pipeline/publish.yml +# # zombienet jobs +# - scripts/ci/gitlab/pipeline/zombienet.yml +build-implementers-guide: + stage: build + extends: + - .docker-env + - .test-refs + - .collect-artifacts-short + # git depth is set on purpose: https://github.com/paritytech/polkadot/issues/6284 + variables: + GIT_DEPTH: 0 + CI_IMAGE: paritytech/mdbook-utils:e14aae4a-20221123 + script: + - git log --pretty=oneline --reverse | head -1 + - mdbook build ./roadmap/implementers-guide + - mkdir -p artifacts + - mv roadmap/implementers-guide/book artifacts/ + - ls -la artifacts/ #### stage: .post -deploy-parity-testnet: +.deploy-parity-testnet: stage: .post extends: - .deploy-testnet-refs @@ -229,7 +244,7 @@ deploy-parity-testnet: project: "parity/infrastructure/ci_cd/pipeline-stopper" branch: "as-improve" -remove-cancel-pipeline-message: +.remove-cancel-pipeline-message: stage: .post rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs @@ -244,7 +259,7 @@ remove-cancel-pipeline-message: project: "parity/infrastructure/ci_cd/pipeline-stopper" branch: "as-improve" -cancel-pipeline-test-linux-stable: +.cancel-pipeline-test-linux-stable: extends: .cancel-pipeline-template needs: - job: test-linux-stable From 7563dbe147c73a34ef85876bb03d57753b4465e5 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 28 Nov 2022 15:00:21 +0100 Subject: [PATCH 02/11] remove git depth --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7eca8c2cca0f..2b4956309d68 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -204,7 +204,6 @@ build-implementers-guide: - .collect-artifacts-short # git depth is set on purpose: https://github.com/paritytech/polkadot/issues/6284 variables: - GIT_DEPTH: 0 CI_IMAGE: paritytech/mdbook-utils:e14aae4a-20221123 script: - git log --pretty=oneline --reverse | head -1 From f62cd58444b13fcea94653e90c1b921a26dc19eb Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 28 Nov 2022 15:01:47 +0100 Subject: [PATCH 03/11] change git strategy --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b4956309d68..b305c74896b0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -204,6 +204,7 @@ build-implementers-guide: - .collect-artifacts-short # git depth is set on purpose: https://github.com/paritytech/polkadot/issues/6284 variables: + GIT_STRATEGY: clone CI_IMAGE: paritytech/mdbook-utils:e14aae4a-20221123 script: - git log --pretty=oneline --reverse | head -1 From a86d440bde45621edc6dd6f539225747f73b7076 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 28 Nov 2022 15:04:02 +0100 Subject: [PATCH 04/11] add git depth --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b305c74896b0..17870e2462e7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -205,6 +205,7 @@ build-implementers-guide: # git depth is set on purpose: https://github.com/paritytech/polkadot/issues/6284 variables: GIT_STRATEGY: clone + GIT_DEPTH: 0 CI_IMAGE: paritytech/mdbook-utils:e14aae4a-20221123 script: - git log --pretty=oneline --reverse | head -1 From 8c5ade7c8953caf1dc174ee2676a4a896f0d2035 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 28 Nov 2022 15:04:24 +0100 Subject: [PATCH 05/11] try k8s runner --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 17870e2462e7..d6af505c2753 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -199,7 +199,7 @@ default: build-implementers-guide: stage: build extends: - - .docker-env + - .kubernetes-env - .test-refs - .collect-artifacts-short # git depth is set on purpose: https://github.com/paritytech/polkadot/issues/6284 From 56b10623f92ebc2284e4420518858b552badd322 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 28 Nov 2022 15:06:13 +0100 Subject: [PATCH 06/11] fix k8s template --- .gitlab-ci.yml | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d6af505c2753..2162725598b9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,6 +40,13 @@ variables: default: cache: {} + retry: + max: 2 + when: + - runner_system_failure + - unknown_failure + - api_failure + interruptible: true .collect-artifacts: artifacts: @@ -72,25 +79,12 @@ default: dotenv: pipeline-stopper.env .kubernetes-env: - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - interruptible: true + image: "${CI_IMAGE}" tags: - kubernetes-parity-build .docker-env: image: "${CI_IMAGE}" - retry: - max: 2 - when: - - runner_system_failure - - unknown_failure - - api_failure - interruptible: true tags: - linux-docker From fa5dc6731a39e5d4fb172188ce9804b8ac1b3883 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 28 Nov 2022 15:09:39 +0100 Subject: [PATCH 07/11] test job --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2162725598b9..03fa579395e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -202,11 +202,11 @@ build-implementers-guide: GIT_DEPTH: 0 CI_IMAGE: paritytech/mdbook-utils:e14aae4a-20221123 script: - - git log --pretty=oneline --reverse | head -1 - mdbook build ./roadmap/implementers-guide - mkdir -p artifacts - mv roadmap/implementers-guide/book artifacts/ - - ls -la artifacts/ + - cat roadmap/implementers-guide/book/html/runtime/shared.html | grep "Last change:" + - echo "Should be 0a633cd4df5" #### stage: .post From 5d089c7abd09a6b1af31081ed1455cb6ee7dc323 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 28 Nov 2022 15:12:13 +0100 Subject: [PATCH 08/11] fix test --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 03fa579395e5..9dac861ffdd2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -205,7 +205,7 @@ build-implementers-guide: - mdbook build ./roadmap/implementers-guide - mkdir -p artifacts - mv roadmap/implementers-guide/book artifacts/ - - cat roadmap/implementers-guide/book/html/runtime/shared.html | grep "Last change:" + - cat artifacts/html/runtime/shared.html | grep "Last change:" - echo "Should be 0a633cd4df5" #### stage: .post From 41586da771bcaf8064f739a8cfa708a4e8bfe3a9 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 28 Nov 2022 15:16:12 +0100 Subject: [PATCH 09/11] fix --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9dac861ffdd2..25304201af36 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -204,9 +204,9 @@ build-implementers-guide: script: - mdbook build ./roadmap/implementers-guide - mkdir -p artifacts - - mv roadmap/implementers-guide/book artifacts/ - - cat artifacts/html/runtime/shared.html | grep "Last change:" + - cat roadmap/implementers-guide/book/html/runtime/shared.html | grep "Last change:" - echo "Should be 0a633cd4df5" + - mv roadmap/implementers-guide/book artifacts/ #### stage: .post From 43c71220b89ca729be6c781a4797f2db14f19d5c Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 28 Nov 2022 15:31:02 +0100 Subject: [PATCH 10/11] return pipeline back --- .gitlab-ci.yml | 56 +++++++++----------------- scripts/ci/gitlab/pipeline/build.yml | 6 +-- scripts/ci/gitlab/pipeline/publish.yml | 13 +++++- 3 files changed, 31 insertions(+), 44 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 25304201af36..8013e871f6e4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,6 +33,7 @@ variables: GIT_DEPTH: 100 CI_SERVER_NAME: "GitLab CI" CI_IMAGE: "paritytech/ci-linux:production" + BUILDAH_IMAGE: "quay.io/buildah/stable:v1.27" DOCKER_OS: "debian:stretch" ARCH: "x86_64" ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.2.78" @@ -144,9 +145,6 @@ default: - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 .build-push-image: - extends: - - .kubernetes-env - image: quay.io/buildah/stable:v1.27 before_script: - test -s ./artifacts/VERSION || exit 1 - test -s ./artifacts/EXTRATAG || exit 1 @@ -174,39 +172,21 @@ default: after_script: - buildah logout --all -# include: -# # weights jobs -# - scripts/ci/gitlab/pipeline/weights.yml -# # check jobs -# - scripts/ci/gitlab/pipeline/check.yml -# # test jobs -# - scripts/ci/gitlab/pipeline/test.yml -# # build jobs -# - scripts/ci/gitlab/pipeline/build.yml -# # short-benchmarks jobs -# - scripts/ci/gitlab/pipeline/short-benchmarks.yml -# # publish jobs -# - scripts/ci/gitlab/pipeline/publish.yml -# # zombienet jobs -# - scripts/ci/gitlab/pipeline/zombienet.yml - -build-implementers-guide: - stage: build - extends: - - .kubernetes-env - - .test-refs - - .collect-artifacts-short - # git depth is set on purpose: https://github.com/paritytech/polkadot/issues/6284 - variables: - GIT_STRATEGY: clone - GIT_DEPTH: 0 - CI_IMAGE: paritytech/mdbook-utils:e14aae4a-20221123 - script: - - mdbook build ./roadmap/implementers-guide - - mkdir -p artifacts - - cat roadmap/implementers-guide/book/html/runtime/shared.html | grep "Last change:" - - echo "Should be 0a633cd4df5" - - mv roadmap/implementers-guide/book artifacts/ +include: + # weights jobs + - scripts/ci/gitlab/pipeline/weights.yml + # check jobs + - scripts/ci/gitlab/pipeline/check.yml + # test jobs + - scripts/ci/gitlab/pipeline/test.yml + # build jobs + - scripts/ci/gitlab/pipeline/build.yml + # short-benchmarks jobs + - scripts/ci/gitlab/pipeline/short-benchmarks.yml + # publish jobs + - scripts/ci/gitlab/pipeline/publish.yml + # zombienet jobs + - scripts/ci/gitlab/pipeline/zombienet.yml #### stage: .post @@ -239,7 +219,7 @@ build-implementers-guide: project: "parity/infrastructure/ci_cd/pipeline-stopper" branch: "as-improve" -.remove-cancel-pipeline-message: +remove-cancel-pipeline-message: stage: .post rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs @@ -254,7 +234,7 @@ build-implementers-guide: project: "parity/infrastructure/ci_cd/pipeline-stopper" branch: "as-improve" -.cancel-pipeline-test-linux-stable: +cancel-pipeline-test-linux-stable: extends: .cancel-pipeline-template needs: - job: test-linux-stable diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index 0a01ea9d8ccd..1553456a9e7b 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -22,7 +22,6 @@ build-linux-stable: RUN_UI_TESTS: 1 script: - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot - - sccache -s # pack artifacts - mkdir -p ./artifacts - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name @@ -98,7 +97,6 @@ build-malus: - .collect-artifacts script: - time cargo build --profile testnet --verbose -p polkadot-test-malus - - sccache -s # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/malus ./artifacts/. @@ -165,18 +163,18 @@ build-implementers-guide: - job: test-deterministic-wasm artifacts: false extends: - - .docker-env + - .kubernetes-env - .test-refs - .collect-artifacts-short # git depth is set on purpose: https://github.com/paritytech/polkadot/issues/6284 variables: + GIT_STRATEGY: clone GIT_DEPTH: 0 CI_IMAGE: paritytech/mdbook-utils:e14aae4a-20221123 script: - mdbook build ./roadmap/implementers-guide - mkdir -p artifacts - mv roadmap/implementers-guide/book artifacts/ - - ls -la artifacts/ build-short-benchmark: stage: build diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml index 9d638ad7b84a..3484fcae336e 100644 --- a/scripts/ci/gitlab/pipeline/publish.yml +++ b/scripts/ci/gitlab/pipeline/publish.yml @@ -7,6 +7,7 @@ publish-polkadot-debug-image: stage: publish extends: + - .kubernetes-env - .build-push-image rules: # Don't run when triggered from another pipeline @@ -18,6 +19,7 @@ publish-polkadot-debug-image: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 variables: + CI_IMAGE: ${BUILDAH_IMAGE} GIT_STRATEGY: none DOCKER_USER: ${PARITYPR_USER} DOCKER_PASS: ${PARITYPR_PASS} @@ -42,9 +44,11 @@ publish-test-collators-image: # service image for Simnet stage: publish extends: + - .kubernetes-env - .build-push-image - .zombienet-refs variables: + CI_IMAGE: ${BUILDAH_IMAGE} GIT_STRATEGY: none DOCKER_USER: ${PARITYPR_USER} DOCKER_PASS: ${PARITYPR_PASS} @@ -68,9 +72,11 @@ publish-malus-image: # service image for Simnet stage: publish extends: + - .kubernetes-env - .build-push-image - .zombienet-refs variables: + CI_IMAGE: ${BUILDAH_IMAGE} GIT_STRATEGY: none DOCKER_USER: ${PARITYPR_USER} DOCKER_PASS: ${PARITYPR_PASS} @@ -93,9 +99,11 @@ publish-malus-image: publish-staking-miner-image: stage: publish extends: + - .kubernetes-env - .build-push-image - .publish-refs variables: + CI_IMAGE: ${BUILDAH_IMAGE} # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile IMAGE_NAME: docker.io/paritytech/staking-miner @@ -114,8 +122,8 @@ publish-s3-release: needs: - job: build-linux-stable artifacts: true - image: paritytech/awscli:latest variables: + CI_IMAGE: paritytech/awscli:latest GIT_STRATEGY: none PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" rules: @@ -152,7 +160,8 @@ publish-rustdoc: stage: publish extends: - .kubernetes-env - image: paritytech/tools:latest + variables: + CI_IMAGE: paritytech/tools:latest rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never From 9df4a0f68dbc71f7a8095e8aba776b5bf52c307a Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 28 Nov 2022 16:00:06 +0100 Subject: [PATCH 11/11] enable disabled deploy-parity-testnet --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8013e871f6e4..57a76ad2c857 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -190,7 +190,7 @@ include: #### stage: .post -.deploy-parity-testnet: +deploy-parity-testnet: stage: .post extends: - .deploy-testnet-refs