build.yml

# This file is part of .gitlab-ci.yml
# Here are all jobs that are executed during "build" stage

# build jobs from polkadot

build-linux-stable:
  stage: build
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .collect-artifacts
  variables:
    RUST_TOOLCHAIN: stable
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
    # Ensure we run the UI tests.
    RUN_UI_TESTS: 1
  script:
    - time cargo build --locked --profile testnet --features pyroscope,fast-runtime --bin polkadot --bin polkadot-prepare-worker --bin polkadot-execute-worker
    - time ROCOCO_EPOCH_DURATION=10  ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-10/
    - time ROCOCO_EPOCH_DURATION=100 ./polkadot/scripts/build-only-wasm.sh rococo-runtime $(pwd)/runtimes/rococo-runtime-100/
    - pwd
    - ls -alR runtimes
    # pack artifacts
    - mkdir -p ./artifacts
    - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
    - mv ./target/testnet/polkadot ./artifacts/.
    - mv ./target/testnet/polkadot-prepare-worker ./artifacts/.
    - mv ./target/testnet/polkadot-execute-worker ./artifacts/.
    - mv ./runtimes/ ./artifacts/.
    - pushd artifacts
    - sha256sum polkadot | tee polkadot.sha256
    - shasum -c polkadot.sha256
    - popd
    - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
    - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
    - echo -n ${VERSION} > ./artifacts/VERSION
    - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
    - echo -n ${CI_JOB_ID} > ./artifacts/BUILD_LINUX_JOB_ID
    - RELEASE_VERSION=$(./artifacts/polkadot -V | awk '{print $2}'| awk -F "-" '{print $1}')
    - echo -n "v${RELEASE_VERSION}" > ./artifacts/BUILD_RELEASE_VERSION
    - cp -r docker/* ./artifacts

build-test-collators:
  stage: build
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .collect-artifacts
  script:
    - time cargo build --locked --profile testnet -p test-parachain-adder-collator
    - time cargo build --locked --profile testnet -p test-parachain-undying-collator
    # pack artifacts
    - mkdir -p ./artifacts
    - mv ./target/testnet/adder-collator ./artifacts/.
    - mv ./target/testnet/undying-collator ./artifacts/.
    - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
    - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
    - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
    - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
    - cp -r ./docker/* ./artifacts

build-malus:
  stage: build
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .collect-artifacts
  script:
    - time cargo build --locked --profile testnet -p polkadot-test-malus --bin malus --bin polkadot-prepare-worker --bin polkadot-execute-worker
    # pack artifacts
    - mkdir -p ./artifacts
    - mv ./target/testnet/malus ./artifacts/.
    - mv ./target/testnet/polkadot-execute-worker ./artifacts/.
    - mv ./target/testnet/polkadot-prepare-worker ./artifacts/.
    - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
    - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
    - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
    - cp -r ./docker/* ./artifacts

build-staking-miner:
  stage: build
  extends:
    - .docker-env
    - .common-refs
    # - .collect-artifacts
  # DAG
  needs:
    - job: build-malus
      artifacts: false
  script:
    - time cargo build -q --locked --release --package staging-staking-miner
    # # pack artifacts
    # - mkdir -p ./artifacts
    # - mv ./target/release/staking-miner ./artifacts/.
    # - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
    # - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
    # - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
    # - cp -r ./scripts/* ./artifacts

build-rustdoc:
  stage: build
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
  variables:
    SKIP_WASM_BUILD: 1
  artifacts:
    name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
    when: on_success
    expire_in: 1 days
    paths:
      - ./crate-docs/
  script:
    # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
    # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable
    - time cargo doc --features try-runtime,experimental --workspace --no-deps
    - rm -f ./target/doc/.lock
    - mv ./target/doc ./crate-docs
    # Inject Simple Analytics (https://www.simpleanalytics.com/) privacy preserving tracker into
    # all .html files
    - |
      inject_simple_analytics() {
        local path="$1"
        local script_content="<script async defer src=\"https://apisa.parity.io/latest.js\"></script><noscript><img src=\"https://apisa.parity.io/latest.js\" alt=\"\" referrerpolicy=\"no-referrer-when-downgrade\" /></noscript>"

        # Function that inject script into the head of an html file using sed.
        process_file() {
            local file="$1"
            echo "Adding Simple Analytics script to $file"
            sed -i "s|</head>|$script_content</head>|" "$file"
        }
        export -f process_file

        # Modify .html files in parallel using xargs, otherwise it can take a long time.
        find "$path" -name '*.html' | xargs -I {} -P "$(nproc)" bash -c 'process_file "$@"' _ {}
      }
      inject_simple_analytics "./crate-docs"
    - echo "<meta http-equiv=refresh content=0;url=polkadot_service/index.html>" > ./crate-docs/index.html

build-implementers-guide:
  stage: build
  extends:
    - .kubernetes-env
    - .common-refs
    - .run-immediately
    # - .collect-artifacts
  # git depth is set on purpose: https://github.com/paritytech/polkadot/issues/6284
  variables:
    GIT_STRATEGY: clone
    GIT_DEPTH: 0
    CI_IMAGE: paritytech/mdbook-utils:e14aae4a-20221123
  script:
    - mdbook build ./polkadot/roadmap/implementers-guide
    - mkdir -p artifacts
    - mv polkadot/roadmap/implementers-guide/book artifacts/

build-short-benchmark:
  stage: build
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .collect-artifacts
  script:
    - cargo build --profile release --locked --features=runtime-benchmarks --bin polkadot --workspace
    - mkdir -p artifacts
    - target/release/polkadot --version
    - cp ./target/release/polkadot ./artifacts/

# build jobs from cumulus

build-linux-stable-cumulus:
  stage: build
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .collect-artifacts
  variables:
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
  script:
    - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___"
    - time cargo build --release --locked -p polkadot-parachain-bin --bin polkadot-parachain
    - echo "___Packing the artifacts___"
    - mkdir -p ./artifacts
    - mv ./target/release/polkadot-parachain ./artifacts/.
    - echo "___The VERSION is either a tag name or the curent branch if triggered not by a tag___"
    - echo ${CI_COMMIT_REF_NAME} | tee ./artifacts/VERSION

build-test-parachain:
  stage: build
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .collect-artifacts
  variables:
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
  script:
    - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___"
    - time cargo build --release --locked -p cumulus-test-service --bin test-parachain
    - echo "___Packing the artifacts___"
    - mkdir -p ./artifacts
    - mv ./target/release/test-parachain ./artifacts/.
    - mkdir -p ./artifacts/zombienet
    - mv ./target/release/wbuild/cumulus-test-runtime/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm ./artifacts/zombienet/.

# build runtime only if files in $RUNTIME_PATH/$RUNTIME_NAME were changed
.build-runtime-template: &build-runtime-template
  stage: build
  extends:
    - .docker-env
    - .test-refs-no-trigger-prs-only
    - .run-immediately
  variables:
    RUNTIME_PATH: "parachains/runtimes/assets"
  script:
    - cd ${RUNTIME_PATH}
    - for directory in $(echo */); do
      echo "_____Running cargo check for ${directory} ______";
      cd ${directory};
      pwd;
      SKIP_WASM_BUILD=1 cargo check --locked;
      cd ..;
      done

# DAG: build-runtime-assets -> build-runtime-collectives -> build-runtime-bridge-hubs
# DAG: build-runtime-assets -> build-runtime-collectives -> build-runtime-contracts
# DAG: build-runtime-assets -> build-runtime-starters -> build-runtime-testing
build-runtime-assets:
  <<: *build-runtime-template
  variables:
    RUNTIME_PATH: "cumulus/parachains/runtimes/assets"

build-runtime-collectives:
  <<: *build-runtime-template
  variables:
    RUNTIME_PATH: "cumulus/parachains/runtimes/collectives"
  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
  needs:
    - job: build-runtime-assets
      artifacts: false

build-runtime-bridge-hubs:
  <<: *build-runtime-template
  variables:
    RUNTIME_PATH: "cumulus/parachains/runtimes/bridge-hubs"
  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
  needs:
    - job: build-runtime-collectives
      artifacts: false

build-runtime-contracts:
  <<: *build-runtime-template
  variables:
    RUNTIME_PATH: "cumulus/parachains/runtimes/contracts"
  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
  needs:
    - job: build-runtime-collectives
      artifacts: false

build-runtime-starters:
  <<: *build-runtime-template
  variables:
    RUNTIME_PATH: "cumulus/parachains/runtimes/starters"
  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
  needs:
    - job: build-runtime-assets
      artifacts: false

build-runtime-testing:
  <<: *build-runtime-template
  variables:
    RUNTIME_PATH: "cumulus/parachains/runtimes/testing"
  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
  needs:
    - job: build-runtime-starters
      artifacts: false

build-short-benchmark-cumulus:
  stage: build
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .collect-artifacts
  script:
    - cargo build --profile release --locked --features=runtime-benchmarks -p polkadot-parachain-bin --bin polkadot-parachain
    - mkdir -p artifacts
    - target/release/polkadot-parachain --version
    - cp ./target/release/polkadot-parachain ./artifacts/

# substrate

build-linux-substrate:
  stage: build
  extends:
    - .docker-env
    - .common-refs
    - .collect-artifacts
  # DAG
  needs:
    - job: build-linux-stable
      artifacts: false
  variables:
    # this variable gets overriden by "rusty-cachier environment inject", use the value as default
    CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
  before_script:
    - mkdir -p ./artifacts/substrate/
    # tldr: we need to checkout the branch HEAD explicitly because of our dynamic versioning approach while building the substrate binary
    # see https://github.com/paritytech/ci_cd/issues/682#issuecomment-1340953589
    - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
  script:
    - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release -p node-cli
    - mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate
    - echo -n "Substrate version = "
    - if [ "${CI_COMMIT_TAG}" ]; then
      echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION;
      else
      ./artifacts/substrate/substrate --version |
      cut -d ' ' -f 2 | tee ./artifacts/substrate/VERSION;
      fi
    - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
    - cp -r ./docker/dockerfiles/substrate_injected.Dockerfile ./artifacts/substrate/
    # - printf '\n# building node-template\n\n'
    # - ./scripts/ci/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz

.build-subkey:
  stage: build
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    # - .collect-artifact
  variables:
    # this variable gets overriden by "rusty-cachier environment inject", use the value as default
    CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
  before_script:
    - mkdir -p ./artifacts/subkey
  script:
    - cd ./substrate/bin/utils/subkey
    - SKIP_WASM_BUILD=1 time cargo build --locked --release
    # - cd -
    # - mv $CARGO_TARGET_DIR/release/subkey ./artifacts/subkey/.
    # - echo -n "Subkey version = "
    # - ./artifacts/subkey/subkey --version |
    #   sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' |
    #   tee ./artifacts/subkey/VERSION;
    # - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
    # - cp -r ./scripts/ci/docker/subkey.Dockerfile ./artifacts/subkey/

build-subkey-linux:
  extends: .build-subkey
  # DAG
  needs:
    - job: build-staking-miner
      artifacts: false
# tbd
# build-subkey-macos:
#   extends: .build-subkey
#   # duplicating before_script & script sections from .build-subkey hidden job
#   # to overwrite rusty-cachier integration as it doesn't work on macos
#   before_script:
#     # skip timestamp script, the osx bash doesn't support printf %()T
#     - !reference [.job-switcher, before_script]
#     - mkdir -p ./artifacts/subkey
#   script:
#     - cd ./bin/utils/subkey
#     - SKIP_WASM_BUILD=1 time cargo build --locked --release
#     - cd -
#     - mv ./target/release/subkey ./artifacts/subkey/.
#     - echo -n "Subkey version = "
#     - ./artifacts/subkey/subkey --version |
#       sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' |
#       tee ./artifacts/subkey/VERSION;
#     - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
#     - cp -r ./scripts/ci/docker/subkey.Dockerfile ./artifacts/subkey/
#   after_script: [""]
#   tags:
#     - osx