[DOPS-2942]: Update substrate build

.dockerignore

@@ -14,3 +14,4 @@ housekeeping
 LICENSE
 rustfmt.toml
 db*
+.env

.gitignore

@@ -27,4 +27,4 @@ db?
 .vscode/launch.json
 
 # Generated folder for benchmark results (order book extra benchmarks)
-benches/
+benches/

Jenkinsfile

@@ -1,16 +1,20 @@
-@Library('jenkins-library') _
+@Library('jenkins-library@feature/dops-2942-update_rust_lib') _
 
 def pipeline = new org.rust.AppPipeline(steps: this,
       initSubmodules: true,
-      envImageName: 'docker.soramitsu.co.jp/sora2/env:sub4',
+      envImageName: 'docker.soramitsu.co.jp/sora2/env:env',
       appImageName: 'docker.soramitsu.co.jp/sora2/substrate',
       codeCoverageCommand: './housekeeping/coverage.sh',
       cargoDoc: true,
       smartContractScanner: false,
+      clippyLinter: false,
       cargoClippyTag: ':substrate',
       cargoClippyCmds: ['housekeeping/clippy.sh'],
-      buildTestCmds: ['housekeeping/build.sh'],
+      buildTestCmds: 'housekeeping/build.sh',
       buildArtifacts: 'framenode_runtime.compact.compressed.wasm, subwasm_report.json, pallet_list.txt',
-      pushToPublicRegistry: true
+      pushToPublicRegistry: true,
+      sonarProjectKey: 'sora:sora2-network',
+      sonarProjectName: 'sora2-network',
+      dojoProductType: 'sora'
 )
 pipeline.runPipeline()

Jenkinsfile-env

@@ -4,9 +4,9 @@ def pipeline = new org.docker.AppPipeline(steps: this,
     dockerImageName:              'sora2/env',
     dockerRegistryCred:           'bot-sora2-rw',
     dockerFileName:               "housekeeping/docker/env/Dockerfile",
-    triggerCommonBuildExpression: (env.BRANCH_NAME in ['substrate-4.0.0']),
+    triggerCommonBuildExpression: (env.BRANCH_NAME in ['develop', 'master']),
     triggerCommonBuildGrepRegex:  'housekeeping',
-    secretScannerExclusion:       '.*Cargo.toml',
+    secretScannerExclusion:       '.*Cargo.toml$|.*env\$',
     nameCI:                       'env-CI',
-    dockerImageTags:              ['substrate-4.0.0':'sub4'])
+    dockerImageTags:              ['master': 'env', 'develop': 'env'])
 pipeline.runPipeline()

bridge-docker/.env

@@ -0,0 +1,3 @@
+DATABASE_URL="postgres://postgres:${DATABASE_PASSWORD}@bridge-postgres:5432/postgres?sslmode=disable"
+DATABASE_PASSWORD="postgres"
+SECRET_KEY_BASE="56NtB48ear7+wMSf0IQuWDAAazhpb31qyc7GiyspBP2vh7t5zlCsF5QDv76chXeN"

housekeeping/build.sh

@@ -7,50 +7,63 @@ wasmReportFile='subwasm_report.json'
 PACKAGE='framenode-runtime'
 RUSTFLAGS='-Dwarnings'
 RUNTIME_DIR='runtime'
+allfeatures='private-net,wip,ready-to-test'
 
-if [[ $buildTag != null ]] && [[ ${TAG_NAME} != null || ${TAG_NAME} != '' ]]; then
-    printf "Tag is %s\n" $buildTag ${TAG_NAME}
-else
-    printf "⚡️ There is no tag here. "
-fi
+# build func
+test() {
+    if [[ $buildTag != null ]] && [[ ${TAG_NAME} != null || ${TAG_NAME} != '' ]]; then
+        printf "⚡️ Testing with features: private-net runtime-benchmarks\n"
+        cargo test --release --features "private-net runtime-benchmarks"
+    elif [[ ${prBranch} = 'master' ]]; then
+        printf "⚡️ This is "${prbranch}" Running tests and migrations %s\n"
+        RUST_LOG="debug cargo test --features try-runtime -- run_migrations"
+    else
+        printf "⚡️ Running Tests for code coverage only\n"
+        export RUSTFLAGS="-Cinstrument-coverage"
+        export SKIP_WASM_BUILD=1
+        export LLVM_PROFILE_FILE="sora2-%p-%m.profraw"
+        rm -rf ~/.cargo/.package-cache
+        cargo fmt -- --check > /dev/null
+        cargo test --features $allfeatures
+    fi
+}
 
-# build
-# If TAG_NAME is defined, build for a specific tag
-if [[ $buildTag != null ]] && [[ ${TAG_NAME} != null || ${TAG_NAME} != '' ]]; then
+build() {
+    printf "Tag is %s\n" ${TAG_NAME}
+    printf "BuildTag is %s\n" ${buildTag}
+    sudoCheckStatus="0"
     if [[ ${TAG_NAME} =~ 'benchmarking'* ]]; then
         featureList='private-net runtime-benchmarks'
-        sudoCheckStatus=0
     elif [[ ${TAG_NAME} =~ 'stage'* ]]; then
         featureList='private-net include-real-files ready-to-test'
-        sudoCheckStatus=0
-    elif [[ ${TAG_NAME} =~ 'test'* ]]; then
+    elif [[ ${TAG_NAME} =~ 'test'* || -n ${buildTag} ]]; then
         featureList='private-net include-real-files reduced-pswap-reward-periods ready-to-test'
-        sudoCheckStatus=0
     elif [[ -n ${TAG_NAME} && ${TAG_NAME} != 'predev' ]]; then
         featureList='include-real-files'
-        sudoCheckStatus=101
+        sudoCheckStatus="101"
+    elif [[ -n ${buildTag} ]]; then
+        featureList='private-net include-real-files reduced-pswap-reward-periods wip ready-to-test'
     fi
-    printf "Building with features: %s\n" "$featureList"
-    printf "Checking sudo pallet: %s\n" "$sudoCheckStatus"
-    cargo test --release --features "private-net runtime-benchmarks"
+    printf "⚡️ Building with features: %s\n" "$featureList"
+    printf "⚡️ Checking sudo pallet: %s\n" "$sudoCheckStatus"
     rm -rf target
     cargo build --release --features "$featureList"
     mv ./target/release/framenode .
-    mv ./target/release/relayer ./relayer.bin
     mv ./target/release/wbuild/framenode-runtime/framenode_runtime.compact.compressed.wasm ./framenode_runtime.compact.compressed.wasm
     subwasm --json info framenode_runtime.compact.compressed.wasm > $wasmReportFile
     subwasm metadata framenode_runtime.compact.compressed.wasm > $palletListFile
     set +e
     subwasm metadata -m Sudo framenode_runtime.compact.compressed.wasm
-    if [[ $(echo $?) -eq $sudoCheckStatus ]]; then echo "✅ sudo check is successful!"; else echo "❌ sudo check is failed!"; exit 1; fi
-else
-    # If TAG_NAME is not defined, run tests and checks
-    if [[ $prBranch == 'master' ]]; then
-        RUST_LOG="debug cargo test --features try-runtime -- run_migrations"
+    if [[ $? -eq $sudoCheckStatus ]]; then 
+        echo "✅ sudo check is successful!"
+    else 
+        echo "❌ sudo check is failed!"
+        exit 1
     fi
-    printf "⚡️ only tests run %s\n"
-    rm -rf ~/.cargo/.package-cache
-    cargo fmt -- --check > /dev/null
-    cargo test
-    cargo test --features "private-net wip ready-to-test runtime-benchmarks"
+}
+
+if [ "$(type -t $1)" = "function" ]; then
+    "$1"
+else
+    echo "Func '$1' is not exists in this workflow. Skipped."
 fi

housekeeping/clippy.sh

@@ -4,8 +4,8 @@ set -e
 if [ "$pr" = true ] && [ "$prBranch" != "master" ]; then
     printf "👷‍♂️ starting clippy \n"
     SKIP_WASM_BUILD=1 cargo clippy --features private-net,ready-to-test,runtime-benchmarks -- -D warnings
-    SKIP_WASM_BUILD=1 cargo clippy --features private-net,ready-to-test,wip,runtime-benchmarks -- -D warnings
+    SKIP_WASM_BUILD=1 cargo clippy --features private-net,ready-to-test,wip,runtime-benchmarks --message-format=json -- -D warnings > clippy_report.json
 else
     printf "👷‍♂️ starting a regular clippy \n"
-    cargo clippy || exit 0
+    cargo clippy --message-format=json > clippy_report.json || exit 0
 fi

housekeeping/coverage.sh

@@ -1,10 +1,9 @@
 #!/bin/sh
+set -e
 
-export RUSTFLAGS="-Cinstrument-coverage"
-export SKIP_WASM_BUILD=1
-export LLVM_PROFILE_FILE="sora2-%p-%m.profraw"
+printf '⚡️ Running coverage %s\n'
+# coverage and generate report
+grcov . --binary-path ./target/debug -s . -t lcov --branch -o ./lcov_report --ignore-not-existing --ignore  "/opt/cargo/**" "target/debug" "node/src" --log-level="ERROR" --llvm-path='/usr/lib/llvm-14/bin'
 
-cargo test --features private-net
-
-grcov . --binary-path ./target/debug -s . -t cobertura --branch -o ./cobertura_report --ignore-not-existing --ignore  "/opt/cargo/**" "target/debug" "node/src"
-find . -type f -name '*.profraw' -delete
+# delete generated *.profraw
+find . -type f -name '*.profraw' -delete

housekeeping/docker/env/Dockerfile

@@ -1,59 +1,45 @@
-FROM debian:bullseye-slim
+FROM debian:bullseye-20240211-slim
 
-# Set environment variables
-ENV RUSTUP_HOME="/opt/rustup"
-ENV CARGO_HOME="/opt/cargo"
-ENV PATH="$PATH:$CARGO_HOME/bin"
+ENV RUSTUP_HOME="/opt/rust"
+ENV CARGO_HOME="/opt/rust"
+ENV PATH="$PATH:$RUSTUP_HOME/bin"
 ENV CARGO_BUILD_DEP_INFO_BASEDIR="."
-ENV CC=clang-13
-ENV CXX=clang++-13
+ENV RUST_VERSION=nightly-2024-01-15
+ENV TZ=Europe/Moscow
+ENV CC=clang-14
+ENV CXX=clang++-14
+ENV SUBWASM_VER=v0.14.0
+ENV GRCOV_VERSION=0.8.19
+ENV LLVM_TOOLS_VER=14
+ENV CCLANG_VER=14
+ARG DEBIAN_FRONTEND=noninteractive
 
-# Install dependencies
 RUN apt-get update && \
     apt-get install --no-install-recommends -y \
     ca-certificates apt-transport-https gnupg \
     libssl-dev pkg-config \
-    curl \
+    wget \
     git binaryen \
     make cmake libssl-dev \
     software-properties-common && \
-    apt-get autoremove -y && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN curl -s https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-    echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-13 main" >> /etc/apt/sources.list.d/llvm-toochain-bullseye-13.list && \
-    echo "deb-src http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-13 main" >> /etc/apt/sources.list.d/llvm-toochain-bullseye-13.list && \
-    apt-get -y update && \
-    apt-get install -y --no-install-recommends \
-    clang-13 lldb-13 lld-13 libclang-13-dev llvm-13 protobuf-compiler && \
-    update-alternatives --install /usr/bin/cc cc /usr/bin/clang-13 100 && \
-    apt-get autoremove -y && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-# Install docker
-RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
-RUN add-apt-repository \
-    "deb https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \
+    wget -O - https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain ${RUST_VERSION} && \
+    rustup toolchain install ${RUST_VERSION} && \
+    rustup target add wasm32-unknown-unknown --toolchain ${RUST_VERSION} && \
+    wget https://apt.llvm.org/llvm.sh && \
+    chmod +x llvm.sh && \
+    ./llvm.sh ${LLVM_TOOLS_VER} && \
+    rm llvm.sh && \
     apt-get update && \
     apt-get install --no-install-recommends -y \
-    docker-ce docker-ce-cli containerd.io && \
+    clang-${CCLANG_VER} lldb-${CCLANG_VER} lld-${CCLANG_VER} libclang-${CCLANG_VER}-dev llvm-${LLVM_TOOLS_VER} protobuf-compiler && \
+    update-alternatives --install /usr/bin/cc cc /usr/bin/${CC} 100 && \
+    wget https://github.com/chevdor/subwasm/releases/download/${SUBWASM_VER}/subwasm_linux_amd64_${SUBWASM_VER}.deb -O subw.deb && \
+    dpkg -i subw.deb && rm subw.deb && \
+    rustup update && \
+    cargo install grcov --version ${GRCOV_VERSION} && \
+    rustup component add llvm-tools-preview && \
     apt-get autoremove -y && \
     apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-# Install rust
-ENV RUST_NIGHTLY_VERSION=nightly-2023-03-21
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain stable && \
-    rustup toolchain install ${RUST_NIGHTLY_VERSION} && \
-    rustup target add wasm32-unknown-unknown && \
-    rustup target add wasm32-unknown-unknown --toolchain ${RUST_NIGHTLY_VERSION} && \
-    rm -rf "${CARGO_HOME}/registry" "${CARGO_HOME}/git"
-
-RUN curl https://github.com/chevdor/subwasm/releases/download/v0.17.0/subwasm_linux_amd64_v0.17.0.deb -sSL -o subw.deb && \
-    dpkg -i subw.deb && rm subw.deb
-
-RUN cargo install grcov && \
-    rustup component add llvm-tools-preview && \
-    rm -rf "${CARGO_HOME}/registry" "${CARGO_HOME}/git"
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf "${CARGO_HOME}/git" && \
+    ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone

housekeeping/docker/release/Dockerfile

@@ -1,5 +1,5 @@
 FROM debian:bullseye
-# Install dependencies
+
 RUN apt-get update && \
     apt-get install --no-install-recommends -y \
     ca-certificates nano curl && \
@@ -13,6 +13,5 @@ RUN mkdir /chain && \
 USER substrate
 
 COPY ./framenode /usr/local/bin/framenode
-COPY ./relayer.bin /usr/local/bin/relayer
 
 ENTRYPOINT ["framenode"]