diff --git a/.dockerignore b/.dockerignore index 4200b300ab..8e1e035a44 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,4 @@ housekeeping LICENSE rustfmt.toml db* +.env diff --git a/.gitignore b/.gitignore index 13f0ddb37f..e842c895ff 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,4 @@ db? .vscode/launch.json # Generated folder for benchmark results (order book extra benchmarks) -benches/ +benches/ \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index f8608eaff5..2fe6d07fcc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,15 +2,19 @@ def pipeline = new org.rust.AppPipeline(steps: this, initSubmodules: true, - envImageName: 'docker.soramitsu.co.jp/sora2/env:sub4', + envImageName: 'docker.soramitsu.co.jp/sora2/env:env', appImageName: 'docker.soramitsu.co.jp/sora2/substrate', codeCoverageCommand: './housekeeping/coverage.sh', cargoDoc: true, smartContractScanner: false, + clippyLinter: false, cargoClippyTag: ':substrate', cargoClippyCmds: ['housekeeping/clippy.sh'], - buildTestCmds: ['housekeeping/build.sh'], + buildTestCmds: 'housekeeping/build.sh', buildArtifacts: 'framenode_runtime.compact.compressed.wasm, subwasm_report.json, pallet_list.txt', - pushToPublicRegistry: true + pushToPublicRegistry: true, + sonarProjectKey: 'sora:sora2-network', + sonarProjectName: 'sora2-network', + dojoProductType: 'sora' ) pipeline.runPipeline() diff --git a/Jenkinsfile-env b/Jenkinsfile-env index 02e25ee8cd..9d58eec620 100644 --- a/Jenkinsfile-env +++ b/Jenkinsfile-env @@ -4,9 +4,9 @@ def pipeline = new org.docker.AppPipeline(steps: this, dockerImageName: 'sora2/env', dockerRegistryCred: 'bot-sora2-rw', dockerFileName: "housekeeping/docker/env/Dockerfile", - triggerCommonBuildExpression: (env.BRANCH_NAME in ['substrate-4.0.0']), + triggerCommonBuildExpression: (env.BRANCH_NAME in ['develop']), triggerCommonBuildGrepRegex: 'housekeeping', - secretScannerExclusion: '.*Cargo.toml', + secretScannerExclusion: '.*Cargo.toml$|.*env\$', nameCI: 'env-CI', - dockerImageTags: ['substrate-4.0.0':'sub4']) + dockerImageTags: ['develop': 'env']) pipeline.runPipeline() diff --git a/bridge-docker/.env b/bridge-docker/.env new file mode 100644 index 0000000000..b19f836dd6 --- /dev/null +++ b/bridge-docker/.env @@ -0,0 +1,3 @@ +DATABASE_URL="postgres://postgres:${DATABASE_PASSWORD}@bridge-postgres:5432/postgres?sslmode=disable" +DATABASE_PASSWORD="postgres" +SECRET_KEY_BASE="56NtB48ear7+wMSf0IQuWDAAazhpb31qyc7GiyspBP2vh7t5zlCsF5QDv76chXeN" \ No newline at end of file diff --git a/housekeeping/build.sh b/housekeeping/build.sh index 1dddd10810..573e7186cb 100755 --- a/housekeeping/build.sh +++ b/housekeeping/build.sh @@ -7,50 +7,63 @@ wasmReportFile='subwasm_report.json' PACKAGE='framenode-runtime' RUSTFLAGS='-Dwarnings' RUNTIME_DIR='runtime' +allfeatures='private-net,ready-to-test' -if [[ $buildTag != null ]] && [[ ${TAG_NAME} != null || ${TAG_NAME} != '' ]]; then - printf "Tag is %s\n" $buildTag ${TAG_NAME} -else - printf "⚡️ There is no tag here. " -fi +# build func +test() { + if [[ -n ${TAG_NAME} ]]; then + printf "⚡️ Testing with features: private-net runtime-benchmarks\n" + cargo test --release --features "private-net runtime-benchmarks" + elif [[ $prBranch = 'master' ]]; then + printf "⚡️ This is "$prbranch" Running tests and migrations %s\n" + export RUST_LOG="debug" + cargo test --features try-runtime -- run_migrations + elif [[ -n $buildTag || $pr = true ]]; then + printf "⚡️ Running Tests for code coverage only\n" + export RUSTFLAGS="-Cinstrument-coverage" + export SKIP_WASM_BUILD=1 + export LLVM_PROFILE_FILE="sora2-%p-%m.profraw" + rm -rf ~/.cargo/.package-cache + cargo fmt -- --check > /dev/null + cargo test --features $allfeatures -- --test-threads=1 + fi +} -# build -# If TAG_NAME is defined, build for a specific tag -if [[ $buildTag != null ]] && [[ ${TAG_NAME} != null || ${TAG_NAME} != '' ]]; then +build() { + printf "Tag is %s\n" ${TAG_NAME} + printf "BuildTag is %s\n" ${buildTag} + sudoCheckStatus="0" if [[ ${TAG_NAME} =~ 'benchmarking'* ]]; then featureList='private-net runtime-benchmarks' - sudoCheckStatus=0 elif [[ ${TAG_NAME} =~ 'stage'* ]]; then featureList='private-net include-real-files ready-to-test' - sudoCheckStatus=0 elif [[ ${TAG_NAME} =~ 'test'* ]]; then featureList='private-net include-real-files reduced-pswap-reward-periods ready-to-test' - sudoCheckStatus=0 elif [[ -n ${TAG_NAME} && ${TAG_NAME} != 'predev' ]]; then featureList='include-real-files' - sudoCheckStatus=101 + sudoCheckStatus="101" + elif [[ -n $buildTag ]]; then + featureList='private-net include-real-files reduced-pswap-reward-periods wip ready-to-test' fi - printf "Building with features: %s\n" "$featureList" - printf "Checking sudo pallet: %s\n" "$sudoCheckStatus" - cargo test --release --features "private-net runtime-benchmarks" - rm -rf target + printf "⚡️ Building with features: %s\n" "$featureList" + printf "⚡️ Checking sudo pallet: %s\n" "$sudoCheckStatus" cargo build --release --features "$featureList" mv ./target/release/framenode . - mv ./target/release/relayer ./relayer.bin mv ./target/release/wbuild/framenode-runtime/framenode_runtime.compact.compressed.wasm ./framenode_runtime.compact.compressed.wasm subwasm --json info framenode_runtime.compact.compressed.wasm > $wasmReportFile subwasm metadata framenode_runtime.compact.compressed.wasm > $palletListFile set +e subwasm metadata -m Sudo framenode_runtime.compact.compressed.wasm - if [[ $(echo $?) -eq $sudoCheckStatus ]]; then echo "✅ sudo check is successful!"; else echo "❌ sudo check is failed!"; exit 1; fi -else - # If TAG_NAME is not defined, run tests and checks - if [[ $prBranch == 'master' ]]; then - RUST_LOG="debug cargo test --features try-runtime -- run_migrations" + if [[ $? -eq $sudoCheckStatus ]]; then + echo "✅ sudo check is successful!" + else + echo "❌ sudo check is failed!" + exit 1 fi - printf "⚡️ only tests run %s\n" - rm -rf ~/.cargo/.package-cache - cargo fmt -- --check > /dev/null - cargo test - cargo test --features "private-net wip ready-to-test runtime-benchmarks" -fi +} + +if [ "$(type -t $1)" = "function" ]; then + "$1" +else + echo "Func '$1' is not exists in this workflow. Skipped." +fi \ No newline at end of file diff --git a/housekeeping/clippy.sh b/housekeeping/clippy.sh index cc72a54bf8..2edb32054b 100755 --- a/housekeeping/clippy.sh +++ b/housekeeping/clippy.sh @@ -4,8 +4,8 @@ set -e if [ "$pr" = true ] && [ "$prBranch" != "master" ]; then printf "👷‍♂️ starting clippy \n" SKIP_WASM_BUILD=1 cargo clippy --features private-net,ready-to-test,runtime-benchmarks -- -D warnings - SKIP_WASM_BUILD=1 cargo clippy --features private-net,ready-to-test,wip,runtime-benchmarks -- -D warnings + SKIP_WASM_BUILD=1 cargo clippy --features private-net,ready-to-test,wip,runtime-benchmarks --message-format=json -- -D warnings > clippy_report.json else printf "👷‍♂️ starting a regular clippy \n" - cargo clippy || exit 0 + cargo clippy --message-format=json > clippy_report.json || exit 0 fi diff --git a/housekeeping/coverage.sh b/housekeeping/coverage.sh index 8e2ebc8630..6a6d5f869a 100755 --- a/housekeeping/coverage.sh +++ b/housekeeping/coverage.sh @@ -1,10 +1,9 @@ #!/bin/sh +set -e -export RUSTFLAGS="-Cinstrument-coverage" -export SKIP_WASM_BUILD=1 -export LLVM_PROFILE_FILE="sora2-%p-%m.profraw" +printf '⚡️ Running coverage %s\n' +# coverage and generate report +grcov . --binary-path ./target/debug -s . -t lcov --branch -o ./lcov_report --ignore-not-existing --ignore "/opt/cargo/**" "target/debug" "node/src" --log-level="ERROR" --llvm-path='/usr/lib/llvm-14/bin' -cargo test --features private-net - -grcov . --binary-path ./target/debug -s . -t cobertura --branch -o ./cobertura_report --ignore-not-existing --ignore "/opt/cargo/**" "target/debug" "node/src" -find . -type f -name '*.profraw' -delete +# delete generated *.profraw +find . -type f -name '*.profraw' -delete \ No newline at end of file diff --git a/housekeeping/docker/env/Dockerfile b/housekeeping/docker/env/Dockerfile index d8e32885d2..e323cd00e5 100644 --- a/housekeeping/docker/env/Dockerfile +++ b/housekeeping/docker/env/Dockerfile @@ -1,59 +1,44 @@ -FROM debian:bullseye-slim +FROM debian:bullseye-20240211-slim -# Set environment variables -ENV RUSTUP_HOME="/opt/rustup" -ENV CARGO_HOME="/opt/cargo" -ENV PATH="$PATH:$CARGO_HOME/bin" +ENV RUSTUP_HOME="/opt/rust" +ENV CARGO_HOME="/opt/rust" +ENV PATH="$PATH:$RUSTUP_HOME/bin" ENV CARGO_BUILD_DEP_INFO_BASEDIR="." -ENV CC=clang-13 -ENV CXX=clang++-13 +ENV RUST_VERSION=nightly-2024-01-15 +ENV TZ=Europe/Moscow +ENV CC=clang-14 +ENV CXX=clang++-14 +ENV SUBWASM_VER=v0.20.0 +ENV GRCOV_VERSION=0.8.19 +ENV LLVM_TOOLS_VER=14 +ENV CCLANG_VER=14 +ARG DEBIAN_FRONTEND=noninteractive -# Install dependencies RUN apt-get update && \ apt-get install --no-install-recommends -y \ ca-certificates apt-transport-https gnupg \ libssl-dev pkg-config \ - curl \ + wget \ git binaryen \ - make cmake libssl-dev \ + make cmake \ software-properties-common && \ - apt-get autoremove -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN curl -s https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-13 main" >> /etc/apt/sources.list.d/llvm-toochain-bullseye-13.list && \ - echo "deb-src http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-13 main" >> /etc/apt/sources.list.d/llvm-toochain-bullseye-13.list && \ - apt-get -y update && \ - apt-get install -y --no-install-recommends \ - clang-13 lldb-13 lld-13 libclang-13-dev llvm-13 protobuf-compiler && \ - update-alternatives --install /usr/bin/cc cc /usr/bin/clang-13 100 && \ - apt-get autoremove -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Install docker -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -RUN add-apt-repository \ - "deb https://download.docker.com/linux/debian $(lsb_release -cs) stable" && \ + wget -O - https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain ${RUST_VERSION} && \ + rustup toolchain install ${RUST_VERSION} && \ + rustup target add wasm32-unknown-unknown --toolchain ${RUST_VERSION} && \ + wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh ${LLVM_TOOLS_VER} && \ + rm llvm.sh && \ apt-get update && \ apt-get install --no-install-recommends -y \ - docker-ce docker-ce-cli containerd.io && \ + clang-${CCLANG_VER} lldb-${CCLANG_VER} lld-${CCLANG_VER} libclang-${CCLANG_VER}-dev llvm-${LLVM_TOOLS_VER} protobuf-compiler && \ + update-alternatives --install /usr/bin/cc cc /usr/bin/${CC} 100 && \ + rustup update && \ + cargo install --locked --git https://github.com/chevdor/subwasm --tag ${SUBWASM_VER} && \ + cargo install grcov --version ${GRCOV_VERSION} && \ + rustup component add llvm-tools-preview && \ apt-get autoremove -y && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Install rust -ENV RUST_NIGHTLY_VERSION=nightly-2023-03-21 -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path --default-toolchain stable && \ - rustup toolchain install ${RUST_NIGHTLY_VERSION} && \ - rustup target add wasm32-unknown-unknown && \ - rustup target add wasm32-unknown-unknown --toolchain ${RUST_NIGHTLY_VERSION} && \ - rm -rf "${CARGO_HOME}/registry" "${CARGO_HOME}/git" - -RUN curl https://github.com/chevdor/subwasm/releases/download/v0.17.0/subwasm_linux_amd64_v0.17.0.deb -sSL -o subw.deb && \ - dpkg -i subw.deb && rm subw.deb - -RUN cargo install grcov && \ - rustup component add llvm-tools-preview && \ - rm -rf "${CARGO_HOME}/registry" "${CARGO_HOME}/git" + rm -rf /var/lib/apt/lists/* && \ + rm -rf "${CARGO_HOME}/git" && \ + ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone \ No newline at end of file diff --git a/housekeeping/docker/release/Dockerfile b/housekeeping/docker/release/Dockerfile index 44fda7d116..ee9f52354b 100644 --- a/housekeeping/docker/release/Dockerfile +++ b/housekeeping/docker/release/Dockerfile @@ -1,5 +1,5 @@ FROM debian:bullseye -# Install dependencies + RUN apt-get update && \ apt-get install --no-install-recommends -y \ ca-certificates nano curl && \ @@ -13,6 +13,5 @@ RUN mkdir /chain && \ USER substrate COPY ./framenode /usr/local/bin/framenode -COPY ./relayer.bin /usr/local/bin/relayer ENTRYPOINT ["framenode"] \ No newline at end of file