From 46af33da8b60ad955797a45d3e0f583b895363e3 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 7 Sep 2021 12:12:14 -0300 Subject: [PATCH 01/17] update check_polkadot_companion_build for taking care of deleted or renamed crates and simplify it a bit --- .../gitlab/check_polkadot_companion_build.sh | 126 +++++++++++++++--- 1 file changed, 109 insertions(+), 17 deletions(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index 72bfaf7151522..292defbd2e1dc 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash # # check if a pr is compatible with polkadot companion pr or master if not # available @@ -43,15 +43,14 @@ git config --global user.email '<>' # Merge master into our branch before building Polkadot to make sure we don't miss # any commits that are required by Polkadot. -git fetch --depth 100 origin -git merge origin/master +git pull origin master -# Clone the current Polkadot master branch into ./polkadot. -# NOTE: we need to pull enough commits to be able to find a common -# ancestor for successfully performing merges below. -git clone --depth 20 https://github.com/paritytech/polkadot.git +substrate_dir="$PWD" +polkadot_dir="$substrate_dir/polkadot" -cd polkadot +git clone --depth 1 https://github.com/paritytech/polkadot.git "$polkadot_dir" + +cd "$polkadot_dir" # either it's a pull request then check for a companion otherwise use # polkadot:master @@ -76,7 +75,8 @@ then boldprint "companion pr specified/detected: #${pr_companion}" git fetch origin refs/pull/${pr_companion}/head:pr/${pr_companion} git checkout pr/${pr_companion} - git merge origin/master + # we want this because `bot merge` will include master into the pr's branch before merging + git pull origin master else boldprint "no companion branch found - building polkadot:master" fi @@ -85,15 +85,107 @@ else boldprint "this is not a pull request - building polkadot:master" fi +cd "$substrate_dir" + +our_crates=() +while IFS= read -r crate; do + # for avoiding duplicate entries + for our_crate in "${our_crates[@]}"; do + if [[ "$crate" == "$our_crate" ]]; then + found=true + break + fi + done + if [ "${found:-}" ]; then + unset found + else + our_crates+=("$crate") + fi +done < <(jq -r ' + . as $in | + paths | + select(.[-1]=="source" and . as $p | $in | getpath($p)==null) as $path | + del($path[-1]) as $path | + $in | getpath($path + ["name"]) +' < <(cargo metadata --quiet --format-version=1)) +our_crates_source="git+https://github.com/paritytech/substrate" + +match_their_crates() { + local target_dir="$1" + shift + + cd "$target_dir" + + local target_name="$(basename "$target_dir")" + local crates_not_found=() + + local found + + # output will be provided in the format: + # crate + # source + # crate + # ... + local next="crate" + while IFS= read -r line; do + case "$next" in + crate) + crate="$line" + next="source" + ;; + source) + if [[ "$line" == "$our_crates_source" ]] || [[ "$line" == "$our_crates_source?" ]]; then + for our_crate in "${our_crates[@]}"; do + if [ "$our_crate" == "$crate" ]; then + found=true + break + fi + done + if [ "${found:-}" ]; then + unset found + else + # for avoiding duplicate entries + for crate_not_found in "${crates_not_found[@]}"; do + if [[ "$crate_not_found" == "$crate" ]]; then + found=true + break + fi + done + if [ "${found:-}" ]; then + unset found + else + crates_not_found+=("$crate") + fi + fi + fi + + next="crate" + ;; + *) + echo "ERROR: Unknown state $next" + exit 1 + ;; + esac + done < <(jq -r ' + . as $in | + paths(select(type=="string")) | + select(.[-1]=="source") as $source_path | + del($source_path[-1]) as $path | + [$in | getpath($path + ["name"]), getpath($path + ["source"])] | + .[] + ' < <(cargo metadata --quiet --format-version=1)) + + if [ "$crates_not_found" ]; then + echo "Errors during crate matching" + printf "Failed to find crate \"%s\" referenced in $target_name\n" "${crates_not_found[@]}" + exit 1 + fi +} +match_their_crates "$polkadot_dir" + # Patch all Substrate crates in Polkadot -diener patch --crates-to-patch ../ --substrate --path Cargo.toml - -# We need to update specifically our patched Substrate crates so that other -# crates that depend on them (e.g. Polkadot, BEEFY) use this unified version -# NOTE: There's no way to only update patched crates, so we use a heuristic -# of updating a crucial Substrate crate (`sp-core`) to minimize the impact of -# updating unrelated dependencies -cargo update -p sp-core +cd "$polkadot_dir" +diener patch --crates-to-patch "$substrate_dir" --substrate --path Cargo.toml # Test Polkadot pr or master branch with this Substrate commit. time cargo test --workspace --release --verbose --features=runtime-benchmarks From 9275261e34a47fb442ffabbb38cf84f261f5fb56 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 7 Sep 2021 12:29:24 -0300 Subject: [PATCH 02/17] scope load_our_crates into a function to ensure variables don't leak --- .../gitlab/check_polkadot_companion_build.sh | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index 292defbd2e1dc..828014cbc57e0 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -88,27 +88,32 @@ fi cd "$substrate_dir" our_crates=() -while IFS= read -r crate; do - # for avoiding duplicate entries - for our_crate in "${our_crates[@]}"; do - if [[ "$crate" == "$our_crate" ]]; then - found=true - break - fi - done - if [ "${found:-}" ]; then - unset found - else - our_crates+=("$crate") - fi -done < <(jq -r ' - . as $in | - paths | - select(.[-1]=="source" and . as $p | $in | getpath($p)==null) as $path | - del($path[-1]) as $path | - $in | getpath($path + ["name"]) -' < <(cargo metadata --quiet --format-version=1)) our_crates_source="git+https://github.com/paritytech/substrate" +load_our_crates() { + local found + + while IFS= read -r crate; do + # for avoiding duplicate entries + for our_crate in "${our_crates[@]}"; do + if [[ "$crate" == "$our_crate" ]]; then + found=true + break + fi + done + if [ "${found:-}" ]; then + unset found + else + our_crates+=("$crate") + fi + done < <(jq -r ' + . as $in | + paths | + select(.[-1]=="source" and . as $p | $in | getpath($p)==null) as $path | + del($path[-1]) as $path | + $in | getpath($path + ["name"]) + ' < <(cargo metadata --quiet --format-version=1)) +} +load_our_crates match_their_crates() { local target_dir="$1" From 77f352429fc64a5a1828dd9e643ad7ac894cf6f2 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 7 Sep 2021 12:31:01 -0300 Subject: [PATCH 03/17] simplify comparison --- .maintain/gitlab/check_polkadot_companion_build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index 828014cbc57e0..25d749ae58c3d 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -95,7 +95,7 @@ load_our_crates() { while IFS= read -r crate; do # for avoiding duplicate entries for our_crate in "${our_crates[@]}"; do - if [[ "$crate" == "$our_crate" ]]; then + if [ "$crate" == "$our_crate" ]; then found=true break fi @@ -139,7 +139,7 @@ match_their_crates() { next="source" ;; source) - if [[ "$line" == "$our_crates_source" ]] || [[ "$line" == "$our_crates_source?" ]]; then + if [ "$line" == "$our_crates_source" ] || [ "$line" == "$our_crates_source?" ]; then for our_crate in "${our_crates[@]}"; do if [ "$our_crate" == "$crate" ]; then found=true @@ -151,7 +151,7 @@ match_their_crates() { else # for avoiding duplicate entries for crate_not_found in "${crates_not_found[@]}"; do - if [[ "$crate_not_found" == "$crate" ]]; then + if [ "$crate_not_found" == "$crate" ]; then found=true break fi From 0c258de4e5ab271c16414d1ad8f188f9e9be84c6 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 7 Sep 2021 14:23:41 -0300 Subject: [PATCH 04/17] sc_utils => sc_foo (for testing's sake) --- Cargo.lock | 40 +++++++++---------- client/api/Cargo.toml | 2 +- client/api/src/client.rs | 2 +- client/api/src/notifications.rs | 2 +- client/cli/Cargo.toml | 2 +- client/cli/src/runner.rs | 2 +- client/consensus/common/Cargo.toml | 2 +- .../common/src/import_queue/basic_queue.rs | 2 +- .../common/src/import_queue/buffered_link.rs | 2 +- client/finality-grandpa/Cargo.toml | 2 +- .../src/communication/gossip.rs | 2 +- .../finality-grandpa/src/communication/mod.rs | 2 +- .../src/communication/periodic.rs | 2 +- .../src/communication/tests.rs | 2 +- client/finality-grandpa/src/import.rs | 2 +- client/finality-grandpa/src/lib.rs | 2 +- client/finality-grandpa/src/notification.rs | 2 +- client/finality-grandpa/src/observer.rs | 4 +- client/finality-grandpa/src/until_imported.rs | 4 +- client/network/Cargo.toml | 2 +- client/network/src/on_demand_layer.rs | 2 +- client/network/src/service.rs | 2 +- client/offchain/Cargo.toml | 2 +- client/offchain/src/api/http.rs | 2 +- client/peerset/Cargo.toml | 2 +- client/peerset/src/lib.rs | 2 +- client/rpc/Cargo.toml | 2 +- client/rpc/src/system/mod.rs | 2 +- client/rpc/src/system/tests.rs | 2 +- client/service/Cargo.toml | 2 +- client/service/src/builder.rs | 2 +- client/service/src/client/client.rs | 2 +- client/service/src/lib.rs | 2 +- client/service/src/metrics.rs | 2 +- client/service/src/task_manager/mod.rs | 2 +- client/transaction-pool/Cargo.toml | 2 +- client/transaction-pool/graph/Cargo.toml | 2 +- client/transaction-pool/src/graph/watcher.rs | 2 +- client/transaction-pool/src/revalidation.rs | 2 +- client/utils/Cargo.toml | 2 +- 40 files changed, 61 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5205f0dd4110..9c2ac8903762e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7307,12 +7307,12 @@ dependencies = [ "regex", "rpassword", "sc-client-api", + "sc-foo", "sc-keystore", "sc-network", "sc-service", "sc-telemetry", "sc-tracing", - "sc-utils", "serde", "serde_json", "sp-blockchain", @@ -7340,8 +7340,8 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.11.1", "sc-executor", + "sc-foo", "sc-transaction-pool-api", - "sc-utils", "sp-api", "sp-blockchain", "sp-consensus", @@ -7398,7 +7398,7 @@ dependencies = [ "log 0.4.14", "parking_lot 0.11.1", "sc-client-api", - "sc-utils", + "sc-foo", "serde", "sp-api", "sp-blockchain", @@ -7747,12 +7747,12 @@ dependencies = [ "sc-block-builder", "sc-client-api", "sc-consensus", + "sc-foo", "sc-keystore", "sc-network", "sc-network-gossip", "sc-network-test", "sc-telemetry", - "sc-utils", "serde_json", "sp-api", "sp-application-crypto", @@ -7798,6 +7798,16 @@ dependencies = [ "substrate-test-runtime-client", ] +[[package]] +name = "sc-foo" +version = "4.0.0-dev" +dependencies = [ + "futures 0.3.16", + "futures-timer 3.0.2", + "lazy_static", + "prometheus", +] + [[package]] name = "sc-informant" version = "0.10.0-dev" @@ -7880,8 +7890,8 @@ dependencies = [ "sc-block-builder", "sc-client-api", "sc-consensus", + "sc-foo", "sc-peerset", - "sc-utils", "serde", "serde_json", "smallvec 1.6.1", @@ -7968,10 +7978,10 @@ dependencies = [ "sc-block-builder", "sc-client-api", "sc-client-db", + "sc-foo", "sc-network", "sc-transaction-pool", "sc-transaction-pool-api", - "sc-utils", "sp-api", "sp-consensus", "sp-core", @@ -7991,7 +8001,7 @@ dependencies = [ "libp2p", "log 0.4.14", "rand 0.7.3", - "sc-utils", + "sc-foo", "serde_json", "wasm-timer", ] @@ -8020,12 +8030,12 @@ dependencies = [ "sc-block-builder", "sc-chain-spec", "sc-client-api", + "sc-foo", "sc-network", "sc-rpc-api", "sc-tracing", "sc-transaction-pool", "sc-transaction-pool-api", - "sc-utils", "serde_json", "sp-api", "sp-blockchain", @@ -8118,6 +8128,7 @@ dependencies = [ "sc-client-db", "sc-consensus", "sc-executor", + "sc-foo", "sc-informant", "sc-keystore", "sc-light", @@ -8129,7 +8140,6 @@ dependencies = [ "sc-tracing", "sc-transaction-pool", "sc-transaction-pool-api", - "sc-utils", "serde", "serde_json", "sp-api", @@ -8302,8 +8312,8 @@ dependencies = [ "retain_mut", "sc-block-builder", "sc-client-api", + "sc-foo", "sc-transaction-pool-api", - "sc-utils", "serde", "sp-api", "sp-blockchain", @@ -8332,16 +8342,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "sc-utils" -version = "4.0.0-dev" -dependencies = [ - "futures 0.3.16", - "futures-timer 3.0.2", - "lazy_static", - "prometheus", -] - [[package]] name = "schannel" version = "0.1.19" diff --git a/client/api/Cargo.toml b/client/api/Cargo.toml index 772f22e822eb2..d820b62a66acc 100644 --- a/client/api/Cargo.toml +++ b/client/api/Cargo.toml @@ -30,7 +30,7 @@ sp-database = { version = "4.0.0-dev", path = "../../primitives/database" } sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } sp-keystore = { version = "0.10.0-dev", default-features = false, path = "../../primitives/keystore" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } -sc-utils = { version = "4.0.0-dev", path = "../utils" } +sc-foo = { version = "4.0.0-dev", path = "../utils" } sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } sp-state-machine = { version = "0.10.0-dev", path = "../../primitives/state-machine" } sp-trie = { version = "4.0.0-dev", path = "../../primitives/trie" } diff --git a/client/api/src/client.rs b/client/api/src/client.rs index 21f8aecad0536..9b64800e3a1a2 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -29,7 +29,7 @@ use std::{collections::HashSet, convert::TryFrom, fmt, sync::Arc}; use crate::{blockchain::Info, notifications::StorageEventStream}; use sc_transaction_pool_api::ChainEvent; -use sc_utils::mpsc::TracingUnboundedReceiver; +use sc_foo::mpsc::TracingUnboundedReceiver; use sp_blockchain; /// Type that implements `futures::Stream` of block import events. diff --git a/client/api/src/notifications.rs b/client/api/src/notifications.rs index 1346afd5e54d2..3a366c1e1b7dc 100644 --- a/client/api/src/notifications.rs +++ b/client/api/src/notifications.rs @@ -25,7 +25,7 @@ use std::{ use fnv::{FnvHashMap, FnvHashSet}; use prometheus_endpoint::{register, CounterVec, Opts, Registry, U64}; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_core::storage::{StorageData, StorageKey}; use sp_runtime::traits::Block as BlockT; diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index 7798507e529f4..b00a89d316732 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -30,7 +30,7 @@ sc-client-api = { version = "4.0.0-dev", path = "../api" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sc-network = { version = "0.10.0-dev", path = "../network" } sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sc-utils = { version = "4.0.0-dev", path = "../utils" } +sc-foo = { version = "4.0.0-dev", path = "../utils" } sp-version = { version = "4.0.0-dev", path = "../../primitives/version" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore" } diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index 2ec200d9285b1..972bd98530a47 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -21,7 +21,7 @@ use chrono::prelude::*; use futures::{future, future::FutureExt, pin_mut, select, Future}; use log::info; use sc_service::{Configuration, Error as ServiceError, TaskManager, TaskType}; -use sc_utils::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL}; +use sc_foo::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL}; use std::marker::PhantomData; #[cfg(target_family = "unix")] diff --git a/client/consensus/common/Cargo.toml b/client/consensus/common/Cargo.toml index 6829bd2c6d8b5..ad81f168d0cac 100644 --- a/client/consensus/common/Cargo.toml +++ b/client/consensus/common/Cargo.toml @@ -24,7 +24,7 @@ sp-core = { path = "../../../primitives/core", version = "4.0.0-dev" } sp-consensus = { path = "../../../primitives/consensus/common", version = "0.10.0-dev" } sp-state-machine = { version = "0.10.0-dev", path = "../../../primitives/state-machine" } sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -sc-utils = { version = "4.0.0-dev", path = "../../utils" } +sc-foo = { version = "4.0.0-dev", path = "../../utils" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } parking_lot = "0.11.1" serde = { version = "1.0", features = ["derive"] } diff --git a/client/consensus/common/src/import_queue/basic_queue.rs b/client/consensus/common/src/import_queue/basic_queue.rs index 9042c8798be4f..7e76a7fdb821e 100644 --- a/client/consensus/common/src/import_queue/basic_queue.rs +++ b/client/consensus/common/src/import_queue/basic_queue.rs @@ -22,7 +22,7 @@ use futures::{ use futures_timer::Delay; use log::{debug, trace}; use prometheus_endpoint::Registry; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_consensus::BlockOrigin; use sp_runtime::{ traits::{Block as BlockT, Header as HeaderT, NumberFor}, diff --git a/client/consensus/common/src/import_queue/buffered_link.rs b/client/consensus/common/src/import_queue/buffered_link.rs index 87ea6dde5c473..3960718d6d771 100644 --- a/client/consensus/common/src/import_queue/buffered_link.rs +++ b/client/consensus/common/src/import_queue/buffered_link.rs @@ -40,7 +40,7 @@ use crate::import_queue::{Link, Origin}; use futures::prelude::*; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_runtime::traits::{Block as BlockT, NumberFor}; use std::{ pin::Pin, diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml index 40385a2faea57..e0ed612c9ffee 100644 --- a/client/finality-grandpa/Cargo.toml +++ b/client/finality-grandpa/Cargo.toml @@ -27,7 +27,7 @@ parity-scale-codec = { version = "2.0.0", features = ["derive"] } sp-application-crypto = { version = "4.0.0-dev", path = "../../primitives/application-crypto" } sp-arithmetic = { version = "4.0.0-dev", path = "../../primitives/arithmetic" } sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sc-utils = { version = "4.0.0-dev", path = "../utils" } +sc-foo = { version = "4.0.0-dev", path = "../utils" } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } diff --git a/client/finality-grandpa/src/communication/gossip.rs b/client/finality-grandpa/src/communication/gossip.rs index d64c7421afa69..964f3fb36c672 100644 --- a/client/finality-grandpa/src/communication/gossip.rs +++ b/client/finality-grandpa/src/communication/gossip.rs @@ -94,7 +94,7 @@ use log::{debug, trace}; use prometheus_endpoint::{register, CounterVec, Opts, PrometheusError, Registry, U64}; use rand::seq::SliceRandom; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG}; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use super::{benefit, cost, Round, SetId}; use crate::{environment, CatchUp, CompactCommit, SignedMessage}; diff --git a/client/finality-grandpa/src/communication/mod.rs b/client/finality-grandpa/src/communication/mod.rs index c370e1d642d7d..280340236cedc 100644 --- a/client/finality-grandpa/src/communication/mod.rs +++ b/client/finality-grandpa/src/communication/mod.rs @@ -58,7 +58,7 @@ use crate::{ use gossip::{ FullCatchUpMessage, FullCommitMessage, GossipMessage, GossipValidator, PeerReport, VoteMessage, }; -use sc_utils::mpsc::TracingUnboundedReceiver; +use sc_foo::mpsc::TracingUnboundedReceiver; use sp_finality_grandpa::{AuthorityId, AuthoritySignature, RoundNumber, SetId as SetIdNumber}; pub mod gossip; diff --git a/client/finality-grandpa/src/communication/periodic.rs b/client/finality-grandpa/src/communication/periodic.rs index 77e55ad652f6c..a09cfab6b3947 100644 --- a/client/finality-grandpa/src/communication/periodic.rs +++ b/client/finality-grandpa/src/communication/periodic.rs @@ -21,7 +21,7 @@ use futures::{future::FutureExt as _, prelude::*, ready, stream::Stream}; use futures_timer::Delay; use log::debug; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use std::{ pin::Pin, task::{Context, Poll}, diff --git a/client/finality-grandpa/src/communication/tests.rs b/client/finality-grandpa/src/communication/tests.rs index 1fac0230b2a84..509ffd4edd8ac 100644 --- a/client/finality-grandpa/src/communication/tests.rs +++ b/client/finality-grandpa/src/communication/tests.rs @@ -28,7 +28,7 @@ use parity_scale_codec::Encode; use sc_network::{config::Role, Event as NetworkEvent, ObservedRole, PeerId}; use sc_network_gossip::Validator; use sc_network_test::{Block, Hash}; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_finality_grandpa::AuthorityList; use sp_keyring::Ed25519Keyring; use sp_runtime::traits::NumberFor; diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index f663bfe94afdf..2ee0b4ec2de6e 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -27,7 +27,7 @@ use sc_consensus::{ BlockCheckParams, BlockImport, BlockImportParams, ImportResult, JustificationImport, }; use sc_telemetry::TelemetryHandle; -use sc_utils::mpsc::TracingUnboundedSender; +use sc_foo::mpsc::TracingUnboundedSender; use sp_api::{Core, RuntimeApiInfo, TransactionFor}; use sp_blockchain::{well_known_cache_keys, BlockStatus}; use sp_consensus::{BlockOrigin, Error as ConsensusError, SelectChain}; diff --git a/client/finality-grandpa/src/lib.rs b/client/finality-grandpa/src/lib.rs index 452659ced6a70..6ce476ef5c96c 100644 --- a/client/finality-grandpa/src/lib.rs +++ b/client/finality-grandpa/src/lib.rs @@ -68,7 +68,7 @@ use sc_client_api::{ }; use sc_consensus::BlockImport; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO}; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; use sp_api::ProvideRuntimeApi; use sp_application_crypto::AppKey; use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata}; diff --git a/client/finality-grandpa/src/notification.rs b/client/finality-grandpa/src/notification.rs index 85d581bd5065e..b1b28d4f11ca9 100644 --- a/client/finality-grandpa/src/notification.rs +++ b/client/finality-grandpa/src/notification.rs @@ -19,7 +19,7 @@ use parking_lot::Mutex; use std::sync::Arc; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_runtime::traits::Block as BlockT; use crate::{justification::GrandpaJustification, Error}; diff --git a/client/finality-grandpa/src/observer.rs b/client/finality-grandpa/src/observer.rs index 70a94cd504726..7319a44aded13 100644 --- a/client/finality-grandpa/src/observer.rs +++ b/client/finality-grandpa/src/observer.rs @@ -29,7 +29,7 @@ use log::{debug, info, warn}; use sc_client_api::backend::Backend; use sc_telemetry::TelemetryHandle; -use sc_utils::mpsc::TracingUnboundedReceiver; +use sc_foo::mpsc::TracingUnboundedReceiver; use sp_blockchain::HeaderMetadata; use sp_consensus::SelectChain; use sp_finality_grandpa::AuthorityId; @@ -403,7 +403,7 @@ mod tests { }; use assert_matches::assert_matches; use sc_network::PeerId; - use sc_utils::mpsc::tracing_unbounded; + use sc_foo::mpsc::tracing_unbounded; use sp_blockchain::HeaderBackend as _; use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt}; diff --git a/client/finality-grandpa/src/until_imported.rs b/client/finality-grandpa/src/until_imported.rs index deb6577264347..45c7a56fbc950 100644 --- a/client/finality-grandpa/src/until_imported.rs +++ b/client/finality-grandpa/src/until_imported.rs @@ -37,7 +37,7 @@ use log::{debug, warn}; use parking_lot::Mutex; use prometheus_endpoint::{register, Gauge, PrometheusError, Registry, U64}; use sc_client_api::{BlockImportNotification, ImportNotifications}; -use sc_utils::mpsc::TracingUnboundedReceiver; +use sc_foo::mpsc::TracingUnboundedReceiver; use sp_finality_grandpa::AuthorityId; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; @@ -561,7 +561,7 @@ mod tests { use futures::future::Either; use futures_timer::Delay; use sc_client_api::BlockImportNotification; - use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; + use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_consensus::BlockOrigin; use substrate_test_runtime_client::runtime::{Block, Hash, Header}; diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index de62a534f8662..7b9fab97ae090 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -55,7 +55,7 @@ sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/comm sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sc-utils = { version = "4.0.0-dev", path = "../utils" } +sc-foo = { version = "4.0.0-dev", path = "../utils" } sp-finality-grandpa = { version = "4.0.0-dev", path = "../../primitives/finality-grandpa" } thiserror = "1" unsigned-varint = { version = "0.6.0", features = [ diff --git a/client/network/src/on_demand_layer.rs b/client/network/src/on_demand_layer.rs index 5bac05c7aefad..a2917a6e2cc76 100644 --- a/client/network/src/on_demand_layer.rs +++ b/client/network/src/on_demand_layer.rs @@ -27,7 +27,7 @@ use sc_client_api::{ RemoteChangesRequest, RemoteHeaderRequest, RemoteReadChildRequest, RemoteReadRequest, StorageProof, }; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_blockchain::Error as ClientError; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; use std::{ diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 6b2928510760c..adb62d2833f47 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -70,7 +70,7 @@ use metrics::{Histogram, HistogramVec, MetricSources, Metrics}; use parking_lot::Mutex; use sc_consensus::{BlockImportError, BlockImportStatus, ImportQueue, Link}; use sc_peerset::PeersetHandle; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_runtime::traits::{Block as BlockT, NumberFor}; use std::{ borrow::Cow, diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index a7ff572e9b0b7..bf6556e4eb3f2 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -29,7 +29,7 @@ sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } sp-offchain = { version = "4.0.0-dev", path = "../../primitives/offchain" } sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sc-utils = { version = "4.0.0-dev", path = "../utils" } +sc-foo = { version = "4.0.0-dev", path = "../utils" } threadpool = "1.7" [target.'cfg(not(target_os = "unknown"))'.dependencies] diff --git a/client/offchain/src/api/http.rs b/client/offchain/src/api/http.rs index ce9fb298d1b0c..367daf3e7f3e7 100644 --- a/client/offchain/src/api/http.rs +++ b/client/offchain/src/api/http.rs @@ -34,7 +34,7 @@ use futures::{channel::mpsc, future, prelude::*}; use hyper::{client, Body, Client as HyperClient}; use hyper_rustls::HttpsConnector; use log::error; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_core::offchain::{HttpError, HttpRequestId, HttpRequestStatus, Timestamp}; use std::{ convert::TryFrom, diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index 9e83ede675d02..4b954c227b97f 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -17,7 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] futures = "0.3.9" libp2p = { version = "0.39.1", default-features = false } -sc-utils = { version = "4.0.0-dev", path = "../utils"} +sc-foo = { version = "4.0.0-dev", path = "../utils"} log = "0.4.8" serde_json = "1.0.41" wasm-timer = "0.2" diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index ecaa1d9f576f1..747824d17e900 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -36,7 +36,7 @@ mod peersstate; use futures::prelude::*; use log::{debug, error, trace}; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use serde_json::json; use std::{ collections::{HashMap, HashSet, VecDeque}, diff --git a/client/rpc/Cargo.toml b/client/rpc/Cargo.toml index 9957bc999a8bb..173588dc80546 100644 --- a/client/rpc/Cargo.toml +++ b/client/rpc/Cargo.toml @@ -27,7 +27,7 @@ serde_json = "1.0.41" sp-session = { version = "4.0.0-dev", path = "../../primitives/session" } sp-offchain = { version = "4.0.0-dev", path = "../../primitives/offchain" } sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sc-utils = { version = "4.0.0-dev", path = "../utils" } +sc-foo = { version = "4.0.0-dev", path = "../utils" } sp-rpc = { version = "4.0.0-dev", path = "../../primitives/rpc" } sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore" } sc-chain-spec = { version = "4.0.0-dev", path = "../chain-spec" } diff --git a/client/rpc/src/system/mod.rs b/client/rpc/src/system/mod.rs index f99994e41a1be..52ed2bd9cd60a 100644 --- a/client/rpc/src/system/mod.rs +++ b/client/rpc/src/system/mod.rs @@ -22,7 +22,7 @@ use self::error::Result; use futures::{channel::oneshot, FutureExt}; use sc_rpc_api::{DenyUnsafe, Receiver}; use sc_tracing::logging; -use sc_utils::mpsc::TracingUnboundedSender; +use sc_foo::mpsc::TracingUnboundedSender; use sp_runtime::traits::{self, Header as HeaderT}; pub use self::{ diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs index cc794b884f066..dbb0ffbe408ac 100644 --- a/client/rpc/src/system/tests.rs +++ b/client/rpc/src/system/tests.rs @@ -21,7 +21,7 @@ use super::*; use assert_matches::assert_matches; use futures::{executor, prelude::*}; use sc_network::{self, config::Role, PeerId}; -use sc_utils::mpsc::tracing_unbounded; +use sc_foo::mpsc::tracing_unbounded; use std::{ env, io::{BufRead, BufReader, Write}, diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index b79e95fbb0912..a645eae1a2842 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -39,7 +39,7 @@ sc-keystore = { version = "4.0.0-dev", path = "../keystore" } sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } sp-trie = { version = "4.0.0-dev", path = "../../primitives/trie" } sp-externalities = { version = "0.10.0-dev", path = "../../primitives/externalities" } -sc-utils = { version = "4.0.0-dev", path = "../utils" } +sc-foo = { version = "4.0.0-dev", path = "../utils" } sp-version = { version = "4.0.0-dev", path = "../../primitives/version" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index f0c037aee232f..8a1f9adcd4144 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -49,7 +49,7 @@ use sc_network::{ }; use sc_telemetry::{telemetry, ConnectionMessage, Telemetry, TelemetryHandle, SUBSTRATE_INFO}; use sc_transaction_pool_api::MaintainedTransactionPool; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_api::{CallApiAt, ProvideRuntimeApi}; use sp_blockchain::{HeaderBackend, HeaderMetadata}; use sp_consensus::block_validation::{ diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index f7d93d036a3fa..cacfd23f23212 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -61,7 +61,7 @@ use sp_blockchain::{ }; use sp_consensus::{BlockOrigin, BlockStatus, Error as ConsensusError}; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_core::{ convert_hash, storage::{well_known_keys, ChildInfo, PrefixedStorageKey, StorageData, StorageKey}, diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index ede6f01a4539e..f19c574a65e77 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -41,7 +41,7 @@ use futures::{stream, Future, FutureExt, Stream, StreamExt}; use log::{debug, error, warn}; use parity_util_mem::MallocSizeOf; use sc_network::PeerId; -use sc_utils::mpsc::TracingUnboundedReceiver; +use sc_foo::mpsc::TracingUnboundedReceiver; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Header as HeaderT}, diff --git a/client/service/src/metrics.rs b/client/service/src/metrics.rs index 4d3c6df92fee7..99cd1fe34d6aa 100644 --- a/client/service/src/metrics.rs +++ b/client/service/src/metrics.rs @@ -25,7 +25,7 @@ use sc_client_api::{ClientInfo, UsageProvider}; use sc_network::{config::Role, NetworkService, NetworkStatus}; use sc_telemetry::{telemetry, TelemetryHandle, SUBSTRATE_INFO}; use sc_transaction_pool_api::{MaintainedTransactionPool, PoolStatus}; -use sc_utils::metrics::register_globals; +use sc_foo::metrics::register_globals; use sp_api::ProvideRuntimeApi; use sp_runtime::traits::{Block, NumberFor, SaturatedConversion, UniqueSaturatedInto}; use std::{ diff --git a/client/service/src/task_manager/mod.rs b/client/service/src/task_manager/mod.rs index 7842acdf0455a..d8613a0a90ea6 100644 --- a/client/service/src/task_manager/mod.rs +++ b/client/service/src/task_manager/mod.rs @@ -33,7 +33,7 @@ use prometheus_endpoint::{ exponential_buckets, register, CounterVec, HistogramOpts, HistogramVec, Opts, PrometheusError, Registry, U64, }; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use std::{panic, pin::Pin, result::Result}; use tracing_futures::Instrument; diff --git a/client/transaction-pool/Cargo.toml b/client/transaction-pool/Cargo.toml index 2184af819adf7..e0224b436dc3a 100644 --- a/client/transaction-pool/Cargo.toml +++ b/client/transaction-pool/Cargo.toml @@ -29,7 +29,7 @@ sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } sp-transaction-pool = { version = "4.0.0-dev", path = "../../primitives/transaction-pool" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "./api" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } -sc-utils = { version = "4.0.0-dev", path = "../utils" } +sc-foo = { version = "4.0.0-dev", path = "../utils" } serde = { version = "1.0.126", features = ["derive"] } linked-hash-map = "0.5.4" retain_mut = "0.1.3" diff --git a/client/transaction-pool/graph/Cargo.toml b/client/transaction-pool/graph/Cargo.toml index b49cadc51c33c..57de7740ced44 100644 --- a/client/transaction-pool/graph/Cargo.toml +++ b/client/transaction-pool/graph/Cargo.toml @@ -20,7 +20,7 @@ log = "0.4.8" parking_lot = "0.11.1" serde = { version = "1.0.101", features = ["derive"] } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sc-utils = { version = "4.0.0-dev", path = "../../utils" } +sc-foo = { version = "4.0.0-dev", path = "../../utils" } sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } sp-transaction-pool = { version = "4.0.0-dev", path = "../../../primitives/transaction-pool" } diff --git a/client/transaction-pool/src/graph/watcher.rs b/client/transaction-pool/src/graph/watcher.rs index 975ee6608886b..eea5c604d0ea1 100644 --- a/client/transaction-pool/src/graph/watcher.rs +++ b/client/transaction-pool/src/graph/watcher.rs @@ -20,7 +20,7 @@ use futures::Stream; use sc_transaction_pool_api::TransactionStatus; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; /// Extrinsic watcher. /// diff --git a/client/transaction-pool/src/revalidation.rs b/client/transaction-pool/src/revalidation.rs index a8b2c1d32036a..fa1072970734b 100644 --- a/client/transaction-pool/src/revalidation.rs +++ b/client/transaction-pool/src/revalidation.rs @@ -25,7 +25,7 @@ use std::{ }; use crate::graph::{ChainApi, ExtrinsicHash, NumberFor, Pool, ValidatedTransaction}; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_runtime::{ generic::BlockId, traits::{SaturatedConversion, Zero}, diff --git a/client/utils/Cargo.toml b/client/utils/Cargo.toml index 99765dd501dd5..b99f43c5dedd8 100644 --- a/client/utils/Cargo.toml +++ b/client/utils/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "sc-utils" +name = "sc-foo" version = "4.0.0-dev" authors = ["Parity Technologies "] edition = "2018" From 82dc31d951f9af5e066379d5bef6e0c46ad242fe Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 7 Sep 2021 18:00:45 -0300 Subject: [PATCH 05/17] have stricter script options --- .maintain/gitlab/check_polkadot_companion_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index 25d749ae58c3d..ef042ac2fce9b 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -9,7 +9,7 @@ # polkadot companion: paritytech/polkadot#567 # -set -e +set -euo pipefail github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls" # use github api v3 in order to access the data without authentication From a572229479728cdc142b01c8a19bb7a0eb3b9e1a Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 7 Sep 2021 18:02:44 -0300 Subject: [PATCH 06/17] no useless file substitution --- .maintain/gitlab/check_polkadot_companion_build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index ef042ac2fce9b..af4c554a651ce 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -105,13 +105,13 @@ load_our_crates() { else our_crates+=("$crate") fi - done < <(jq -r ' + done < <(cargo metadata --quiet --format-version=1 | jq -r ' . as $in | paths | select(.[-1]=="source" and . as $p | $in | getpath($p)==null) as $path | del($path[-1]) as $path | $in | getpath($path + ["name"]) - ' < <(cargo metadata --quiet --format-version=1)) + ') } load_our_crates @@ -171,14 +171,14 @@ match_their_crates() { exit 1 ;; esac - done < <(jq -r ' + done < <(cargo metadata --quiet --format-version=1 | jq -r ' . as $in | paths(select(type=="string")) | select(.[-1]=="source") as $source_path | del($source_path[-1]) as $path | [$in | getpath($path + ["name"]), getpath($path + ["source"])] | .[] - ' < <(cargo metadata --quiet --format-version=1)) + ') if [ "$crates_not_found" ]; then echo "Errors during crate matching" From 56e0a67ffedcd81565d4ee11a54bdf38379e7552 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 7 Sep 2021 18:12:20 -0300 Subject: [PATCH 07/17] isolate check-polkadot-companion-build for faster iteration deletion will be reverted later --- .gitlab-ci.yml | 603 ------------------------------------------------- 1 file changed, 603 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 74ed64315d62e..ddb23773f93a8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -165,615 +165,12 @@ default: | tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json' - sccache -s -#### stage: .pre - -skip-if-draft: - image: paritytech/tools:latest - <<: *kubernetes-env - stage: .pre - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - script: - - echo "Commit message is ${CI_COMMIT_MESSAGE}" - - echo "Ref is ${CI_COMMIT_REF_NAME}" - - echo "pipeline source is ${CI_PIPELINE_SOURCE}" - - ./.maintain/gitlab/skip_if_draft.sh - -#### stage: check - -check-runtime: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - variables: - <<: *default-vars - GITLAB_API: "https://gitlab.parity.io/api/v4" - GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" - script: - - ./.maintain/gitlab/check_runtime.sh - allow_failure: true - -check-signed-tag: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - - ./.maintain/gitlab/check_signed.sh - -check-line-width: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - script: - - ./.maintain/gitlab/check_line_width.sh - allow_failure: true - -test-dependency-rules: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - script: - - .maintain/ensure-deps.sh - -test-prometheus-alerting-rules: - stage: check - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_COMMIT_BRANCH - changes: - - .gitlab-ci.yml - - .maintain/monitoring/**/* - script: - - promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml - - cat .maintain/monitoring/alerting-rules/alerting-rules.yaml | - promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml - -#### stage: test - -cargo-deny: - stage: test - <<: *docker-env - <<: *nightly-pipeline - script: - - cargo deny check --hide-inclusion-graph -c .maintain/deny.toml - after_script: - - echo "___The complete log is in the artifacts___" - - cargo deny check -c .maintain/deny.toml 2> deny.log - artifacts: - name: $CI_COMMIT_SHORT_SHA - expire_in: 3 days - when: always - paths: - - deny.log - # FIXME: Temorarily allow to fail. - allow_failure: true - -cargo-fmt: - stage: test - <<: *docker-env - <<: *test-refs - script: - - cargo +nightly fmt --all -- --check - -cargo-check-benches: - stage: test - <<: *docker-env - <<: *test-refs - <<: *collect-artifacts - before_script: - # merges in the master branch on PRs - - *merge-ref-into-master-script - - *rust-info-script - script: - - *cargo-check-benches-script - -node-bench-regression-guard: - # it's not belong to `build` semantically, but dag jobs can't depend on each other - # within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632 - # more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402 - stage: build - <<: *docker-env - <<: *test-refs-no-trigger-prs-only - needs: - # this is a DAG - - job: cargo-check-benches - artifacts: true - # this does not like a DAG, just polls the artifact - - project: $CI_PROJECT_PATH - job: cargo-check-benches - ref: master - artifacts: true - variables: - CI_IMAGE: "paritytech/node-bench-regression-guard:latest" - before_script: [""] - script: - - 'node-bench-regression-guard --reference artifacts/benches/master-* - --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA' - -cargo-check-subkey: - stage: test - <<: *docker-env - <<: *test-refs - script: - - cd ./bin/utils/subkey - - SKIP_WASM_BUILD=1 time cargo check --release - - sccache -s - -cargo-check-try-runtime: - stage: test - <<: *docker-env - <<: *test-refs - script: - - time cargo check --features try-runtime - - sccache -s - -cargo-check-wasmer-sandbox: - stage: test - <<: *docker-env - <<: *test-refs - script: - - time cargo check --features wasmer-sandbox - - sccache -s - -test-deterministic-wasm: - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - WASM_BUILD_NO_COLOR: 1 - script: - # build runtime - - cargo build --verbose --release -p node-runtime - # make checksum - - sha256sum target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256 - # clean up – FIXME: can we reuse some of the artifacts? - - cargo clean - # build again - - cargo build --verbose --release -p node-runtime - # confirm checksum - - sha256sum -c checksum.sha256 - - sccache -s - -test-linux-stable: &test-linux - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - RUST_BACKTRACE: 1 - WASM_BUILD_NO_COLOR: 1 - script: - # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests - - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml - - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec - - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout - - sccache -s - -#unleash-check: - #stage: test - #<<: *docker-env - #<<: *test-refs-no-trigger - #script: - #- cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - #- cargo unleash de-dev-deps - # Reuse build artifacts when running checks (cuts down check time by 3x) - # TODO: Implement this optimization in cargo-unleash rather than here - #- mkdir -p target/unleash - #- export CARGO_TARGET_DIR=target/unleash - #- cargo unleash check ${CARGO_UNLEASH_PKG_DEF} - # FIXME: this job must not fail, or unleash-to-crates-io will publish broken stuff - #allow_failure: true - -test-frame-examples-compile-to-wasm: - # into one job - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y" - RUST_BACKTRACE: 1 - script: - - cd frame/example-offchain-worker/ - - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - - cd ../example - - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features - - sccache -s - -test-linux-stable-int: - <<: *test-linux - stage: test - script: - - echo "___Logs will be partly shown at the end in case of failure.___" - - echo "___Full log will be saved to the job artifacts only in case of failure.___" - - WASM_BUILD_NO_COLOR=1 - RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace - time cargo test -p node-cli --release --verbose --locked -- --ignored - &> ${CI_COMMIT_SHORT_SHA}_int_failure.log - - sccache -s - after_script: - - awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log - artifacts: - name: $CI_COMMIT_SHORT_SHA - when: on_failure - expire_in: 3 days - paths: - - ${CI_COMMIT_SHORT_SHA}_int_failure.log - -check-tracing: - stage: test - <<: *docker-env - <<: *test-refs - script: - # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases - - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features - - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features --features=with-tracing - - sccache -s - -test-full-crypto-feature: - stage: test - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y" - RUST_BACKTRACE: 1 - script: - - cd primitives/core/ - - time cargo +nightly build --verbose --no-default-features --features full_crypto - - cd ../application-crypto - - time cargo +nightly build --verbose --no-default-features --features full_crypto - - sccache -s - -test-wasmer-sandbox: - stage: test - <<: *docker-env - <<: *test-refs-wasmer-sandbox - variables: - <<: *default-vars - script: - - time cargo test --release --features runtime-benchmarks,wasmer-sandbox - - sccache -s - -cargo-check-macos: - stage: test - # shell runner on mac ignores the image set in *docker-env - <<: *docker-env - <<: *test-refs-no-trigger - script: - - SKIP_WASM_BUILD=1 time cargo check --release - - sccache -s - tags: - - osx - -#### stage: build - -check-polkadot-companion-status: - stage: build - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - script: - - ./.maintain/gitlab/check_polkadot_companion_status.sh - check-polkadot-companion-build: stage: build <<: *docker-env <<: *test-refs-no-trigger - needs: - - job: test-linux-stable-int - artifacts: false script: - ./.maintain/gitlab/check_polkadot_companion_build.sh after_script: - cd polkadot && git rev-parse --abbrev-ref HEAD allow_failure: true - -build-linux-substrate: &build-binary - stage: build - <<: *collect-artifacts - <<: *docker-env - <<: *build-refs - needs: - - job: test-linux-stable - artifacts: false - before_script: - - mkdir -p ./artifacts/substrate/ - script: - - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose - - mv ./target/release/substrate ./artifacts/substrate/. - - echo -n "Substrate version = " - - if [ "${CI_COMMIT_TAG}" ]; then - echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION; - else - ./artifacts/substrate/substrate --version | - sed -n -E 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' | - tee ./artifacts/substrate/VERSION; - fi - - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256 - - printf '\n# building node-template\n\n' - - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz - - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/ - - sccache -s - -build-linux-subkey: &build-subkey - stage: build - <<: *collect-artifacts - <<: *docker-env - <<: *build-refs - needs: - - job: cargo-check-subkey - artifacts: false - before_script: - - mkdir -p ./artifacts/subkey - script: - - cd ./bin/utils/subkey - - SKIP_WASM_BUILD=1 time cargo build --release --verbose - - cd - - - mv ./target/release/subkey ./artifacts/subkey/. - - echo -n "Subkey version = " - - ./artifacts/subkey/subkey --version | - sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' | - tee ./artifacts/subkey/VERSION; - - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256 - - cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/ - - sccache -s - -build-macos-subkey: - <<: *build-subkey - tags: - - osx - -build-rustdoc: - stage: build - <<: *docker-env - <<: *test-refs - variables: - <<: *default-vars - SKIP_WASM_BUILD: 1 - artifacts: - name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" - when: on_success - expire_in: 7 days - paths: - - ./crate-docs/ - script: - # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` - - RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html" - time cargo +nightly doc --no-deps --workspace --all-features --verbose - - rm -f ./target/doc/.lock - - mv ./target/doc ./crate-docs - # FIXME: remove me after CI image gets nonroot - - chown -R nonroot:nonroot ./crate-docs - - echo "" > ./crate-docs/index.html - - sccache -s - -#### stage: publish - -.build-push-docker-image: &build-push-docker-image - <<: *build-refs - <<: *kubernetes-env - image: quay.io/buildah/stable - variables: &docker-build-vars - <<: *default-vars - GIT_STRATEGY: none - DOCKERFILE: $PRODUCT.Dockerfile - IMAGE_NAME: docker.io/parity/$PRODUCT - before_script: - - cd ./artifacts/$PRODUCT/ - - VERSION="$(cat ./VERSION)" - - echo "${PRODUCT} version = ${VERSION}" - - test -z "${VERSION}" && exit 1 - script: - - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" || - ( echo "no docker credentials provided"; exit 1 ) - - buildah bud - --format=docker - --build-arg VCS_REF="${CI_COMMIT_SHA}" - --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" - --tag "$IMAGE_NAME:$VERSION" - --tag "$IMAGE_NAME:latest" - --file "$DOCKERFILE" . - - echo "$Docker_Hub_Pass_Parity" | - buildah login --username "$Docker_Hub_User_Parity" --password-stdin docker.io - - buildah info - - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION" - - buildah push --format=v2s2 "$IMAGE_NAME:latest" - after_script: - - buildah logout --all - # pass artifacts to the trigger-simnet job - - echo "SUBSTRATE_IMAGE_NAME=${IMAGE_NAME}" | tee -a ./artifacts/$PRODUCT/build.env - - IMAGE_TAG="$(cat ./artifacts/$PRODUCT/VERSION)" - - echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}" | tee -a ./artifacts/$PRODUCT/build.env - - cat ./artifacts/$PRODUCT/build.env - -publish-docker-substrate: - stage: publish - <<: *build-push-docker-image - <<: *build-refs - needs: - - job: build-linux-substrate - artifacts: true - variables: - <<: *docker-build-vars - PRODUCT: substrate - artifacts: - reports: - # this artifact is used in trigger-simnet job - # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance - dotenv: ./artifacts/substrate/build.env - -publish-docker-subkey: - stage: publish - <<: *build-push-docker-image - needs: - - job: build-linux-subkey - artifacts: true - variables: - <<: *docker-build-vars - PRODUCT: subkey - -publish-s3-release: - stage: publish - <<: *build-refs - <<: *kubernetes-env - needs: - - job: build-linux-substrate - artifacts: true - - job: build-linux-subkey - artifacts: true - image: paritytech/awscli:latest - variables: - GIT_STRATEGY: none - BUCKET: "releases.parity.io" - PREFIX: "substrate/${ARCH}-${DOCKER_OS}" - script: - - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ - - echo "update objects in latest path" - - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/ - after_script: - - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/ - --recursive --human-readable --summarize - -publish-rustdoc: - stage: publish - <<: *kubernetes-env - image: paritytech/tools:latest - variables: - GIT_DEPTH: 100 - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME == "master" - # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other - # artifacts from being dowloaded by this job. - needs: - - job: build-rustdoc - artifacts: true - script: - - rm -rf /tmp/* - # Set git config - - rm -rf .git/config - - git config user.email "devops-team@parity.io" - - git config user.name "${GITHUB_USER}" - - git config remote.origin.url "https://${GITHUB_TOKEN}@github.com/paritytech/substrate.git" - - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - - git fetch origin gh-pages - # Save README and docs - - cp -r ./crate-docs/ /tmp/doc/ - - cp README.md /tmp/doc/ - - git checkout gh-pages - # Remove everything and restore generated docs and README - - rm -rf ./* - - mv /tmp/doc/* . - # Upload files - - git add --all --force - # `git commit` has an exit code of > 0 if there is nothing to commit. - # This causes GitLab to exit immediately and marks this job failed. - # We don't want to mark the entire job failed if there's nothing to - # publish though, hence the `|| true`. - - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || - echo "___Nothing to commit___" - - git push origin gh-pages --force - after_script: - - rm -rf .git/ ./* - -publish-draft-release: - stage: publish - image: paritytech/tools:latest - rules: - - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - - ./.maintain/gitlab/publish_draft_release.sh - allow_failure: true - -unleash-to-crates-io: - stage: publish - <<: *docker-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ - # FIXME: wait until https://github.com/paritytech/cargo-unleash/issues/50 is fixed, also - # remove allow_failure: true on the check job - # - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS} - - cargo unleash em-dragons --no-check --owner github:paritytech:core-devs ${CARGO_UNLEASH_PKG_DEF} - allow_failure: true - -#### stage: deploy - -deploy-prometheus-alerting-rules: - stage: deploy - needs: - - job: test-prometheus-alerting-rules - artifacts: false - allow_failure: true - trigger: - project: parity/infrastructure/cloud-infra - variables: - SUBSTRATE_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" - SUBSTRATE_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" - UPSTREAM_TRIGGER_PROJECT: "${CI_PROJECT_PATH}" - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_COMMIT_REF_NAME == "master" - changes: - - .gitlab-ci.yml - - .maintain/monitoring/**/* - -# Runs "quick" and "long" tests on nightly schedule and on commit / merge to master -# A "quick" test is a smoke test where basic check-expect tests run by -# checking values from metrics exposed by the app. -# A "long" test is the load testing where we send 50K transactions into the -# network and check if all completed successfully -simnet-tests: - stage: deploy - image: docker.io/paritytech/simnet:${SIMNET_REF} - <<: *kubernetes-env - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME == "master" - needs: - - job: publish-docker-substrate - # variables: - # `build.env` brings here `${SUBSTRATE_IMAGE_NAME}` and `${SUBSTRATE_IMAGE_TAG}` - # (`$VERSION` here, # i.e. `2643-0.8.29-5f689e0a-6b24dc54`). - # ${SIMNET_REF} is a gitlab variable - before_script: - - echo "Simnet Tests Config - docker.io/paritytech/simnet:${SIMNET_REF} - ${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IAMGE_TAG}" - script: - - /home/nonroot/simnet/gurke/scripts/run-test-environment-manager.sh - --github-remote-dir="https://github.com/paritytech/substrate/tree/master/simnet_tests" - --config="simnet_tests/configs/default_local_testnet.toml" - --image="${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}" - retry: 2 - tags: - - parity-simnet From 87e91fd5e991ab3dac3a499f21b718883a795521 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 7 Sep 2021 19:29:38 -0300 Subject: [PATCH 08/17] jq update workaround --- .../gitlab/check_polkadot_companion_build.sh | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index af4c554a651ce..48fa6cd674615 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -37,6 +37,17 @@ from this pull request. otherwise, it will uses master instead EOT +# FIXME: update jq in the CI file + +jq="$PWD/jq16" +curl -sqL https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o "$jq" +chmod +x "$jq" +jq_sha256sum="af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 $jq" +if [ "$(sha256sum "$jq")" != "$jq_sha256sum" ]; then + echo "ERROR: jq sha256sum mismatch" + exit 1 +fi + # Set the user name and email to make merging work git config --global user.name 'CI system' git config --global user.email '<>' @@ -105,7 +116,7 @@ load_our_crates() { else our_crates+=("$crate") fi - done < <(cargo metadata --quiet --format-version=1 | jq -r ' + done < <(cargo metadata --quiet --format-version=1 | "$jq" -r ' . as $in | paths | select(.[-1]=="source" and . as $p | $in | getpath($p)==null) as $path | @@ -124,14 +135,13 @@ match_their_crates() { local target_name="$(basename "$target_dir")" local crates_not_found=() - local found - # output will be provided in the format: # crate # source # crate # ... local next="crate" + local found while IFS= read -r line; do case "$next" in crate) @@ -171,7 +181,7 @@ match_their_crates() { exit 1 ;; esac - done < <(cargo metadata --quiet --format-version=1 | jq -r ' + done < <(cargo metadata --quiet --format-version=1 | "$jq" -r ' . as $in | paths(select(type=="string")) | select(.[-1]=="source") as $source_path | @@ -180,7 +190,7 @@ match_their_crates() { .[] ') - if [ "$crates_not_found" ]; then + if [ "${crates_not_found[@]}" ]; then echo "Errors during crate matching" printf "Failed to find crate \"%s\" referenced in $target_name\n" "${crates_not_found[@]}" exit 1 From a8d3de2945d9d100d3863be105f27d06a51712f5 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 7 Sep 2021 20:16:15 -0300 Subject: [PATCH 09/17] fix pattern comparison --- .maintain/gitlab/check_polkadot_companion_build.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index 48fa6cd674615..52302796fa012 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -51,6 +51,7 @@ fi # Set the user name and email to make merging work git config --global user.name 'CI system' git config --global user.email '<>' +git config --global pull.rebase false # Merge master into our branch before building Polkadot to make sure we don't miss # any commits that are required by Polkadot. @@ -96,13 +97,13 @@ else boldprint "this is not a pull request - building polkadot:master" fi -cd "$substrate_dir" - our_crates=() our_crates_source="git+https://github.com/paritytech/substrate" load_our_crates() { local found + cd "$substrate_dir" + while IFS= read -r crate; do # for avoiding duplicate entries for our_crate in "${our_crates[@]}"; do @@ -135,6 +136,7 @@ match_their_crates() { local target_name="$(basename "$target_dir")" local crates_not_found=() + echo "$target_dir" # output will be provided in the format: # crate # source @@ -149,7 +151,8 @@ match_their_crates() { next="source" ;; source) - if [ "$line" == "$our_crates_source" ] || [ "$line" == "$our_crates_source?" ]; then + if [ "$line" == "$our_crates_source" ] || [[ "$line" == "$our_crates_source?"* ]]; then + echo "$line" for our_crate in "${our_crates[@]}"; do if [ "$our_crate" == "$crate" ]; then found=true From 40e91039870ae88c9903cce5705e77e51549d2b2 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 7 Sep 2021 20:18:55 -0300 Subject: [PATCH 10/17] improve error message --- .maintain/gitlab/check_polkadot_companion_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index 52302796fa012..5f56cc0013880 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -195,7 +195,7 @@ match_their_crates() { if [ "${crates_not_found[@]}" ]; then echo "Errors during crate matching" - printf "Failed to find crate \"%s\" referenced in $target_name\n" "${crates_not_found[@]}" + printf "Failed to detect our crate \"%s\" referenced in $target_name\n" "${crates_not_found[@]}" exit 1 fi } From 766e5bd6d3c57968ae7a454001fdbdc0e26054ec Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 7 Sep 2021 20:25:25 -0300 Subject: [PATCH 11/17] Revert "sc_utils => sc_foo (for testing's sake)" This reverts commit 0c258de4e5ab271c16414d1ad8f188f9e9be84c6. --- Cargo.lock | 40 +++++++++---------- client/api/Cargo.toml | 2 +- client/api/src/client.rs | 2 +- client/api/src/notifications.rs | 2 +- client/cli/Cargo.toml | 2 +- client/cli/src/runner.rs | 2 +- client/consensus/common/Cargo.toml | 2 +- .../common/src/import_queue/basic_queue.rs | 2 +- .../common/src/import_queue/buffered_link.rs | 2 +- client/finality-grandpa/Cargo.toml | 2 +- .../src/communication/gossip.rs | 2 +- .../finality-grandpa/src/communication/mod.rs | 2 +- .../src/communication/periodic.rs | 2 +- .../src/communication/tests.rs | 2 +- client/finality-grandpa/src/import.rs | 2 +- client/finality-grandpa/src/lib.rs | 2 +- client/finality-grandpa/src/notification.rs | 2 +- client/finality-grandpa/src/observer.rs | 4 +- client/finality-grandpa/src/until_imported.rs | 4 +- client/network/Cargo.toml | 2 +- client/network/src/on_demand_layer.rs | 2 +- client/network/src/service.rs | 2 +- client/offchain/Cargo.toml | 2 +- client/offchain/src/api/http.rs | 2 +- client/peerset/Cargo.toml | 2 +- client/peerset/src/lib.rs | 2 +- client/rpc/Cargo.toml | 2 +- client/rpc/src/system/mod.rs | 2 +- client/rpc/src/system/tests.rs | 2 +- client/service/Cargo.toml | 2 +- client/service/src/builder.rs | 2 +- client/service/src/client/client.rs | 2 +- client/service/src/lib.rs | 2 +- client/service/src/metrics.rs | 2 +- client/service/src/task_manager/mod.rs | 2 +- client/transaction-pool/Cargo.toml | 2 +- client/transaction-pool/graph/Cargo.toml | 2 +- client/transaction-pool/src/graph/watcher.rs | 2 +- client/transaction-pool/src/revalidation.rs | 2 +- client/utils/Cargo.toml | 2 +- 40 files changed, 61 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c2ac8903762e..d5205f0dd4110 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7307,12 +7307,12 @@ dependencies = [ "regex", "rpassword", "sc-client-api", - "sc-foo", "sc-keystore", "sc-network", "sc-service", "sc-telemetry", "sc-tracing", + "sc-utils", "serde", "serde_json", "sp-blockchain", @@ -7340,8 +7340,8 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.11.1", "sc-executor", - "sc-foo", "sc-transaction-pool-api", + "sc-utils", "sp-api", "sp-blockchain", "sp-consensus", @@ -7398,7 +7398,7 @@ dependencies = [ "log 0.4.14", "parking_lot 0.11.1", "sc-client-api", - "sc-foo", + "sc-utils", "serde", "sp-api", "sp-blockchain", @@ -7747,12 +7747,12 @@ dependencies = [ "sc-block-builder", "sc-client-api", "sc-consensus", - "sc-foo", "sc-keystore", "sc-network", "sc-network-gossip", "sc-network-test", "sc-telemetry", + "sc-utils", "serde_json", "sp-api", "sp-application-crypto", @@ -7798,16 +7798,6 @@ dependencies = [ "substrate-test-runtime-client", ] -[[package]] -name = "sc-foo" -version = "4.0.0-dev" -dependencies = [ - "futures 0.3.16", - "futures-timer 3.0.2", - "lazy_static", - "prometheus", -] - [[package]] name = "sc-informant" version = "0.10.0-dev" @@ -7890,8 +7880,8 @@ dependencies = [ "sc-block-builder", "sc-client-api", "sc-consensus", - "sc-foo", "sc-peerset", + "sc-utils", "serde", "serde_json", "smallvec 1.6.1", @@ -7978,10 +7968,10 @@ dependencies = [ "sc-block-builder", "sc-client-api", "sc-client-db", - "sc-foo", "sc-network", "sc-transaction-pool", "sc-transaction-pool-api", + "sc-utils", "sp-api", "sp-consensus", "sp-core", @@ -8001,7 +7991,7 @@ dependencies = [ "libp2p", "log 0.4.14", "rand 0.7.3", - "sc-foo", + "sc-utils", "serde_json", "wasm-timer", ] @@ -8030,12 +8020,12 @@ dependencies = [ "sc-block-builder", "sc-chain-spec", "sc-client-api", - "sc-foo", "sc-network", "sc-rpc-api", "sc-tracing", "sc-transaction-pool", "sc-transaction-pool-api", + "sc-utils", "serde_json", "sp-api", "sp-blockchain", @@ -8128,7 +8118,6 @@ dependencies = [ "sc-client-db", "sc-consensus", "sc-executor", - "sc-foo", "sc-informant", "sc-keystore", "sc-light", @@ -8140,6 +8129,7 @@ dependencies = [ "sc-tracing", "sc-transaction-pool", "sc-transaction-pool-api", + "sc-utils", "serde", "serde_json", "sp-api", @@ -8312,8 +8302,8 @@ dependencies = [ "retain_mut", "sc-block-builder", "sc-client-api", - "sc-foo", "sc-transaction-pool-api", + "sc-utils", "serde", "sp-api", "sp-blockchain", @@ -8342,6 +8332,16 @@ dependencies = [ "thiserror", ] +[[package]] +name = "sc-utils" +version = "4.0.0-dev" +dependencies = [ + "futures 0.3.16", + "futures-timer 3.0.2", + "lazy_static", + "prometheus", +] + [[package]] name = "schannel" version = "0.1.19" diff --git a/client/api/Cargo.toml b/client/api/Cargo.toml index d820b62a66acc..772f22e822eb2 100644 --- a/client/api/Cargo.toml +++ b/client/api/Cargo.toml @@ -30,7 +30,7 @@ sp-database = { version = "4.0.0-dev", path = "../../primitives/database" } sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } sp-keystore = { version = "0.10.0-dev", default-features = false, path = "../../primitives/keystore" } sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } -sc-foo = { version = "4.0.0-dev", path = "../utils" } +sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } sp-state-machine = { version = "0.10.0-dev", path = "../../primitives/state-machine" } sp-trie = { version = "4.0.0-dev", path = "../../primitives/trie" } diff --git a/client/api/src/client.rs b/client/api/src/client.rs index 9b64800e3a1a2..21f8aecad0536 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -29,7 +29,7 @@ use std::{collections::HashSet, convert::TryFrom, fmt, sync::Arc}; use crate::{blockchain::Info, notifications::StorageEventStream}; use sc_transaction_pool_api::ChainEvent; -use sc_foo::mpsc::TracingUnboundedReceiver; +use sc_utils::mpsc::TracingUnboundedReceiver; use sp_blockchain; /// Type that implements `futures::Stream` of block import events. diff --git a/client/api/src/notifications.rs b/client/api/src/notifications.rs index 3a366c1e1b7dc..1346afd5e54d2 100644 --- a/client/api/src/notifications.rs +++ b/client/api/src/notifications.rs @@ -25,7 +25,7 @@ use std::{ use fnv::{FnvHashMap, FnvHashSet}; use prometheus_endpoint::{register, CounterVec, Opts, Registry, U64}; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_core::storage::{StorageData, StorageKey}; use sp_runtime::traits::Block as BlockT; diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index b00a89d316732..7798507e529f4 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -30,7 +30,7 @@ sc-client-api = { version = "4.0.0-dev", path = "../api" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sc-network = { version = "0.10.0-dev", path = "../network" } sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sc-foo = { version = "4.0.0-dev", path = "../utils" } +sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-version = { version = "4.0.0-dev", path = "../../primitives/version" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore" } diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index 972bd98530a47..2ec200d9285b1 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -21,7 +21,7 @@ use chrono::prelude::*; use futures::{future, future::FutureExt, pin_mut, select, Future}; use log::info; use sc_service::{Configuration, Error as ServiceError, TaskManager, TaskType}; -use sc_foo::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL}; +use sc_utils::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL}; use std::marker::PhantomData; #[cfg(target_family = "unix")] diff --git a/client/consensus/common/Cargo.toml b/client/consensus/common/Cargo.toml index ad81f168d0cac..6829bd2c6d8b5 100644 --- a/client/consensus/common/Cargo.toml +++ b/client/consensus/common/Cargo.toml @@ -24,7 +24,7 @@ sp-core = { path = "../../../primitives/core", version = "4.0.0-dev" } sp-consensus = { path = "../../../primitives/consensus/common", version = "0.10.0-dev" } sp-state-machine = { version = "0.10.0-dev", path = "../../../primitives/state-machine" } sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } -sc-foo = { version = "4.0.0-dev", path = "../../utils" } +sc-utils = { version = "4.0.0-dev", path = "../../utils" } sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } parking_lot = "0.11.1" serde = { version = "1.0", features = ["derive"] } diff --git a/client/consensus/common/src/import_queue/basic_queue.rs b/client/consensus/common/src/import_queue/basic_queue.rs index 7e76a7fdb821e..9042c8798be4f 100644 --- a/client/consensus/common/src/import_queue/basic_queue.rs +++ b/client/consensus/common/src/import_queue/basic_queue.rs @@ -22,7 +22,7 @@ use futures::{ use futures_timer::Delay; use log::{debug, trace}; use prometheus_endpoint::Registry; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_consensus::BlockOrigin; use sp_runtime::{ traits::{Block as BlockT, Header as HeaderT, NumberFor}, diff --git a/client/consensus/common/src/import_queue/buffered_link.rs b/client/consensus/common/src/import_queue/buffered_link.rs index 3960718d6d771..87ea6dde5c473 100644 --- a/client/consensus/common/src/import_queue/buffered_link.rs +++ b/client/consensus/common/src/import_queue/buffered_link.rs @@ -40,7 +40,7 @@ use crate::import_queue::{Link, Origin}; use futures::prelude::*; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_runtime::traits::{Block as BlockT, NumberFor}; use std::{ pin::Pin, diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml index e0ed612c9ffee..40385a2faea57 100644 --- a/client/finality-grandpa/Cargo.toml +++ b/client/finality-grandpa/Cargo.toml @@ -27,7 +27,7 @@ parity-scale-codec = { version = "2.0.0", features = ["derive"] } sp-application-crypto = { version = "4.0.0-dev", path = "../../primitives/application-crypto" } sp-arithmetic = { version = "4.0.0-dev", path = "../../primitives/arithmetic" } sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sc-foo = { version = "4.0.0-dev", path = "../utils" } +sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" } sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } diff --git a/client/finality-grandpa/src/communication/gossip.rs b/client/finality-grandpa/src/communication/gossip.rs index 964f3fb36c672..d64c7421afa69 100644 --- a/client/finality-grandpa/src/communication/gossip.rs +++ b/client/finality-grandpa/src/communication/gossip.rs @@ -94,7 +94,7 @@ use log::{debug, trace}; use prometheus_endpoint::{register, CounterVec, Opts, PrometheusError, Registry, U64}; use rand::seq::SliceRandom; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG}; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use super::{benefit, cost, Round, SetId}; use crate::{environment, CatchUp, CompactCommit, SignedMessage}; diff --git a/client/finality-grandpa/src/communication/mod.rs b/client/finality-grandpa/src/communication/mod.rs index 280340236cedc..c370e1d642d7d 100644 --- a/client/finality-grandpa/src/communication/mod.rs +++ b/client/finality-grandpa/src/communication/mod.rs @@ -58,7 +58,7 @@ use crate::{ use gossip::{ FullCatchUpMessage, FullCommitMessage, GossipMessage, GossipValidator, PeerReport, VoteMessage, }; -use sc_foo::mpsc::TracingUnboundedReceiver; +use sc_utils::mpsc::TracingUnboundedReceiver; use sp_finality_grandpa::{AuthorityId, AuthoritySignature, RoundNumber, SetId as SetIdNumber}; pub mod gossip; diff --git a/client/finality-grandpa/src/communication/periodic.rs b/client/finality-grandpa/src/communication/periodic.rs index a09cfab6b3947..77e55ad652f6c 100644 --- a/client/finality-grandpa/src/communication/periodic.rs +++ b/client/finality-grandpa/src/communication/periodic.rs @@ -21,7 +21,7 @@ use futures::{future::FutureExt as _, prelude::*, ready, stream::Stream}; use futures_timer::Delay; use log::debug; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use std::{ pin::Pin, task::{Context, Poll}, diff --git a/client/finality-grandpa/src/communication/tests.rs b/client/finality-grandpa/src/communication/tests.rs index 509ffd4edd8ac..1fac0230b2a84 100644 --- a/client/finality-grandpa/src/communication/tests.rs +++ b/client/finality-grandpa/src/communication/tests.rs @@ -28,7 +28,7 @@ use parity_scale_codec::Encode; use sc_network::{config::Role, Event as NetworkEvent, ObservedRole, PeerId}; use sc_network_gossip::Validator; use sc_network_test::{Block, Hash}; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_finality_grandpa::AuthorityList; use sp_keyring::Ed25519Keyring; use sp_runtime::traits::NumberFor; diff --git a/client/finality-grandpa/src/import.rs b/client/finality-grandpa/src/import.rs index 2ee0b4ec2de6e..f663bfe94afdf 100644 --- a/client/finality-grandpa/src/import.rs +++ b/client/finality-grandpa/src/import.rs @@ -27,7 +27,7 @@ use sc_consensus::{ BlockCheckParams, BlockImport, BlockImportParams, ImportResult, JustificationImport, }; use sc_telemetry::TelemetryHandle; -use sc_foo::mpsc::TracingUnboundedSender; +use sc_utils::mpsc::TracingUnboundedSender; use sp_api::{Core, RuntimeApiInfo, TransactionFor}; use sp_blockchain::{well_known_cache_keys, BlockStatus}; use sp_consensus::{BlockOrigin, Error as ConsensusError, SelectChain}; diff --git a/client/finality-grandpa/src/lib.rs b/client/finality-grandpa/src/lib.rs index 6ce476ef5c96c..452659ced6a70 100644 --- a/client/finality-grandpa/src/lib.rs +++ b/client/finality-grandpa/src/lib.rs @@ -68,7 +68,7 @@ use sc_client_api::{ }; use sc_consensus::BlockImport; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO}; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; use sp_api::ProvideRuntimeApi; use sp_application_crypto::AppKey; use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata}; diff --git a/client/finality-grandpa/src/notification.rs b/client/finality-grandpa/src/notification.rs index b1b28d4f11ca9..85d581bd5065e 100644 --- a/client/finality-grandpa/src/notification.rs +++ b/client/finality-grandpa/src/notification.rs @@ -19,7 +19,7 @@ use parking_lot::Mutex; use std::sync::Arc; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_runtime::traits::Block as BlockT; use crate::{justification::GrandpaJustification, Error}; diff --git a/client/finality-grandpa/src/observer.rs b/client/finality-grandpa/src/observer.rs index 7319a44aded13..70a94cd504726 100644 --- a/client/finality-grandpa/src/observer.rs +++ b/client/finality-grandpa/src/observer.rs @@ -29,7 +29,7 @@ use log::{debug, info, warn}; use sc_client_api::backend::Backend; use sc_telemetry::TelemetryHandle; -use sc_foo::mpsc::TracingUnboundedReceiver; +use sc_utils::mpsc::TracingUnboundedReceiver; use sp_blockchain::HeaderMetadata; use sp_consensus::SelectChain; use sp_finality_grandpa::AuthorityId; @@ -403,7 +403,7 @@ mod tests { }; use assert_matches::assert_matches; use sc_network::PeerId; - use sc_foo::mpsc::tracing_unbounded; + use sc_utils::mpsc::tracing_unbounded; use sp_blockchain::HeaderBackend as _; use substrate_test_runtime_client::{TestClientBuilder, TestClientBuilderExt}; diff --git a/client/finality-grandpa/src/until_imported.rs b/client/finality-grandpa/src/until_imported.rs index 45c7a56fbc950..deb6577264347 100644 --- a/client/finality-grandpa/src/until_imported.rs +++ b/client/finality-grandpa/src/until_imported.rs @@ -37,7 +37,7 @@ use log::{debug, warn}; use parking_lot::Mutex; use prometheus_endpoint::{register, Gauge, PrometheusError, Registry, U64}; use sc_client_api::{BlockImportNotification, ImportNotifications}; -use sc_foo::mpsc::TracingUnboundedReceiver; +use sc_utils::mpsc::TracingUnboundedReceiver; use sp_finality_grandpa::AuthorityId; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; @@ -561,7 +561,7 @@ mod tests { use futures::future::Either; use futures_timer::Delay; use sc_client_api::BlockImportNotification; - use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedSender}; + use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_consensus::BlockOrigin; use substrate_test_runtime_client::runtime::{Block, Hash, Header}; diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index 7b9fab97ae090..de62a534f8662 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -55,7 +55,7 @@ sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/comm sc-consensus = { version = "0.10.0-dev", path = "../consensus/common" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sc-foo = { version = "4.0.0-dev", path = "../utils" } +sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-finality-grandpa = { version = "4.0.0-dev", path = "../../primitives/finality-grandpa" } thiserror = "1" unsigned-varint = { version = "0.6.0", features = [ diff --git a/client/network/src/on_demand_layer.rs b/client/network/src/on_demand_layer.rs index a2917a6e2cc76..5bac05c7aefad 100644 --- a/client/network/src/on_demand_layer.rs +++ b/client/network/src/on_demand_layer.rs @@ -27,7 +27,7 @@ use sc_client_api::{ RemoteChangesRequest, RemoteHeaderRequest, RemoteReadChildRequest, RemoteReadRequest, StorageProof, }; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_blockchain::Error as ClientError; use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; use std::{ diff --git a/client/network/src/service.rs b/client/network/src/service.rs index adb62d2833f47..6b2928510760c 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -70,7 +70,7 @@ use metrics::{Histogram, HistogramVec, MetricSources, Metrics}; use parking_lot::Mutex; use sc_consensus::{BlockImportError, BlockImportStatus, ImportQueue, Link}; use sc_peerset::PeersetHandle; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_runtime::traits::{Block as BlockT, NumberFor}; use std::{ borrow::Cow, diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index bf6556e4eb3f2..a7ff572e9b0b7 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -29,7 +29,7 @@ sp-api = { version = "4.0.0-dev", path = "../../primitives/api" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } sp-offchain = { version = "4.0.0-dev", path = "../../primitives/offchain" } sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sc-foo = { version = "4.0.0-dev", path = "../utils" } +sc-utils = { version = "4.0.0-dev", path = "../utils" } threadpool = "1.7" [target.'cfg(not(target_os = "unknown"))'.dependencies] diff --git a/client/offchain/src/api/http.rs b/client/offchain/src/api/http.rs index 367daf3e7f3e7..ce9fb298d1b0c 100644 --- a/client/offchain/src/api/http.rs +++ b/client/offchain/src/api/http.rs @@ -34,7 +34,7 @@ use futures::{channel::mpsc, future, prelude::*}; use hyper::{client, Body, Client as HyperClient}; use hyper_rustls::HttpsConnector; use log::error; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_core::offchain::{HttpError, HttpRequestId, HttpRequestStatus, Timestamp}; use std::{ convert::TryFrom, diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index 4b954c227b97f..9e83ede675d02 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -17,7 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] futures = "0.3.9" libp2p = { version = "0.39.1", default-features = false } -sc-foo = { version = "4.0.0-dev", path = "../utils"} +sc-utils = { version = "4.0.0-dev", path = "../utils"} log = "0.4.8" serde_json = "1.0.41" wasm-timer = "0.2" diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 747824d17e900..ecaa1d9f576f1 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -36,7 +36,7 @@ mod peersstate; use futures::prelude::*; use log::{debug, error, trace}; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use serde_json::json; use std::{ collections::{HashMap, HashSet, VecDeque}, diff --git a/client/rpc/Cargo.toml b/client/rpc/Cargo.toml index 173588dc80546..9957bc999a8bb 100644 --- a/client/rpc/Cargo.toml +++ b/client/rpc/Cargo.toml @@ -27,7 +27,7 @@ serde_json = "1.0.41" sp-session = { version = "4.0.0-dev", path = "../../primitives/session" } sp-offchain = { version = "4.0.0-dev", path = "../../primitives/offchain" } sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sc-foo = { version = "4.0.0-dev", path = "../utils" } +sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-rpc = { version = "4.0.0-dev", path = "../../primitives/rpc" } sp-keystore = { version = "0.10.0-dev", path = "../../primitives/keystore" } sc-chain-spec = { version = "4.0.0-dev", path = "../chain-spec" } diff --git a/client/rpc/src/system/mod.rs b/client/rpc/src/system/mod.rs index 52ed2bd9cd60a..f99994e41a1be 100644 --- a/client/rpc/src/system/mod.rs +++ b/client/rpc/src/system/mod.rs @@ -22,7 +22,7 @@ use self::error::Result; use futures::{channel::oneshot, FutureExt}; use sc_rpc_api::{DenyUnsafe, Receiver}; use sc_tracing::logging; -use sc_foo::mpsc::TracingUnboundedSender; +use sc_utils::mpsc::TracingUnboundedSender; use sp_runtime::traits::{self, Header as HeaderT}; pub use self::{ diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs index dbb0ffbe408ac..cc794b884f066 100644 --- a/client/rpc/src/system/tests.rs +++ b/client/rpc/src/system/tests.rs @@ -21,7 +21,7 @@ use super::*; use assert_matches::assert_matches; use futures::{executor, prelude::*}; use sc_network::{self, config::Role, PeerId}; -use sc_foo::mpsc::tracing_unbounded; +use sc_utils::mpsc::tracing_unbounded; use std::{ env, io::{BufRead, BufReader, Write}, diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index a645eae1a2842..b79e95fbb0912 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -39,7 +39,7 @@ sc-keystore = { version = "4.0.0-dev", path = "../keystore" } sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } sp-trie = { version = "4.0.0-dev", path = "../../primitives/trie" } sp-externalities = { version = "0.10.0-dev", path = "../../primitives/externalities" } -sc-foo = { version = "4.0.0-dev", path = "../utils" } +sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-version = { version = "4.0.0-dev", path = "../../primitives/version" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 8a1f9adcd4144..f0c037aee232f 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -49,7 +49,7 @@ use sc_network::{ }; use sc_telemetry::{telemetry, ConnectionMessage, Telemetry, TelemetryHandle, SUBSTRATE_INFO}; use sc_transaction_pool_api::MaintainedTransactionPool; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_api::{CallApiAt, ProvideRuntimeApi}; use sp_blockchain::{HeaderBackend, HeaderMetadata}; use sp_consensus::block_validation::{ diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index cacfd23f23212..f7d93d036a3fa 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -61,7 +61,7 @@ use sp_blockchain::{ }; use sp_consensus::{BlockOrigin, BlockStatus, Error as ConsensusError}; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_core::{ convert_hash, storage::{well_known_keys, ChildInfo, PrefixedStorageKey, StorageData, StorageKey}, diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index f19c574a65e77..ede6f01a4539e 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -41,7 +41,7 @@ use futures::{stream, Future, FutureExt, Stream, StreamExt}; use log::{debug, error, warn}; use parity_util_mem::MallocSizeOf; use sc_network::PeerId; -use sc_foo::mpsc::TracingUnboundedReceiver; +use sc_utils::mpsc::TracingUnboundedReceiver; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Header as HeaderT}, diff --git a/client/service/src/metrics.rs b/client/service/src/metrics.rs index 99cd1fe34d6aa..4d3c6df92fee7 100644 --- a/client/service/src/metrics.rs +++ b/client/service/src/metrics.rs @@ -25,7 +25,7 @@ use sc_client_api::{ClientInfo, UsageProvider}; use sc_network::{config::Role, NetworkService, NetworkStatus}; use sc_telemetry::{telemetry, TelemetryHandle, SUBSTRATE_INFO}; use sc_transaction_pool_api::{MaintainedTransactionPool, PoolStatus}; -use sc_foo::metrics::register_globals; +use sc_utils::metrics::register_globals; use sp_api::ProvideRuntimeApi; use sp_runtime::traits::{Block, NumberFor, SaturatedConversion, UniqueSaturatedInto}; use std::{ diff --git a/client/service/src/task_manager/mod.rs b/client/service/src/task_manager/mod.rs index d8613a0a90ea6..7842acdf0455a 100644 --- a/client/service/src/task_manager/mod.rs +++ b/client/service/src/task_manager/mod.rs @@ -33,7 +33,7 @@ use prometheus_endpoint::{ exponential_buckets, register, CounterVec, HistogramOpts, HistogramVec, Opts, PrometheusError, Registry, U64, }; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use std::{panic, pin::Pin, result::Result}; use tracing_futures::Instrument; diff --git a/client/transaction-pool/Cargo.toml b/client/transaction-pool/Cargo.toml index e0224b436dc3a..2184af819adf7 100644 --- a/client/transaction-pool/Cargo.toml +++ b/client/transaction-pool/Cargo.toml @@ -29,7 +29,7 @@ sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } sp-transaction-pool = { version = "4.0.0-dev", path = "../../primitives/transaction-pool" } sc-transaction-pool-api = { version = "4.0.0-dev", path = "./api" } sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" } -sc-foo = { version = "4.0.0-dev", path = "../utils" } +sc-utils = { version = "4.0.0-dev", path = "../utils" } serde = { version = "1.0.126", features = ["derive"] } linked-hash-map = "0.5.4" retain_mut = "0.1.3" diff --git a/client/transaction-pool/graph/Cargo.toml b/client/transaction-pool/graph/Cargo.toml index 57de7740ced44..b49cadc51c33c 100644 --- a/client/transaction-pool/graph/Cargo.toml +++ b/client/transaction-pool/graph/Cargo.toml @@ -20,7 +20,7 @@ log = "0.4.8" parking_lot = "0.11.1" serde = { version = "1.0.101", features = ["derive"] } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } -sc-foo = { version = "4.0.0-dev", path = "../../utils" } +sc-utils = { version = "4.0.0-dev", path = "../../utils" } sp-core = { version = "4.0.0-dev", path = "../../../primitives/core" } sp-runtime = { version = "4.0.0-dev", path = "../../../primitives/runtime" } sp-transaction-pool = { version = "4.0.0-dev", path = "../../../primitives/transaction-pool" } diff --git a/client/transaction-pool/src/graph/watcher.rs b/client/transaction-pool/src/graph/watcher.rs index eea5c604d0ea1..975ee6608886b 100644 --- a/client/transaction-pool/src/graph/watcher.rs +++ b/client/transaction-pool/src/graph/watcher.rs @@ -20,7 +20,7 @@ use futures::Stream; use sc_transaction_pool_api::TransactionStatus; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; /// Extrinsic watcher. /// diff --git a/client/transaction-pool/src/revalidation.rs b/client/transaction-pool/src/revalidation.rs index fa1072970734b..a8b2c1d32036a 100644 --- a/client/transaction-pool/src/revalidation.rs +++ b/client/transaction-pool/src/revalidation.rs @@ -25,7 +25,7 @@ use std::{ }; use crate::graph::{ChainApi, ExtrinsicHash, NumberFor, Pool, ValidatedTransaction}; -use sc_foo::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_runtime::{ generic::BlockId, traits::{SaturatedConversion, Zero}, diff --git a/client/utils/Cargo.toml b/client/utils/Cargo.toml index b99f43c5dedd8..99765dd501dd5 100644 --- a/client/utils/Cargo.toml +++ b/client/utils/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "sc-foo" +name = "sc-utils" version = "4.0.0-dev" authors = ["Parity Technologies "] edition = "2018" From 05e0550575e93974a88c97f4c53887b3d447277c Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Wed, 8 Sep 2021 10:46:21 -0300 Subject: [PATCH 12/17] more informative and readable code --- .../gitlab/check_polkadot_companion_build.sh | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index 5f56cc0013880..c107fc161b2b2 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -99,7 +99,7 @@ fi our_crates=() our_crates_source="git+https://github.com/paritytech/substrate" -load_our_crates() { +discover_our_crates() { local found cd "$substrate_dir" @@ -117,6 +117,8 @@ load_our_crates() { else our_crates+=("$crate") fi + # dependencies with {"source": null} are the ones we own, hence the getpath($p)==null in the jq + # script below done < <(cargo metadata --quiet --format-version=1 | "$jq" -r ' . as $in | paths | @@ -125,7 +127,7 @@ load_our_crates() { $in | getpath($path + ["name"]) ') } -load_our_crates +discover_our_crates match_their_crates() { local target_dir="$1" @@ -135,24 +137,23 @@ match_their_crates() { local target_name="$(basename "$target_dir")" local crates_not_found=() + local found - echo "$target_dir" - # output will be provided in the format: + # output will be consumed in the format: # crate # source # crate # ... local next="crate" - local found while IFS= read -r line; do case "$next" in crate) - crate="$line" next="source" + crate="$line" ;; source) + next="crate" if [ "$line" == "$our_crates_source" ] || [[ "$line" == "$our_crates_source?"* ]]; then - echo "$line" for our_crate in "${our_crates[@]}"; do if [ "$our_crate" == "$crate" ]; then found=true @@ -176,8 +177,6 @@ match_their_crates() { fi fi fi - - next="crate" ;; *) echo "ERROR: Unknown state $next" @@ -194,8 +193,9 @@ match_their_crates() { ') if [ "${crates_not_found[@]}" ]; then - echo "Errors during crate matching" + echo -e "Errors during crate matching\n" printf "Failed to detect our crate \"%s\" referenced in $target_name\n" "${crates_not_found[@]}" + echo -e "\nNote: this error generally happens if you have deleted or renamed a crate and did not update it in $target_name" exit 1 fi } From e06e297157cbcee4526f8c8f227d3770b83796aa Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 10 Sep 2021 06:24:24 -0300 Subject: [PATCH 13/17] offer suggestion on how to solve renamed and deleted crates' issues --- .maintain/gitlab/check_polkadot_companion_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index c107fc161b2b2..1c6e8c866cf9a 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -195,7 +195,7 @@ match_their_crates() { if [ "${crates_not_found[@]}" ]; then echo -e "Errors during crate matching\n" printf "Failed to detect our crate \"%s\" referenced in $target_name\n" "${crates_not_found[@]}" - echo -e "\nNote: this error generally happens if you have deleted or renamed a crate and did not update it in $target_name" + echo -e "Note: this error generally happens if you have deleted or renamed a crate and did not update it in $target_name. Consider opening a companion pull request on $target_name and referencing it in this pull request's description like:\n$target_name companion: [your companion PR here]" exit 1 fi } From 4220b05b49bf4bc38a45811d25f07fecdfa1ca96 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 10 Sep 2021 06:24:46 -0300 Subject: [PATCH 14/17] use base jq from Ubuntu --- .maintain/gitlab/check_polkadot_companion_build.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index 1c6e8c866cf9a..16302247ccf83 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -37,17 +37,6 @@ from this pull request. otherwise, it will uses master instead EOT -# FIXME: update jq in the CI file - -jq="$PWD/jq16" -curl -sqL https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o "$jq" -chmod +x "$jq" -jq_sha256sum="af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 $jq" -if [ "$(sha256sum "$jq")" != "$jq_sha256sum" ]; then - echo "ERROR: jq sha256sum mismatch" - exit 1 -fi - # Set the user name and email to make merging work git config --global user.name 'CI system' git config --global user.email '<>' From 9cb4b8a4fe98102d1fac145d34182d9f95b989ea Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 10 Sep 2021 06:53:06 -0300 Subject: [PATCH 15/17] remove dependency https://github.com/paritytech/substrate/pull/9721#discussion_r704421671 --- .gitlab-ci.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bfdb5bb3d092a..cc1f626e27d48 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -188,13 +188,13 @@ default: file: false AWX_TOKEN: vault: cicd/gitlab/$CI_PROJECT_PATH/AWX_TOKEN@kv - file: false + file: false CRATES_TOKEN: vault: cicd/gitlab/$CI_PROJECT_PATH/CRATES_TOKEN@kv file: false DOCKER_CHAOS_TOKEN: vault: cicd/gitlab/$CI_PROJECT_PATH/DOCKER_CHAOS_TOKEN@kv - file: false + file: false DOCKER_CHAOS_USER: vault: cicd/gitlab/$CI_PROJECT_PATH/DOCKER_CHAOS_USER@kv file: false @@ -547,9 +547,6 @@ check-polkadot-companion-build: <<: *docker-env <<: *test-refs-no-trigger <<: *vault-secrets - needs: - - job: test-linux-stable-int - artifacts: false script: - ./.maintain/gitlab/check_polkadot_companion_build.sh after_script: @@ -819,7 +816,7 @@ deploy-prometheus-alerting-rules: # Runs "quick" and "long" tests on nightly schedule and on commit / merge to master # A "quick" test is a smoke test where basic check-expect tests run by # checking values from metrics exposed by the app. -# A "long" test is the load testing where we send 50K transactions into the +# A "long" test is the load testing where we send 50K transactions into the # network and check if all completed successfully simnet-tests: stage: deploy From 7aca1eb7257422e6c230e04e5b44d91e1da6dff7 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 10 Sep 2021 09:13:12 -0300 Subject: [PATCH 16/17] make error handling more robust --- .../gitlab/check_polkadot_companion_build.sh | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index 16302247ccf83..f5d60107b15d9 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -9,7 +9,8 @@ # polkadot companion: paritytech/polkadot#567 # -set -euo pipefail +set -eu -o pipefail +shopt -s inherit_errexit github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls" # use github api v3 in order to access the data without authentication @@ -93,7 +94,11 @@ discover_our_crates() { cd "$substrate_dir" + # workaround for early exits not being detected in command substitution + # https://unix.stackexchange.com/questions/541969/nested-command-substitution-does-not-stop-a-script-on-a-failure-even-if-e-and-s + local last_line while IFS= read -r crate; do + last_line="$crate" # for avoiding duplicate entries for our_crate in "${our_crates[@]}"; do if [ "$crate" == "$our_crate" ]; then @@ -108,13 +113,17 @@ discover_our_crates() { fi # dependencies with {"source": null} are the ones we own, hence the getpath($p)==null in the jq # script below - done < <(cargo metadata --quiet --format-version=1 | "$jq" -r ' + done < <(cargo metadata --quiet --format-version=1 | jq -r ' . as $in | paths | select(.[-1]=="source" and . as $p | $in | getpath($p)==null) as $path | del($path[-1]) as $path | $in | getpath($path + ["name"]) ') + if [ -z "${last_line+_}" ]; then + echo "No lines were read for cargo metadata of $PWD (some error probably occurred)" + exit 1 + fi } discover_our_crates @@ -128,6 +137,9 @@ match_their_crates() { local crates_not_found=() local found + # workaround for early exits not being detected in command substitution + # https://unix.stackexchange.com/questions/541969/nested-command-substitution-does-not-stop-a-script-on-a-failure-even-if-e-and-s + local last_line # output will be consumed in the format: # crate # source @@ -135,6 +147,7 @@ match_their_crates() { # ... local next="crate" while IFS= read -r line; do + last_line="$line" case "$next" in crate) next="source" @@ -172,7 +185,7 @@ match_their_crates() { exit 1 ;; esac - done < <(cargo metadata --quiet --format-version=1 | "$jq" -r ' + done < <(cargo metadata --quiet --format-version=1 | jq -r ' . as $in | paths(select(type=="string")) | select(.[-1]=="source") as $source_path | @@ -180,6 +193,10 @@ match_their_crates() { [$in | getpath($path + ["name"]), getpath($path + ["source"])] | .[] ') + if [ -z "${last_line+_}" ]; then + echo "No lines were read for cargo metadata of $PWD (some error probably occurred)" + exit 1 + fi if [ "${crates_not_found[@]}" ]; then echo -e "Errors during crate matching\n" From b9e919b4423f500f481a379da00cd7c1c13c03ae Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Fri, 10 Sep 2021 10:29:14 -0300 Subject: [PATCH 17/17] newlines --- .maintain/gitlab/check_polkadot_companion_build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh index f5d60107b15d9..6cdf3f16ad299 100755 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/.maintain/gitlab/check_polkadot_companion_build.sh @@ -97,6 +97,7 @@ discover_our_crates() { # workaround for early exits not being detected in command substitution # https://unix.stackexchange.com/questions/541969/nested-command-substitution-does-not-stop-a-script-on-a-failure-even-if-e-and-s local last_line + while IFS= read -r crate; do last_line="$crate" # for avoiding duplicate entries @@ -140,6 +141,7 @@ match_their_crates() { # workaround for early exits not being detected in command substitution # https://unix.stackexchange.com/questions/541969/nested-command-substitution-does-not-stop-a-script-on-a-failure-even-if-e-and-s local last_line + # output will be consumed in the format: # crate # source @@ -201,7 +203,7 @@ match_their_crates() { if [ "${crates_not_found[@]}" ]; then echo -e "Errors during crate matching\n" printf "Failed to detect our crate \"%s\" referenced in $target_name\n" "${crates_not_found[@]}" - echo -e "Note: this error generally happens if you have deleted or renamed a crate and did not update it in $target_name. Consider opening a companion pull request on $target_name and referencing it in this pull request's description like:\n$target_name companion: [your companion PR here]" + echo -e "\nNote: this error generally happens if you have deleted or renamed a crate and did not update it in $target_name. Consider opening a companion pull request on $target_name and referencing it in this pull request's description like:\n$target_name companion: [your companion PR here]" exit 1 fi }