From a6955f7a9062dd8806a3bbcee0089f0bbacee0de Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 29 Apr 2021 21:50:48 +0200 Subject: [PATCH 001/161] doc: initial testing outline --- doc/testing.md | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 doc/testing.md diff --git a/doc/testing.md b/doc/testing.md new file mode 100644 index 000000000000..d476d0ad21bf --- /dev/null +++ b/doc/testing.md @@ -0,0 +1,96 @@ +# testing + +Automated testing is an essential tool to assure correctness. + +## Scopes + +The testing strategy for polkadot is 4-fold: + +### Unit testing + +Boring, small scale correctness tests of individual functions. + +### Integration tests + +One particular subsystem (subsystem under test) interacts with a +mocked overseer that is made to assert incoming and outgoing messages +of the subsystem under test. + +### System tests + +Launching small scale networks, with multiple adversarial nodes without any further tooling required. +This should include tests around the thresholds in order to evaluate the error handling once certain +assumed invariants fail. + +### Scale testing + +Launching many nodes with configurable network speed and node features in a cluster of nodes. +At this scale the [`simnet`][simnet] comes into play which launches a full cluster of nodes. +Asserts are made based on metrics. + +--- + +## Coverage + +Coverage gives a _hint_ of the actually covered source lines by tests and test applications. + +The state of the art is currently [tarpaulin][tarpaulin] which unfortunately yields a +lot false negatives (lines that are in fact covered, marked as uncovered, which leads to +lower coverage percentages). +Since late 2020 rust has gained [MIR based coverage tooling]( +https://blog.rust-lang.org/inside-rust/2020/11/12/source-based-code-coverage.html). + +```sh +# setup +rustup component add llvm-tools-preview +cargo install grcov miniserve + +# wasm is not happy with the instrumentation +export SKIP_BUILD_WASM=true +# the actully collected coverage data +export LLVM_PROFILE_FILE="llvmcoveragedata-%p-%m.profraw" +# required rustc flags +export RUSTFLAGS="-Zinstrument-coverage" +# assure target dir is clean +cargo clean +# build +cargo +nightly build +# run tests to get coverage data +cargo +nightly test --all +# create the report out of all the test binaries +grcov . --binary-path ./target/debug -s . -t html --branch --ignore-not-existing -o ./coverage/ +miniserve -r ./coverage +``` + +## Fuzzing + +Fuzzing is an approach to verify correctness against arbitrary or partially structured inputs. + +Currently implemented fuzzing targets: + +* `erasure-coding` +* `bridges/storage-proof` + +The tooling of choice here is `honggfuzz-rs` as it allows _fastest_ coverage according to "some paper" which is a positive feature when run as part of PRs. + +Fuzzing is generally not applicable for data secured by cryptographic hashes or signatures. Either the input has to be specifically crafted, such that the discarded input +percentage stays in an acceptable range. +System level fuzzing is hence simply not feasible due to the amount of state that is required. + +Other candidates to implement fuzzing are: + +* `rpc` +* ... + +## Writing small scope integration tests with preconfigured workers + +Requirements: + +* spawn nodes with preconfigured behaviours +* allow multiple types of configuration to be specified +* allow extensability via external crates +*... + +--- + +[simnet]: https://github.com/paritytech/simnet_scripts From 0cb612025913a51df39a0c0221aa4cb3cdaf4200 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 30 Apr 2021 11:51:11 +0200 Subject: [PATCH 002/161] impro --- doc/testing.md | 76 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 7 deletions(-) diff --git a/doc/testing.md b/doc/testing.md index d476d0ad21bf..462c2a223210 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -22,11 +22,11 @@ Launching small scale networks, with multiple adversarial nodes without any furt This should include tests around the thresholds in order to evaluate the error handling once certain assumed invariants fail. -### Scale testing +### Testing at scale Launching many nodes with configurable network speed and node features in a cluster of nodes. At this scale the [`simnet`][simnet] comes into play which launches a full cluster of nodes. -Asserts are made based on metrics. +Asserts are(?) made based on metrics. --- @@ -45,23 +45,41 @@ https://blog.rust-lang.org/inside-rust/2020/11/12/source-based-code-coverage.htm rustup component add llvm-tools-preview cargo install grcov miniserve +export CARGO_INCREMENTAL=0 # wasm is not happy with the instrumentation export SKIP_BUILD_WASM=true +export BUILD_DUMMY_WASM_BINARY=true # the actully collected coverage data export LLVM_PROFILE_FILE="llvmcoveragedata-%p-%m.profraw" -# required rustc flags +# build wasm without instrumentation +export WASM_TARGET_DIRECTORY=/tmp/wasm +cargo +nightly build +# required rust flags export RUSTFLAGS="-Zinstrument-coverage" # assure target dir is clean cargo clean -# build -cargo +nightly build # run tests to get coverage data cargo +nightly test --all -# create the report out of all the test binaries + +# create the *html* report out of all the test binaries +# mostly useful for local inspection grcov . --binary-path ./target/debug -s . -t html --branch --ignore-not-existing -o ./coverage/ miniserve -r ./coverage + +# create a *codecov* compatible report +grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info ``` +The test coverage in `lcov` can the be published to . + +```sh +bash <(curl -s https://codecov.io/bash) -f lcov.info +``` + +or just printed as part of the PR using a gh action i.e. [jest-lcov-reporter](https://github.com/marketplace/actions/jest-lcov-reporter). + +For full examples on how to use [grcov /w polkadot specifics see the gh repo](https://github.com/mozilla/grcov#coverallscodecov-output). + ## Fuzzing Fuzzing is an approach to verify correctness against arbitrary or partially structured inputs. @@ -89,8 +107,52 @@ Requirements: * spawn nodes with preconfigured behaviours * allow multiple types of configuration to be specified * allow extensability via external crates -*... +* ... --- + +## Implementation of different behaviour strain nodes. + +### Goals + +The main goals are is to allow creating a test node which +exhibits a certain behaviour by utilizing a subset of _wrapped_ or _replaced_ subsystems easily. +The runtime must not matter at all for these tests and should be simplistic. +The execution must be fast, this mostly means to assure a close to zero network latency as +well as shorting the block time and epoch times down to a few `100ms` and a few dozend blocks per epoch. + +### Approach + +`AllSubsystems` is an intermediate mocking type. As such it is a prime target for modification. +`AllSubsystemsGen` is a proc-macro that should be extended as needed for per subsystem specific +logic that would otherwise cause significant boilerplate additions. + +There are common patterns, where i.e. a subsystem produces garbage, or does not produce any output. +These most be provided by default for re-usability. Another option would be a `CopyCat` node that +picks one other node and just repeats whatever the initial node does. These are just _ideas_ +and might not prove viable or yield signifcant outcomes. + +### Impl + +```rust +launch_integration_testcase!{ +"TestRuntime" => +"Alice": SubsystemsAll, +"Bob": SubsystemsAll, +"Charles": Default, +"David": "Bob", +"Eve": "Bob", +} +``` + +> TODO format sucks, revise, how does cli interact with it? Do we want configurable profiles? + +The coordination of multiple subsystems across nodes must be made possible via +a side-channel. That means those nodes most be able to prepare attacks that +are collaborative based on each others actions. + +> TODO + + [simnet]: https://github.com/paritytech/simnet_scripts From c4c8b7931fee2af6a6e8e3035c90420d97ab3779 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 4 May 2021 21:16:44 +0200 Subject: [PATCH 003/161] fix: avoid underflow in some circumstances --- node/metered-channel/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/metered-channel/src/lib.rs b/node/metered-channel/src/lib.rs index e2fc0d84b50f..01c3e95311f0 100644 --- a/node/metered-channel/src/lib.rs +++ b/node/metered-channel/src/lib.rs @@ -59,11 +59,11 @@ impl Meter { } fn note_sent(&self) { - self.sent.fetch_add(1, Ordering::Relaxed); + self.sent.fetch_add(1, Ordering::Release); } fn retract_sent(&self) { - self.sent.fetch_sub(1, Ordering::Relaxed); + self.sent.fetch_sub(1, Ordering::Acquire); } fn note_received(&self) { From cd8ae9f9fe950211232b5ae33304a8bb9977eb0a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 4 May 2021 21:44:49 +0200 Subject: [PATCH 004/161] addendums, quotes --- doc/testing.md | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/doc/testing.md b/doc/testing.md index 462c2a223210..eda3abaad845 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -15,6 +15,9 @@ Boring, small scale correctness tests of individual functions. One particular subsystem (subsystem under test) interacts with a mocked overseer that is made to assert incoming and outgoing messages of the subsystem under test. +This is largely present today, but has some fragmentation in the evolved +integration test implmentation. A proc-macro/macro+rules would allow +for more consistent implementation and structure. ### System tests @@ -22,6 +25,8 @@ Launching small scale networks, with multiple adversarial nodes without any furt This should include tests around the thresholds in order to evaluate the error handling once certain assumed invariants fail. +For this purpose based on `AllSubsystems` and proc-macro `AllSubsystemsGen`. + ### Testing at scale Launching many nodes with configurable network speed and node features in a cluster of nodes. @@ -35,8 +40,9 @@ Asserts are(?) made based on metrics. Coverage gives a _hint_ of the actually covered source lines by tests and test applications. The state of the art is currently [tarpaulin][tarpaulin] which unfortunately yields a -lot false negatives (lines that are in fact covered, marked as uncovered, which leads to -lower coverage percentages). +lot of false negatives. Lines that are in fact covered, marked as uncovered due to a mere linebreak in a statment can cause these artifacts. This leads to +lower coverage percentages than there actually is. + Since late 2020 rust has gained [MIR based coverage tooling]( https://blog.rust-lang.org/inside-rust/2020/11/12/source-based-code-coverage.html). @@ -57,7 +63,7 @@ cargo +nightly build # required rust flags export RUSTFLAGS="-Zinstrument-coverage" # assure target dir is clean -cargo clean +rm -r target/{debug,tests} # run tests to get coverage data cargo +nightly test --all @@ -100,6 +106,23 @@ Other candidates to implement fuzzing are: * `rpc` * ... +## Performance metrics + +There are various ways of performance metrics. + +* timing with `criterion` +* cache hits/misses w/ `iai` harness or `criterion-perf` +* `coz` a performance based compiler + +Most of them are standard tools to aid in the creation of statistical tests regarding change in time of certain unit tests. + +`coz` is meant for runtime. In our case, the system is far too large to yield a sufficent number of measurements in finite time. +An alternative approach could be to record incoming package streams per subsystem and store dumps of them, which in return could be replayed repeatedly at an +accelerated speed, with which enough metrics could be obtained to yield +information on which areas would improve the metrics. +This unfortunately will not yield much information, since most if not all of the subsystem code is linear based on the input to generate one or multiple output messages, it is unlikely to get any useful metrics without mocking a sufficiently large part of the other subsystem which overlaps with [#Integration tests] which is unfortunately not repeatable as of now. +As such the effort gain seems low and this is not pursued at the current time. + ## Writing small scope integration tests with preconfigured workers Requirements: @@ -146,13 +169,18 @@ launch_integration_testcase!{ } ``` -> TODO format sucks, revise, how does cli interact with it? Do we want configurable profiles? +> There was a suggestion about adding predefined profiles at runtime, +> which would be a nice thing to have for integration and re-usability +> with [simnet][simnet] rather than a tool that yields anything by itself. The coordination of multiple subsystems across nodes must be made possible via a side-channel. That means those nodes most be able to prepare attacks that -are collaborative based on each others actions. +are collaborative based on each others actions or sync deliberately based on a +attack leader. -> TODO +> There must be clear picture on what kind of +> scenerios we want to test this way which scenarios +> are out of scope and should be handled by [simnet][simnet]. [simnet]: https://github.com/paritytech/simnet_scripts From 98e1211bbca7ae3774e30bb29cc51206328bbe8c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 5 May 2021 10:44:10 +0200 Subject: [PATCH 005/161] bike-shedding --- doc/testing.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/testing.md b/doc/testing.md index eda3abaad845..7738e34fedb5 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -12,6 +12,10 @@ Boring, small scale correctness tests of individual functions. ### Integration tests +There are two variants of integration tests: + +#### Subsystem tests + One particular subsystem (subsystem under test) interacts with a mocked overseer that is made to assert incoming and outgoing messages of the subsystem under test. @@ -19,7 +23,7 @@ This is largely present today, but has some fragmentation in the evolved integration test implmentation. A proc-macro/macro+rules would allow for more consistent implementation and structure. -### System tests +#### Behavior tests Launching small scale networks, with multiple adversarial nodes without any further tooling required. This should include tests around the thresholds in order to evaluate the error handling once certain @@ -27,6 +31,8 @@ assumed invariants fail. For this purpose based on `AllSubsystems` and proc-macro `AllSubsystemsGen`. +This assumes a simplistic test runtime. + ### Testing at scale Launching many nodes with configurable network speed and node features in a cluster of nodes. From 6002b0d59d8aed41c191641024eba1e64405a2eb Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 1 Apr 2021 16:18:34 +0200 Subject: [PATCH 006/161] better var names --- .../src/requester/fetch_task/mod.rs | 8 ++++---- .../src/requester/fetch_task/tests.rs | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/node/network/availability-distribution/src/requester/fetch_task/mod.rs b/node/network/availability-distribution/src/requester/fetch_task/mod.rs index bdb51bdd2a1c..ecba462df175 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/mod.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/mod.rs @@ -263,7 +263,7 @@ impl RunningTask { async fn run_inner(mut self) { let mut bad_validators = Vec::new(); let mut succeeded = false; - let mut count: u32 = 0; + let mut retry_count: u32 = 0; let mut _span = self.span.child("fetch-task") .with_chunk_index(self.request.index.0) .with_relay_parent(self.relay_parent); @@ -271,10 +271,10 @@ impl RunningTask { while let Some(validator) = self.group.pop() { let _try_span = _span.child("try"); // Report retries: - if count > 0 { + if retry_count > 0 { self.metrics.on_retry(); } - count +=1; + retry_count +=1; // Send request: let resp = match self.do_request(&validator).await { @@ -319,7 +319,7 @@ impl RunningTask { _span.add_string_tag("success", "true"); break; } - _span.add_int_tag("tries", count as _); + _span.add_int_tag("tries", retry_count as _); if succeeded { self.metrics.on_fetch(SUCCEEDED); self.conclude(bad_validators).await; diff --git a/node/network/availability-distribution/src/requester/fetch_task/tests.rs b/node/network/availability-distribution/src/requester/fetch_task/tests.rs index db8790435b2c..ec0ebcb4182a 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/tests.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/tests.rs @@ -40,7 +40,8 @@ use super::*; fn task_can_be_canceled() { let (task, _rx) = get_test_running_task(); let (handle, kill) = oneshot::channel(); - std::mem::drop(handle); + drop(handle); + let running_task = task.run(kill); futures::pin_mut!(running_task); let waker = noop_waker(); From dbcd8f3f7feed1531aad08158da101eef4ed6b6a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 5 May 2021 14:30:41 +0200 Subject: [PATCH 007/161] git foo bar --- Cargo.lock | 432 ++++++++++++++++++++++++-------------- Cargo.toml | 6 + behavior-tests/Cargo.toml | 16 ++ behavior-tests/src/lib.rs | 57 +++++ 4 files changed, 359 insertions(+), 152 deletions(-) create mode 100644 behavior-tests/Cargo.toml create mode 100644 behavior-tests/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 542f6c9fb8b2..5b782ce633d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -517,6 +517,19 @@ dependencies = [ "sp-std", ] +[[package]] +name = "behavior-tests" +version = "0.1.0" +dependencies = [ + "env_logger 0.8.2", + "log", + "polkadot-service", + "sc-service", + "sp-core", + "test-runner", + "thiserror", +] + [[package]] name = "bincode" version = "1.3.1" @@ -704,6 +717,17 @@ dependencies = [ "sp-std", ] +[[package]] +name = "bp-kusama" +version = "0.1.0" +dependencies = [ + "bp-messages", + "bp-polkadot-core", + "bp-runtime", + "sp-api", + "sp-std", +] + [[package]] name = "bp-messages" version = "0.1.0" @@ -715,6 +739,17 @@ dependencies = [ "sp-std", ] +[[package]] +name = "bp-polkadot" +version = "0.1.0" +dependencies = [ + "bp-messages", + "bp-polkadot-core", + "bp-runtime", + "sp-api", + "sp-std", +] + [[package]] name = "bp-polkadot-core" version = "0.1.0" @@ -777,6 +812,21 @@ dependencies = [ "sp-std", ] +[[package]] +name = "bp-westend" +version = "0.1.0" +dependencies = [ + "bp-header-chain", + "bp-messages", + "bp-polkadot-core", + "bp-runtime", + "parity-scale-codec", + "sp-api", + "sp-runtime", + "sp-std", + "sp-version", +] + [[package]] name = "bp-wococo" version = "0.1.0" @@ -1842,7 +1892,7 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "fork-tree" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", ] @@ -1860,7 +1910,7 @@ dependencies = [ [[package]] name = "frame-benchmarking" version = "3.1.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -1879,7 +1929,7 @@ dependencies = [ [[package]] name = "frame-benchmarking-cli" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "Inflector", "chrono", @@ -1902,7 +1952,7 @@ dependencies = [ [[package]] name = "frame-election-provider-support" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -1915,7 +1965,7 @@ dependencies = [ [[package]] name = "frame-executive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -1930,7 +1980,7 @@ dependencies = [ [[package]] name = "frame-metadata" version = "13.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", "serde", @@ -1941,7 +1991,7 @@ dependencies = [ [[package]] name = "frame-support" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "bitflags", "frame-metadata", @@ -1967,7 +2017,7 @@ dependencies = [ [[package]] name = "frame-support-procedural" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "Inflector", "frame-support-procedural-tools", @@ -1979,7 +2029,7 @@ dependencies = [ [[package]] name = "frame-support-procedural-tools" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate 1.0.0", @@ -1991,7 +2041,7 @@ dependencies = [ [[package]] name = "frame-support-procedural-tools-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "proc-macro2", "quote", @@ -2001,7 +2051,7 @@ dependencies = [ [[package]] name = "frame-support-test" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-metadata", "frame-support", @@ -2021,7 +2071,7 @@ dependencies = [ [[package]] name = "frame-system" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "impl-trait-for-tuples", @@ -2038,7 +2088,7 @@ dependencies = [ [[package]] name = "frame-system-benchmarking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -2052,7 +2102,7 @@ dependencies = [ [[package]] name = "frame-system-rpc-runtime-api" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", "sp-api", @@ -2061,7 +2111,7 @@ dependencies = [ [[package]] name = "frame-try-runtime" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "parity-scale-codec", @@ -4447,7 +4497,7 @@ checksum = "13370dae44474229701bb69b90b4f4dca6404cb0357a2d50d635f1171dc3aa7b" [[package]] name = "pallet-authority-discovery" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -4462,7 +4512,7 @@ dependencies = [ [[package]] name = "pallet-authorship" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -4476,7 +4526,7 @@ dependencies = [ [[package]] name = "pallet-babe" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4499,7 +4549,7 @@ dependencies = [ [[package]] name = "pallet-balances" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4528,7 +4578,7 @@ dependencies = [ [[package]] name = "pallet-bounties" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4564,7 +4614,7 @@ dependencies = [ [[package]] name = "pallet-collective" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4580,7 +4630,7 @@ dependencies = [ [[package]] name = "pallet-democracy" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4595,7 +4645,7 @@ dependencies = [ [[package]] name = "pallet-election-provider-multi-phase" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -4615,7 +4665,7 @@ dependencies = [ [[package]] name = "pallet-elections-phragmen" version = "4.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4632,7 +4682,7 @@ dependencies = [ [[package]] name = "pallet-gilt" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4646,7 +4696,7 @@ dependencies = [ [[package]] name = "pallet-grandpa" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4667,7 +4717,7 @@ dependencies = [ [[package]] name = "pallet-identity" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "enumflags2", "frame-benchmarking", @@ -4682,7 +4732,7 @@ dependencies = [ [[package]] name = "pallet-im-online" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4701,7 +4751,7 @@ dependencies = [ [[package]] name = "pallet-indices" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4717,7 +4767,7 @@ dependencies = [ [[package]] name = "pallet-membership" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4732,7 +4782,7 @@ dependencies = [ [[package]] name = "pallet-mmr" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "ckb-merkle-mountain-range", "frame-benchmarking", @@ -4749,7 +4799,7 @@ dependencies = [ [[package]] name = "pallet-mmr-primitives" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -4765,7 +4815,7 @@ dependencies = [ [[package]] name = "pallet-mmr-rpc" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -4783,7 +4833,7 @@ dependencies = [ [[package]] name = "pallet-multisig" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4798,7 +4848,7 @@ dependencies = [ [[package]] name = "pallet-nicks" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -4811,7 +4861,7 @@ dependencies = [ [[package]] name = "pallet-offences" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -4827,7 +4877,7 @@ dependencies = [ [[package]] name = "pallet-offences-benchmarking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -4849,7 +4899,7 @@ dependencies = [ [[package]] name = "pallet-proxy" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4864,7 +4914,7 @@ dependencies = [ [[package]] name = "pallet-randomness-collective-flip" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -4877,7 +4927,7 @@ dependencies = [ [[package]] name = "pallet-recovery" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "enumflags2", "frame-support", @@ -4891,7 +4941,7 @@ dependencies = [ [[package]] name = "pallet-scheduler" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4906,7 +4956,7 @@ dependencies = [ [[package]] name = "pallet-session" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -4925,7 +4975,7 @@ dependencies = [ [[package]] name = "pallet-session-benchmarking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -4941,7 +4991,7 @@ dependencies = [ [[package]] name = "pallet-society" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -4954,7 +5004,7 @@ dependencies = [ [[package]] name = "pallet-staking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -4978,7 +5028,7 @@ dependencies = [ [[package]] name = "pallet-staking-reward-curve" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -4989,7 +5039,7 @@ dependencies = [ [[package]] name = "pallet-staking-reward-fn" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "log", "sp-arithmetic", @@ -4998,7 +5048,7 @@ dependencies = [ [[package]] name = "pallet-sudo" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -5011,7 +5061,7 @@ dependencies = [ [[package]] name = "pallet-timestamp" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -5029,7 +5079,7 @@ dependencies = [ [[package]] name = "pallet-tips" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -5044,7 +5094,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-support", "frame-system", @@ -5060,7 +5110,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -5077,7 +5127,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc-runtime-api" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "pallet-transaction-payment", "parity-scale-codec", @@ -5088,7 +5138,7 @@ dependencies = [ [[package]] name = "pallet-treasury" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -5104,7 +5154,7 @@ dependencies = [ [[package]] name = "pallet-utility" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-benchmarking", "frame-support", @@ -5119,7 +5169,7 @@ dependencies = [ [[package]] name = "pallet-vesting" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "enumflags2", "frame-benchmarking", @@ -7339,7 +7389,7 @@ dependencies = [ [[package]] name = "remote-externalities" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "env_logger 0.8.2", "hex-literal", @@ -7627,7 +7677,7 @@ dependencies = [ [[package]] name = "sc-authority-discovery" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "derive_more", @@ -7656,7 +7706,7 @@ dependencies = [ [[package]] name = "sc-basic-authorship" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "futures 0.3.14", "futures-timer 3.0.2", @@ -7679,7 +7729,7 @@ dependencies = [ [[package]] name = "sc-block-builder" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", "sc-client-api", @@ -7695,7 +7745,7 @@ dependencies = [ [[package]] name = "sc-chain-spec" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -7716,7 +7766,7 @@ dependencies = [ [[package]] name = "sc-chain-spec-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -7727,7 +7777,7 @@ dependencies = [ [[package]] name = "sc-cli" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "chrono", "fdlimit", @@ -7765,7 +7815,7 @@ dependencies = [ [[package]] name = "sc-client-api" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "derive_more", "fnv", @@ -7799,7 +7849,7 @@ dependencies = [ [[package]] name = "sc-client-db" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "blake2-rfc", "hash-db", @@ -7829,7 +7879,7 @@ dependencies = [ [[package]] name = "sc-consensus" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parking_lot 0.11.1", "sc-client-api", @@ -7841,7 +7891,7 @@ dependencies = [ [[package]] name = "sc-consensus-babe" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "derive_more", @@ -7887,7 +7937,7 @@ dependencies = [ [[package]] name = "sc-consensus-babe-rpc" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "derive_more", "futures 0.3.14", @@ -7911,7 +7961,7 @@ dependencies = [ [[package]] name = "sc-consensus-epochs" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "fork-tree", "parity-scale-codec", @@ -7921,10 +7971,45 @@ dependencies = [ "sp-runtime", ] +[[package]] +name = "sc-consensus-manual-seal" +version = "0.9.0" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" +dependencies = [ + "assert_matches", + "async-trait", + "derive_more", + "futures 0.3.14", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "log", + "parity-scale-codec", + "parking_lot 0.11.1", + "sc-client-api", + "sc-consensus-babe", + "sc-consensus-epochs", + "sc-transaction-pool", + "serde", + "sp-api", + "sp-blockchain", + "sp-consensus", + "sp-consensus-babe", + "sp-consensus-slots", + "sp-core", + "sp-inherents", + "sp-keyring", + "sp-keystore", + "sp-runtime", + "sp-timestamp", + "sp-transaction-pool", + "substrate-prometheus-endpoint", +] + [[package]] name = "sc-consensus-slots" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "futures 0.3.14", @@ -7952,7 +8037,7 @@ dependencies = [ [[package]] name = "sc-consensus-uncles" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "sc-client-api", "sp-authorship", @@ -7963,7 +8048,7 @@ dependencies = [ [[package]] name = "sc-executor" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "derive_more", "lazy_static", @@ -7992,7 +8077,7 @@ dependencies = [ [[package]] name = "sc-executor-common" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "derive_more", "parity-scale-codec", @@ -8010,7 +8095,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmi" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "log", "parity-scale-codec", @@ -8025,7 +8110,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmtime" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "log", "parity-scale-codec", @@ -8043,7 +8128,7 @@ dependencies = [ [[package]] name = "sc-finality-grandpa" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "derive_more", @@ -8083,7 +8168,7 @@ dependencies = [ [[package]] name = "sc-finality-grandpa-rpc" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "derive_more", "finality-grandpa", @@ -8107,7 +8192,7 @@ dependencies = [ [[package]] name = "sc-finality-grandpa-warp-sync" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "derive_more", "futures 0.3.14", @@ -8128,7 +8213,7 @@ dependencies = [ [[package]] name = "sc-informant" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "ansi_term 0.12.1", "futures 0.3.14", @@ -8146,7 +8231,7 @@ dependencies = [ [[package]] name = "sc-keystore" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "derive_more", @@ -8166,7 +8251,7 @@ dependencies = [ [[package]] name = "sc-light" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "hash-db", "lazy_static", @@ -8185,7 +8270,7 @@ dependencies = [ [[package]] name = "sc-network" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-std", "async-trait", @@ -8238,7 +8323,7 @@ dependencies = [ [[package]] name = "sc-network-gossip" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "futures 0.3.14", "futures-timer 3.0.2", @@ -8255,7 +8340,7 @@ dependencies = [ [[package]] name = "sc-offchain" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "bytes 0.5.6", "fnv", @@ -8283,7 +8368,7 @@ dependencies = [ [[package]] name = "sc-peerset" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "futures 0.3.14", "libp2p", @@ -8296,7 +8381,7 @@ dependencies = [ [[package]] name = "sc-proposer-metrics" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "log", "substrate-prometheus-endpoint", @@ -8305,7 +8390,7 @@ dependencies = [ [[package]] name = "sc-rpc" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "futures 0.3.14", "hash-db", @@ -8340,7 +8425,7 @@ dependencies = [ [[package]] name = "sc-rpc-api" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "derive_more", "futures 0.3.14", @@ -8365,7 +8450,7 @@ dependencies = [ [[package]] name = "sc-rpc-server" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "futures 0.1.29", "jsonrpc-core", @@ -8383,7 +8468,7 @@ dependencies = [ [[package]] name = "sc-service" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "directories", @@ -8447,7 +8532,7 @@ dependencies = [ [[package]] name = "sc-state-db" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "log", "parity-scale-codec", @@ -8462,7 +8547,7 @@ dependencies = [ [[package]] name = "sc-sync-state-rpc" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -8482,7 +8567,7 @@ dependencies = [ [[package]] name = "sc-telemetry" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "chrono", "futures 0.3.14", @@ -8502,7 +8587,7 @@ dependencies = [ [[package]] name = "sc-tracing" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "ansi_term 0.12.1", "atty", @@ -8539,7 +8624,7 @@ dependencies = [ [[package]] name = "sc-tracing-proc-macro" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -8550,7 +8635,7 @@ dependencies = [ [[package]] name = "sc-transaction-graph" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "derive_more", "futures 0.3.14", @@ -8572,7 +8657,7 @@ dependencies = [ [[package]] name = "sc-transaction-pool" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "futures 0.3.14", "futures-diagnose", @@ -9003,7 +9088,7 @@ dependencies = [ [[package]] name = "sp-allocator" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "log", "sp-core", @@ -9015,7 +9100,7 @@ dependencies = [ [[package]] name = "sp-api" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "hash-db", "log", @@ -9032,7 +9117,7 @@ dependencies = [ [[package]] name = "sp-api-proc-macro" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "blake2-rfc", "proc-macro-crate 1.0.0", @@ -9044,7 +9129,7 @@ dependencies = [ [[package]] name = "sp-application-crypto" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", "serde", @@ -9056,7 +9141,7 @@ dependencies = [ [[package]] name = "sp-arithmetic" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "integer-sqrt", "num-traits", @@ -9070,7 +9155,7 @@ dependencies = [ [[package]] name = "sp-authority-discovery" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", "sp-api", @@ -9082,7 +9167,7 @@ dependencies = [ [[package]] name = "sp-authorship" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "parity-scale-codec", @@ -9094,7 +9179,7 @@ dependencies = [ [[package]] name = "sp-block-builder" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", "sp-api", @@ -9106,7 +9191,7 @@ dependencies = [ [[package]] name = "sp-blockchain" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "futures 0.3.14", "log", @@ -9124,7 +9209,7 @@ dependencies = [ [[package]] name = "sp-chain-spec" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "serde", "serde_json", @@ -9133,7 +9218,7 @@ dependencies = [ [[package]] name = "sp-consensus" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "futures 0.3.14", @@ -9160,7 +9245,7 @@ dependencies = [ [[package]] name = "sp-consensus-babe" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "merlin", @@ -9182,7 +9267,7 @@ dependencies = [ [[package]] name = "sp-consensus-slots" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", "sp-arithmetic", @@ -9192,7 +9277,7 @@ dependencies = [ [[package]] name = "sp-consensus-vrf" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", "schnorrkel", @@ -9204,7 +9289,7 @@ dependencies = [ [[package]] name = "sp-core" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "base58", "blake2-rfc", @@ -9248,7 +9333,7 @@ dependencies = [ [[package]] name = "sp-database" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "kvdb", "parking_lot 0.11.1", @@ -9257,7 +9342,7 @@ dependencies = [ [[package]] name = "sp-debug-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "proc-macro2", "quote", @@ -9267,7 +9352,7 @@ dependencies = [ [[package]] name = "sp-externalities" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "environmental", "parity-scale-codec", @@ -9278,7 +9363,7 @@ dependencies = [ [[package]] name = "sp-finality-grandpa" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "finality-grandpa", "log", @@ -9295,7 +9380,7 @@ dependencies = [ [[package]] name = "sp-inherents" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "impl-trait-for-tuples", @@ -9309,7 +9394,7 @@ dependencies = [ [[package]] name = "sp-io" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "futures 0.3.14", "hash-db", @@ -9334,7 +9419,7 @@ dependencies = [ [[package]] name = "sp-keyring" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "lazy_static", "sp-core", @@ -9345,7 +9430,7 @@ dependencies = [ [[package]] name = "sp-keystore" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "derive_more", @@ -9362,7 +9447,7 @@ dependencies = [ [[package]] name = "sp-maybe-compressed-blob" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "ruzstd", "zstd", @@ -9371,7 +9456,7 @@ dependencies = [ [[package]] name = "sp-npos-elections" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", "serde", @@ -9384,7 +9469,7 @@ dependencies = [ [[package]] name = "sp-npos-elections-compact" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -9395,7 +9480,7 @@ dependencies = [ [[package]] name = "sp-offchain" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "sp-api", "sp-core", @@ -9405,7 +9490,7 @@ dependencies = [ [[package]] name = "sp-panic-handler" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "backtrace", ] @@ -9413,7 +9498,7 @@ dependencies = [ [[package]] name = "sp-rpc" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "rustc-hash", "serde", @@ -9424,7 +9509,7 @@ dependencies = [ [[package]] name = "sp-runtime" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "either", "hash256-std-hasher", @@ -9445,7 +9530,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -9462,7 +9547,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "Inflector", "proc-macro-crate 1.0.0", @@ -9474,7 +9559,7 @@ dependencies = [ [[package]] name = "sp-serializer" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "serde", "serde_json", @@ -9483,7 +9568,7 @@ dependencies = [ [[package]] name = "sp-session" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", "sp-api", @@ -9496,7 +9581,7 @@ dependencies = [ [[package]] name = "sp-staking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -9506,7 +9591,7 @@ dependencies = [ [[package]] name = "sp-state-machine" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "hash-db", "log", @@ -9529,12 +9614,12 @@ dependencies = [ [[package]] name = "sp-std" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" [[package]] name = "sp-storage" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "impl-serde", "parity-scale-codec", @@ -9547,7 +9632,7 @@ dependencies = [ [[package]] name = "sp-tasks" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "log", "sp-core", @@ -9560,7 +9645,7 @@ dependencies = [ [[package]] name = "sp-timestamp" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "futures-timer 3.0.2", @@ -9577,7 +9662,7 @@ dependencies = [ [[package]] name = "sp-tracing" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "erased-serde", "log", @@ -9595,7 +9680,7 @@ dependencies = [ [[package]] name = "sp-transaction-pool" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "derive_more", "futures 0.3.14", @@ -9611,7 +9696,7 @@ dependencies = [ [[package]] name = "sp-trie" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "hash-db", "memory-db", @@ -9625,7 +9710,7 @@ dependencies = [ [[package]] name = "sp-utils" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "futures 0.3.14", "futures-core", @@ -9637,7 +9722,7 @@ dependencies = [ [[package]] name = "sp-version" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "impl-serde", "parity-scale-codec", @@ -9650,7 +9735,7 @@ dependencies = [ [[package]] name = "sp-version-proc-macro" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "parity-scale-codec", "proc-macro-crate 1.0.0", @@ -9662,7 +9747,7 @@ dependencies = [ [[package]] name = "sp-wasm-interface" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -9819,7 +9904,7 @@ dependencies = [ [[package]] name = "substrate-browser-utils" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "chrono", "console_error_panic_hook", @@ -9845,7 +9930,7 @@ dependencies = [ [[package]] name = "substrate-build-script-utils" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "platforms", ] @@ -9853,7 +9938,7 @@ dependencies = [ [[package]] name = "substrate-frame-rpc-system" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-system-rpc-runtime-api", "futures 0.3.14", @@ -9876,7 +9961,7 @@ dependencies = [ [[package]] name = "substrate-prometheus-endpoint" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-std", "derive_more", @@ -9890,7 +9975,7 @@ dependencies = [ [[package]] name = "substrate-test-client" version = "2.0.1" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "async-trait", "futures 0.1.29", @@ -9919,7 +10004,7 @@ dependencies = [ [[package]] name = "substrate-test-utils" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "futures 0.3.14", "substrate-test-utils-derive", @@ -9929,7 +10014,7 @@ dependencies = [ [[package]] name = "substrate-test-utils-derive" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "proc-macro-crate 1.0.0", "quote", @@ -10087,6 +10172,48 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "test-runner" +version = "0.9.0" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" +dependencies = [ + "env_logger 0.7.1", + "frame-system", + "futures 0.3.14", + "jsonrpc-core", + "log", + "sc-basic-authorship", + "sc-cli", + "sc-client-api", + "sc-consensus-manual-seal", + "sc-executor", + "sc-informant", + "sc-network", + "sc-rpc", + "sc-rpc-server", + "sc-service", + "sc-transaction-graph", + "sc-transaction-pool", + "sp-api", + "sp-block-builder", + "sp-blockchain", + "sp-consensus", + "sp-core", + "sp-externalities", + "sp-inherents", + "sp-io", + "sp-keyring", + "sp-keystore", + "sp-offchain", + "sp-runtime", + "sp-runtime-interface", + "sp-session", + "sp-state-machine", + "sp-transaction-pool", + "sp-wasm-interface", + "tokio 0.2.21", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -10243,6 +10370,7 @@ dependencies = [ "libc", "memchr", "mio", + "mio-named-pipes", "mio-uds", "num_cpus", "pin-project-lite 0.1.7", @@ -10671,7 +10799,7 @@ checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" [[package]] name = "try-runtime-cli" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#a76cc733f4fe895d8be002ce13fdae0ca1ba555e" +source = "git+https://github.com/paritytech/substrate?branch=master#66c85aeb29fbf4786e76be328cf73241f72c2f18" dependencies = [ "frame-try-runtime", "log", diff --git a/Cargo.toml b/Cargo.toml index 6143ebf99da4..dbcc85373183 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,12 @@ tempfile = "3.2.0" [workspace] members = [ + "bridges/primitives/chain-kusama", + "bridges/primitives/chain-polkadot", + "bridges/primitives/chain-rococo", + "bridges/primitives/chain-westend", + "bridges/primitives/runtime", + "behavior-tests", "cli", "core-primitives", "erasure-coding", diff --git a/behavior-tests/Cargo.toml b/behavior-tests/Cargo.toml new file mode 100644 index 000000000000..0ad9f27a09f9 --- /dev/null +++ b/behavior-tests/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "behavior-tests" +version = "0.1.0" +authors = ["Bernhard Schuster "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +thiserror = "1" +polkadot-service = { path = "../node/service", default-features = false } +sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } +sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } +test-runner = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } +log = "0.4" +env_logger = "0.8" diff --git a/behavior-tests/src/lib.rs b/behavior-tests/src/lib.rs new file mode 100644 index 000000000000..cb4d53e0ec2c --- /dev/null +++ b/behavior-tests/src/lib.rs @@ -0,0 +1,57 @@ + +use polkadot_service as service; +use sc_service::{error::Error as ServiceError, Configuration, TaskManager}; +use sp_core::crypto::set_default_ss58_version; +use test_runner::default_config; +use log::info; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("foo")] + Foo +} + +pub type Result = std::result::Result; + +/// Launch a modified node with a given all subsystems instance +pub fn modded_node(subsystems: S, chain_spec: Box) -> Result<()> { + + set_default_ss58_version(chain_spec); + + let grandpa_pause = None; + + let jaeger_agent = None; + + let telemetry_worker = None; + + let mut runtime = tokio::runtime::Builder::new() + .threaded_scheduler() + .enable_all() + .build(); + + let config = default_config(); + let initialize = move |config: Configuration| async move { + let task_manager = + service::build_full( + config, + service::IsCollator::No, + grandpa_pause, + jaeger_agent, + telemetry_worker, + ) + .map(|full| full.task_manager)?; + Ok::<_, Error>(task_manager) + }; + + let mut task_manager = tokio.block_on(initialize(self.config))?; + let res = tokio.block_on(main(task_manager.future().fuse())); + tokio.block_on(task_manager.clean_shutdown()); +} + +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} From 08fae4598cfd73326e8bb037493c2e23bffd828a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 6 May 2021 19:11:14 +0200 Subject: [PATCH 008/161] snapshot WIP --- Cargo.lock | 14 + Cargo.toml | 2 + behavior-tests/Cargo.toml | 3 + behavior-tests/src/lib.rs | 59 ++- doc/testing.md | 2 +- node/network/availability-recovery/src/lib.rs | 10 +- node/overseer/overseer-gen/Cargo.toml | 18 + node/overseer/overseer-gen/src/lib.rs | 432 ++++++++++++++++++ node/overseer/overseer-gen/tests/ui/err-01.rs | 39 ++ node/overseer/overseer-gen/tests/ui/ok-01.rs | 133 ++++++ node/service/src/lib.rs | 9 +- 11 files changed, 699 insertions(+), 22 deletions(-) create mode 100644 node/overseer/overseer-gen/Cargo.toml create mode 100644 node/overseer/overseer-gen/src/lib.rs create mode 100644 node/overseer/overseer-gen/tests/ui/err-01.rs create mode 100644 node/overseer/overseer-gen/tests/ui/ok-01.rs diff --git a/Cargo.lock b/Cargo.lock index 5b782ce633d6..57f1218f1513 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -522,12 +522,15 @@ name = "behavior-tests" version = "0.1.0" dependencies = [ "env_logger 0.8.2", + "futures 0.3.14", "log", "polkadot-service", + "sc-chain-spec", "sc-service", "sp-core", "test-runner", "thiserror", + "tokio 0.2.21", ] [[package]] @@ -6300,6 +6303,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "polkadot-procmacro-overseer-gen" +version = "0.1.0" +dependencies = [ + "assert_matches", + "proc-macro2", + "quote", + "syn", + "trybuild", +] + [[package]] name = "polkadot-procmacro-overseer-subsystems-gen" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index dbcc85373183..61438e1b96b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,6 +71,8 @@ members = [ "node/network/collator-protocol", "node/network/gossip-support", "node/overseer", + "node/overseer/overseer-gen", + "node/overseer/subsystems-gen", "node/primitives", "node/service", "node/subsystem", diff --git a/behavior-tests/Cargo.toml b/behavior-tests/Cargo.toml index 0ad9f27a09f9..b9aee7671bf7 100644 --- a/behavior-tests/Cargo.toml +++ b/behavior-tests/Cargo.toml @@ -11,6 +11,9 @@ thiserror = "1" polkadot-service = { path = "../node/service", default-features = false } sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } +sc-chain-spec = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } test-runner = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } log = "0.4" env_logger = "0.8" +tokio = { version = "0.2.21", features = [ "signal", "rt-core", "rt-threaded", "blocking" ] } +futures = "0.3" diff --git a/behavior-tests/src/lib.rs b/behavior-tests/src/lib.rs index cb4d53e0ec2c..8d35997adb91 100644 --- a/behavior-tests/src/lib.rs +++ b/behavior-tests/src/lib.rs @@ -1,22 +1,48 @@ use polkadot_service as service; -use sc_service::{error::Error as ServiceError, Configuration, TaskManager}; +use sc_service::{error::Error as ServiceError, Configuration, TaskManager, TaskExecutor}; use sp_core::crypto::set_default_ss58_version; use test_runner::default_config; use log::info; +use sc_chain_spec::ChainSpec; +use futures::future::FutureExt; #[derive(thiserror::Error, Debug)] pub enum Error { - #[error("foo")] - Foo + #[error(transparent)] + SubstrateService(#[from] sc_service::Error), + + + #[error(transparent)] + PolkadotService(#[from] polkadot_service::Error), + + #[error(transparent)] + Io(#[from]std::io::Error) } -pub type Result = std::result::Result; +pub type Result = std::result::Result; + + +macro_rules! behavior_testcase { + ($name:ident => (&name:literal : $x:ty)* ) => { + let _ = env_logger::Builder::new().is_test(true).try_init(); + + + for + }; +} + +/// ```rust +bahvior_test!(alpha_omega => { + "Alice" +} +/// ``` + /// Launch a modified node with a given all subsystems instance -pub fn modded_node(subsystems: S, chain_spec: Box) -> Result<()> { +pub fn modded_node(subsystemselection: Sel, chain_spec: Box) -> Result<()> { - set_default_ss58_version(chain_spec); + // set_default_ss58_version(&chain_spec); let grandpa_pause = None; @@ -24,12 +50,19 @@ pub fn modded_node(subsystems: S, chain_spec: Box) -> Result<( let telemetry_worker = None; - let mut runtime = tokio::runtime::Builder::new() + let mut tokio_rt = tokio::runtime::Builder::new() .threaded_scheduler() .enable_all() - .build(); + .build()?; + + + let handle = tokio_rt.handle().clone(); + let task_executor: TaskExecutor = (move |future, _task_type| { + handle.spawn(future).map(|_| ()) + }).into(); + - let config = default_config(); + let config = default_config(task_executor, chain_spec); let initialize = move |config: Configuration| async move { let task_manager = service::build_full( @@ -43,9 +76,11 @@ pub fn modded_node(subsystems: S, chain_spec: Box) -> Result<( Ok::<_, Error>(task_manager) }; - let mut task_manager = tokio.block_on(initialize(self.config))?; - let res = tokio.block_on(main(task_manager.future().fuse())); - tokio.block_on(task_manager.clean_shutdown()); + let mut task_manager = tokio_rt.block_on(initialize(config))?; + // futures::pin_mut!(task_manager); + let res = tokio_rt.block_on(task_manager.future().fuse()); + tokio_rt.block_on(task_manager.clean_shutdown()); + Ok(()) } #[cfg(test)] diff --git a/doc/testing.md b/doc/testing.md index 7738e34fedb5..aff12c493080 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -165,7 +165,7 @@ and might not prove viable or yield signifcant outcomes. ### Impl ```rust -launch_integration_testcase!{ +behavior_testcase!{ "TestRuntime" => "Alice": SubsystemsAll, "Bob": SubsystemsAll, diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index f8eab8689a95..6c54b703e3ee 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -49,6 +49,7 @@ use polkadot_node_network_protocol::{ request::RequestError, }, }; +use polkadot_node_subsystem_util::metrics::Metrics; use polkadot_node_subsystem_util::request_session_info; use polkadot_erasure_coding::{branches, branch_hash, recovery_threshold, obtain_chunks_v1}; mod error; @@ -67,6 +68,7 @@ const LRU_SIZE: usize = 16; /// The Availability Recovery Subsystem. pub struct AvailabilityRecoverySubsystem { fast_path: bool, + metrics: Metrics, } struct RequestFromBackersPhase { @@ -742,13 +744,13 @@ async fn query_full_data( impl AvailabilityRecoverySubsystem { /// Create a new instance of `AvailabilityRecoverySubsystem` which starts with a fast path to request data from backers. - pub fn with_fast_path() -> Self { - Self { fast_path: true } + pub fn with_fast_path(metrics: Metrics) -> Self { + Self { fast_path: true, metrics } } /// Create a new instance of `AvailabilityRecoverySubsystem` which requests only chunks - pub fn with_chunks_only() -> Self { - Self { fast_path: false } + pub fn with_chunks_only(metrics: Metrics) -> Self { + Self { fast_path: false, metrics } } async fn run( diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml new file mode 100644 index 000000000000..e54533e9904c --- /dev/null +++ b/node/overseer/overseer-gen/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "polkadot-procmacro-overseer-gen" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" +description = "Generate an overseer including builder pattern and message wrapper from a single struct." + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0.60", features = ["full", "extra-traits"] } +quote = "1.0.9" +proc-macro2 = "1.0.24" +assert_matches = "1.5.0" + +[dev-dependencies] +trybuild = "1.0.41" diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs new file mode 100644 index 000000000000..922b094c2917 --- /dev/null +++ b/node/overseer/overseer-gen/src/lib.rs @@ -0,0 +1,432 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use std::collections::HashSet; +use proc_macro2::{Span, TokenStream}; +use quote::quote; + +use syn::{Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, parse2}; +use syn::FieldsNamed; +use syn::AttrStyle; +use syn::Field; +use syn::Variant; +use quote::ToTokens; + +#[proc_macro_attribute] +pub fn overlord(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream { + let attr: TokenStream = attr.into(); + let item: TokenStream = item.into(); + impl_overseer_gen(attr, item).unwrap_or_else(|err| err.to_compile_error()).into() +} + +#[derive(Clone)] +struct SubSysField { + /// Name of the field. + name: Ident, + /// Generate generic type name for the `AllSubsystems` type. + generic: Ident, + /// Type of the subsystem. + ty: Ident, + /// Type to be consumed by the subsystem. + consumes: Vec, +} + +fn try_type_to_ident(ty: Type, span: Span) -> Result { + match ty { + Type::Path(path) => path.path.get_ident().cloned() + .ok_or_else(|| { + Error::new(span, "Expected an identifier, but got a path.") + }), + _ => { + Err(Error::new(span, "Type must be a path expression.")) + } + } +} + +struct AttrArgs { + wrapper_enum_name: Ident, + signal_capacity: usize, + message_capacity: usize, +} + +fn parse_attr(_attr: TokenStream) -> Result { + Ok(AttrArgs { + wrapper_enum_name: Ident::new("AllMessages", Span::call_site()), + signal_capacity: 64usize, + message_capacity: 1024usize, + }) +} + +use syn::Token; +use syn::punctuated::Punctuated; +use syn::parse::Parse; +use syn::token::Paren; + +struct SubSystemTag { + attrs: Vec, + paren_token: Paren, + consumes: Punctuated, +} + +impl Parse for SubSystemTag { + fn parse(input: syn::parse::ParseStream) -> Result { + let content; + Ok(Self { + attrs: Attribute::parse_outer(input)?, + paren_token: syn::parenthesized!(content in input), + consumes: content.parse_terminated(Ident::parse)?, + }) + } +} + +/// Creates a list of generic identifiers used for the subsystems +fn parse_overseer_struct_field(baggage_generics: HashSet, fields: FieldsNamed) -> Result<(Vec, Vec)> { + let n = fields.named.len(); + let mut subsystems = Vec::with_capacity(n); + let mut baggage = Vec::with_capacity(n); + for (idx, Field { attrs, vis, ident, ty, .. }) in fields + .named + .into_iter() + .enumerate() { + + let mut consumes = attrs.iter().filter(|attr| { + attr.style == AttrStyle::Outer + }).filter_map(|attr| { + attr.path.get_ident().filter(|ident| { + *ident == "subsystem" + }).map(|_ident| { + let tokens = attr.tokens.clone(); + tokens + }) + }); + + let mut consumes_idents = Vec::with_capacity(attrs.len()); + for tokens in consumes { + let variant = syn::parse2::(dbg!(tokens))?; + consumes_idents.extend(variant.consumes.into_iter()); + } + + let ident = ident.unwrap(); + if !consumes_idents.is_empty() { + subsystems.push(SubSysField { + name: ident, + generic: Ident::new(format!("Sub{}", idx).as_str(), Span::call_site()), + ty: try_type_to_ident(ty, Span::call_site())?, + consumes: consumes_idents, + }); + } else { + baggage.push(BaggageField { + field_name: ident, + field_ty: try_type_to_ident(ty, Span::call_site())?, + generic: false,// XXX FIXME FIXME FIXME + }); + } + } + Ok((subsystems, baggage)) +} + +/// Add a feature to replace one subsystem of the subsystems type. +// fn impl_replace_subsystem() -> Result { +// let orig_generics = vec![]; +// let added_generics = vec![]; +// let added_generics_modded = vec![]; +// let keeper = vec![]; +// let overseer = Ident::new("Xxxx", Span::call_site()); +// let msg = "Xyz"; +// let x = quote!{ +// impl #orig_generics #overseer #orig_generics { +// #[doc = #msg] +// pub fn #fname < NEW > (self, replacement: NEW) -> #strukt_ty #modified_generics { +// #strukt_ty :: #modified_generics { +// #replacable_item: replacement, +// #( +// #keeper: self.#keeper, +// )* +// } +// } +// } +// }; +// Ok(x) +// } + +/// Fields that are _not_ subsystems. +struct BaggageField { + field_name: Ident, + field_ty: Ident, + generic: bool, +} + +/// Implement a builder pattern. +fn impl_builder(name: Ident, baggage: &[BaggageField], subsystems: &[SubSysField]) -> Result { + let builder = Ident::new((name.to_string() + "Builder").as_str(), Span::call_site()); + + let overseer = name.clone(); + let mut orig_generic_ty = &baggage.iter().filter(|b| b.generic).map(|b| b.field_ty.clone()).collect::>(); + let mut orig_generic_name = &baggage.iter().filter(|b| b.generic).map(|b| b.field_name.clone()).collect::>(); + + let mut field_name = &subsystems.iter().map(|x| x.name.clone() ).collect::>(); + let mut field_ty = &subsystems.iter().map(|x| x.ty.clone()).collect::>(); + + // FIXME + let subsystem_generics = Ident::new("FIXME", Span::call_site()); + + let x = quote!{ + + impl #name { + fn builder() -> #builder { + #builder :: default() + } + } + + struct #builder { + #( + #field_name : ::std::option::Option< #field_ty >, + )* + } + + + impl #builder { + #( + fn #field_name (#field_name: #field_ty ) -> #builder { + self.#field_name = Some( #field_name ); + } + )* + } + + impl #builder { + + fn build< #( #orig_generic_name, )* >(mut self, + #( + #orig_generic_name : #orig_generic_ty, + )* + ) -> #overseer < + #( #orig_generic_ty , )*, + SubSystems< #subsystem_generics > + > { + #overseer { + #( + #field_name : self. #field_ty .unwrap(), + )* + } + } + } + }; + Ok(x) +} + +// fn impl_replacable_subsystem(x: Vec<_>) -> Result { +// let msg = "Generated by #[overlord] derive proc-macro."; +// let mut additive = TokenStream::new(); + +// // generate an impl of `fn replace_#name` +// for NameTyTup { field: replacable_item, ty: replacable_item_ty } in replacable_items { +// let keeper = all_fields.iter().filter(|ntt| ntt.field != replacable_item).map(|ntt| ntt.field.clone()); +// let strukt_ty = strukt_ty.clone(); +// let fname = Ident::new(&format!("replace_{}", replacable_item), span); +// // adjust the generics such that the appropriate member type is replaced +// let mut modified_generics = orig_generics.clone(); +// modified_generics.params = modified_generics.params.into_iter().map(|mut generic| { +// match generic { +// GenericParam::Type(ref mut param) => { +// param.eq_token = None; +// param.default = None; +// if match &replacable_item_ty { +// Type::Path(path) => +// path.path.get_ident().filter(|&ident| ident == ¶m.ident).is_some(), +// _ => false +// } { +// param.ident = Ident::new("NEW", span); +// } +// } +// _ => {} +// } +// generic +// }).collect(); + +// #[derive(Clone)] +// struct NameTyTup { +// field: Ident, +// ty: Type, +// } + +// additive.extend(quote! { +// impl #orig_generics #strukt_ty #orig_generics { +// #[doc = #msg] +// pub fn #fname < NEW > (self, replacement: NEW) -> #strukt_ty #modified_generics { +// #strukt_ty :: #modified_generics { +// #replacable_item: replacement, +// #( +// #keeper: self.#keeper, +// )* +// } +// } +// } +// }); +// } + +// Ok(additive) +// } + + +fn impl_messages_wrapper_enum(messages_wrapper: Ident, subsystems: &[SubSysField]) -> Result { + let mut field_ty = subsystems.iter().map(|ssf| { + ssf.ty.clone() + }); + + let msg = "Generated message type wrapper"; + let x = quote!{ + #[doc = #msg] + #[derive(Debug, Clone)] + enum #messages_wrapper { + #( + #field_ty ( #field_ty ), + )* + } + }; + Ok(x) +} + + +fn impl_overseer_struct(overseer_name: Ident, subsystems: &[SubSysField], baggage: &[BaggageField]) -> Result { + let x = quote!{ + struct #overseer_name { + #( + #field_ty: field_name, + )* + + #( + #baggage_ty: baggage_field_name, + )* + } + }; + + x.extend(impl_builder(overseer_name, baggage, subsystems)?); + + Ok(x) +} + +fn impl_channels_out_struct(subsystems: &[SubSysField]) -> Result { + let mut field_name = subsystems.iter().map(|ssf| { + ssf.name.clone() + }); + let mut field_ty = subsystems.iter().map(|ssf| { + ssf.ty.clone() + }); + let x = quote!{ + #[derive(Debug)] + struct MessagePacket { + signals_received: usize, + message: T, + } + + fn make_packet(signals_received: usize, message: T) -> MessagePacket { + MessagePacket { + signals_received, + message, + } + } + + pub struct ChannelsOut { + #( + pub #field_name: ::metered::MeteredSender>, + )* + } + }; + Ok(x) +} + +fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result { + let args = parse_attr(attr)?; + let message_wrapper = args.wrapper_enum_name; + + let span = proc_macro2::Span::call_site(); + let ds = parse2::(orig.clone())?; + match ds.fields { + syn::Fields::Named(named) => { + let overseer_name = ds.ident.clone(); + + // collect the indepedentent subsystem generics + // which need to be carried along, there are the non-generated ones + let mut orig_generics = ds.generics; + + // remove default types + let mut baggage_generics = HashSet::with_capacity(orig_generics.params.len()); + orig_generics.params = orig_generics.params.into_iter().map(|mut generic| { + match generic { + GenericParam::Type(ref mut param) => { + baggage_generics.insert(param.ident.clone()); + param.eq_token = None; + param.default = None; + } + _ => {} + } + generic + }).collect(); + + let (subsystems, bags) = parse_overseer_struct_field(baggage_generics, named)?; + + let mut additive = impl_overseer_struct(&subsystems, &bags); + + additive.extend(impl_messages_wrapper_enum(message_wrapper, &subsystems[..])); + additive.extend(impl_channels_out_struct(&subsystems[..])); + // additive.extend(impl_replace_subsystem(&subsystems[..])); + // additive.extend(impl_(&subsystems[..])); + + + Ok(additive) + } + syn::Fields::Unit => Err(Error::new(span, "Must be a struct with named fields. Not an unit struct.")), + syn::Fields::Unnamed(_) => { + Err(Error::new(span, "Must be a struct with named fields. Not an unnamed fields struct.")) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic() { + let attr = quote! { + overloard(AllMessages, x, y) + }; + + let item = quote! { + pub struct Ooooh { + #[subsystem(Foo)] + sub0: FooSubsystem, + + #[subsystem(Bar | Twain)] + yyy: BarSubsystem, + + spawner: S, + metrics: Metrics, + } + }; + + let output = impl_overseer_gen(attr, item).expect("Simple example always works. qed"); + println!("//generated:"); + println!("{:#?}", output); + } + + #[test] + fn ui() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/err-*.rs"); + t.pass("tests/ui/ok-*.rs"); + } +} diff --git a/node/overseer/overseer-gen/tests/ui/err-01.rs b/node/overseer/overseer-gen/tests/ui/err-01.rs new file mode 100644 index 000000000000..209efbdc752f --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/err-01.rs @@ -0,0 +1,39 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_gen::overlord; + +struct X; + +struct Orange; + +#[derive(Default, Clone, Copy)] +struct AwesomeSubSys; + + +#[derive(Default, Clone, Copy)] +struct TequilaInABar; + + +#[derive(Clone, AllSubsystemsGen)] +#[overlord(Wrapper)] +struct Overseer { + #[subsystem(X)] + sub0: AwesomeSubSys, + + #[subsystem(Orange)] + shots_of: TequilaInABar + + other: Stuff, +} + +struct Spawner; + +fn main() { + let overseer = Overseer::>::builder() + .sub0(FooSubSys::default()) + .build(Spawner); + + // try to replace one subsystem with another that can not handle `X`. + // since it's missing the trait bound. + let overseer = overseer.replace_sub0(TequilaInABar::default()); +} diff --git a/node/overseer/overseer-gen/tests/ui/ok-01.rs b/node/overseer/overseer-gen/tests/ui/ok-01.rs new file mode 100644 index 000000000000..876895c83aa5 --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/ok-01.rs @@ -0,0 +1,133 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_gen::overlord; + + +#[derive(Debug, Clone)] +struct Foo { + pub x: String, +} + +#[derive(Debug, Clone, Copy)] +struct Bar; + +#[derive(Debug, Clone, Copy)] +enum Gap { + X, Y +} + +#[derive(Debug, Clone, Copy)] +struct Arg { + x: &'static [u8], +} + +struct SubsystemA { + +} + + +// generated +struct SubSystems {} + +impl SubSystemPick for SubSystems +where + A: OverseerSubsystem, + V: OverseerSubsystem, +{ + +} + +#[overlord(AllMessages, capacity=1024, signals=64)] +struct Overseer { + #[subsystem(Foo)] + subsystem_a: SubsystemA, + // generic A + + #[subsystem(Bar)] + foo_bar: SubsystemB, + // generic B + + #[subsystem(Gap)] + clown: Clown, + // generic C + + spawner: S, + metrics: Metrics, +} + +impl Overseer { + pub fn subsystems() -> &mut SubSystems { + self.subsystems + } +} + +impl Overseer { + fn builder(metrics: Metrics) -> Self { + OverseerBuilder { + + } + } +} + +const SUBSYSTEM_A: u16 = 0b0001; +const FOO_BAR: u16 = 0b0010; + +struct OverseerBuilder { + // # all members + subsystem_a: Option, + foo_bar: .., + clown: .., +} + +impl OverseerBuilder<0b0111> { + // # for all unset members + fn build(mut self, metrics: Metrics) -> Overseer where S: SubSystemPick { + Overseeer { + subsystems: SubSystems { + subsystem_a: self.subsystem_a.unwrap(), + foo_bar: self.foo_bar.unwrap(), + clown: self.clown.unwrap(), + }, + } + } +} + +impl OverseerBuilder<0u16> { + fn subsystem_a(mut self, subsystem_a: SubsystemA) -> OverseerBuilder { + self.subsystem_a = Some(subsystem_a); + self + } + + fn foo_bar(mut self, foo_bar: FooBar) -> OverseerBuilder { + // unimplemented!() + } + + fn clown(mut self) -> OverseerBuilder { + self.clown = Some(Clown::default()); + } +} + +impl OverseerBuilder { + fn foo_bar(mut self) -> OverseerBuilder { + // unimplemented!() + } + + fn clown(mut self) -> OverseerBuilder { + self.clown = Some(Clown::default()); + } +} + + +impl OverseerBuilder { + fn foo_bar(mut self) -> OverseerBuilder { + // unimplemented!() + } +} + +fn main() { + Overseer::builder() + .subsystem_a(a, b) + .foo_bar() + .build(metrics) + +} diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index fde49cacd831..ed0e8192c1aa 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -420,7 +420,7 @@ fn new_partial( } #[cfg(feature = "full-node")] -fn real_overseer( +fn real_overseer( leaves: impl IntoIterator, keystore: Arc, runtime_client: Arc, @@ -434,13 +434,12 @@ fn real_overseer( spawner: Spawner, is_collator: IsCollator, candidate_validation_config: CandidateValidationConfig, -) -> Result<(Overseer>, OverseerHandler), Error> +) -> Result<(Overseer, SubsystemSelection>, OverseerHandler), Error> where RuntimeClient: 'static + ProvideRuntimeApi + HeaderBackend + AuxStore, RuntimeClient::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, Spawner: 'static + SpawnNamed + Clone + Unpin, { - use polkadot_node_subsystem_util::metrics::Metrics; use polkadot_availability_distribution::AvailabilityDistributionSubsystem; use polkadot_node_core_av_store::AvailabilityStoreSubsystem; @@ -675,7 +674,7 @@ where /// This is an advanced feature and not recommended for general use. Generally, `build_full` is /// a better choice. #[cfg(feature = "full-node")] -pub fn new_full( +pub fn new_full( mut config: Configuration, is_collator: IsCollator, grandpa_pause: Option<(u32, u32)>, @@ -864,7 +863,7 @@ pub fn new_full( .and_then(move |k| authority_discovery_service.map(|a| (a, k))); let overseer_handler = if let Some((authority_discovery_service, keystore)) = maybe_params { - let (overseer, overseer_handler) = real_overseer( + let (overseer, overseer_handler) = real_overseer::<_,_,SubsystemSelection>( active_leaves, keystore, overseer_client.clone(), From 75ba380d0905ce5fd19d61bb9e1870e092d7b814 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 7 May 2021 17:39:49 +0200 Subject: [PATCH 009/161] more fixins --- node/overseer/overseer-gen/Cargo.toml | 3 +- node/overseer/overseer-gen/src/builder.rs | 79 ++++ node/overseer/overseer-gen/src/impls.rs | 291 +++++++++++++ node/overseer/overseer-gen/src/inc.rs | 20 + node/overseer/overseer-gen/src/inc/static.rs | 13 + node/overseer/overseer-gen/src/lib.rs | 387 +----------------- node/overseer/overseer-gen/src/replace.rs | 82 ++++ ...r-01.rs => err-01-replace-w-inadequate.rs} | 0 .../overseer-gen/tests/ui/err-02-enum.rs | 27 ++ node/overseer/overseer-gen/tests/ui/gen.rs | 80 ++++ .../overseer-gen/tests/ui/ok-02-boring.rs | 26 ++ node/overseer/overseer-gen/tests/ui/ok-02.rs | 26 ++ .../overseer-gen/tests/ui/ok-03-w-generic.rs | 30 ++ .../tests/ui/{ok-01.rs => xxxx.rs} | 0 14 files changed, 693 insertions(+), 371 deletions(-) create mode 100644 node/overseer/overseer-gen/src/builder.rs create mode 100644 node/overseer/overseer-gen/src/impls.rs create mode 100644 node/overseer/overseer-gen/src/inc.rs create mode 100644 node/overseer/overseer-gen/src/inc/static.rs create mode 100644 node/overseer/overseer-gen/src/replace.rs rename node/overseer/overseer-gen/tests/ui/{err-01.rs => err-01-replace-w-inadequate.rs} (100%) create mode 100644 node/overseer/overseer-gen/tests/ui/err-02-enum.rs create mode 100644 node/overseer/overseer-gen/tests/ui/gen.rs create mode 100644 node/overseer/overseer-gen/tests/ui/ok-02-boring.rs create mode 100644 node/overseer/overseer-gen/tests/ui/ok-02.rs create mode 100644 node/overseer/overseer-gen/tests/ui/ok-03-w-generic.rs rename node/overseer/overseer-gen/tests/ui/{ok-01.rs => xxxx.rs} (100%) diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index e54533e9904c..21f7b82224fd 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -11,7 +11,8 @@ proc-macro = true [dependencies] syn = { version = "1.0.60", features = ["full", "extra-traits"] } quote = "1.0.9" -proc-macro2 = "1.0.24" +proc-macro2 = "1.0.26" +proc-macro-crate = "1.0.0" assert_matches = "1.5.0" [dev-dependencies] diff --git a/node/overseer/overseer-gen/src/builder.rs b/node/overseer/overseer-gen/src/builder.rs new file mode 100644 index 000000000000..ec8931ecbb39 --- /dev/null +++ b/node/overseer/overseer-gen/src/builder.rs @@ -0,0 +1,79 @@ +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use std::collections::HashSet; + +use quote::ToTokens; +use syn::AttrStyle; +use syn::Field; +use syn::FieldsNamed; +use syn::Variant; +use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; + +use super::*; + +/// Implement a builder pattern. +pub(crate) fn impl_builder( + name: Ident, + subsystems: &[SubSysField], + baggage: &[BaggageField], +) -> Result { + let builder = Ident::new((name.to_string() + "Builder").as_str(), Span::call_site()); + + let overseer = name.clone(); + + let mut field_name = &subsystems.iter().map(|x| x.name.clone()).collect::>(); + let mut field_ty = &subsystems.iter().map(|x| x.generic.clone()).collect::>(); + + let mut baggage_generic_ty = &baggage.iter().filter(|b| b.generic).map(|b| b.field_ty.clone()).collect::>(); + + let mut baggage_name = &baggage.iter().map(|x| x.field_name.clone()).collect::>(); + let mut baggage_ty = &baggage.iter().map(|x| x.field_ty.clone()).collect::>(); + + let generics = quote! { + < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > + }; + + let where_clause = quote! { + where + #( #field_ty : Subsystem, )* + }; + + let x = quote! { + + impl #generics #name #generics #where_clause { + fn builder() -> #builder { + #builder :: default() + } + } + + struct #builder #generics { + #( + #field_name : ::std::option::Option< #field_ty >, + )* + #( + #baggage_name : ::std::option::Option< #baggage_name >, + )* + } + + impl #generics #builder #generics #where_clause { + #( + fn #field_name (mut self, new: #field_ty ) -> #builder { + self.#field_name = Some( new ); + self + } + )* + + fn build(mut self, ctx: Ctx) -> #overseer #generics { + #overseer :: #generics { + #( + #field_name : self. #field_name .unwrap(), + )* + #( + #baggage_name : self. #baggage_name .unwrap(), + )* + } + } + } + }; + Ok(x) +} diff --git a/node/overseer/overseer-gen/src/impls.rs b/node/overseer/overseer-gen/src/impls.rs new file mode 100644 index 000000000000..cf1a49475614 --- /dev/null +++ b/node/overseer/overseer-gen/src/impls.rs @@ -0,0 +1,291 @@ +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use std::collections::HashSet; + +use quote::ToTokens; +use syn::AttrStyle; +use syn::Field; +use syn::FieldsNamed; +use syn::Variant; +use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; + +use super::*; +#[derive(Clone)] +pub(crate) struct SubSysField { + /// Name of the field. + pub(crate) name: Ident, + /// Generate generic type name for the `AllSubsystems` type. + pub(crate) generic: Ident, + /// Type of the subsystem. + pub(crate) ty: Ident, + /// Type to be consumed by the subsystem. + pub(crate) consumes: Vec, +} + +fn try_type_to_ident(ty: Type, span: Span) -> Result { + match ty { + Type::Path(path) => { + path.path.get_ident().cloned().ok_or_else(|| Error::new(span, "Expected an identifier, but got a path.")) + } + _ => Err(Error::new(span, "Type must be a path expression.")), + } +} + +pub(crate) struct AttrArgs { + pub(crate) wrapper_enum_name: Ident, + pub(crate) signal_capacity: usize, + pub(crate) message_capacity: usize, +} + +fn parse_attr(_attr: TokenStream) -> Result { + Ok(AttrArgs { + wrapper_enum_name: Ident::new("AllMessages", Span::call_site()), + signal_capacity: 64usize, + message_capacity: 1024usize, + }) +} + +use syn::parse::Parse; +use syn::punctuated::Punctuated; +use syn::token::Paren; +use syn::Token; + +pub(crate) struct SubSystemTag { + pub(crate) attrs: Vec, + pub(crate) paren_token: Paren, + pub(crate) consumes: Punctuated, +} + +impl Parse for SubSystemTag { + fn parse(input: syn::parse::ParseStream) -> Result { + let content; + Ok(Self { + attrs: Attribute::parse_outer(input)?, + paren_token: syn::parenthesized!(content in input), + consumes: content.parse_terminated(Ident::parse)?, + }) + } +} + +/// Creates a list of generic identifiers used for the subsystems +pub(crate) fn parse_overseer_struct_field( + baggage_generics: HashSet, + fields: FieldsNamed, +) -> Result<(Vec, Vec)> { + let n = fields.named.len(); + let mut subsystems = Vec::with_capacity(n); + let mut baggage = Vec::with_capacity(n); + for (idx, Field { attrs, vis, ident, ty, .. }) in fields.named.into_iter().enumerate() { + let mut consumes = attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| { + attr.path.get_ident().filter(|ident| *ident == "subsystem").map(|_ident| { + let tokens = attr.tokens.clone(); + tokens + }) + }); + + let mut consumes_idents = Vec::with_capacity(attrs.len()); + for tokens in consumes { + let variant = syn::parse2::(dbg!(tokens))?; + consumes_idents.extend(variant.consumes.into_iter()); + } + + let ident = ident.unwrap(); + if !consumes_idents.is_empty() { + subsystems.push(SubSysField { + name: ident, + generic: Ident::new(format!("Sub{}", idx).as_str(), Span::call_site()), + ty: try_type_to_ident(ty, Span::call_site())?, + consumes: consumes_idents, + }); + } else { + baggage.push(BaggageField { + field_name: ident, + field_ty: try_type_to_ident(ty, Span::call_site())?, + generic: false, // XXX FIXME FIXME FIXME + }); + } + } + Ok((subsystems, baggage)) +} + +use syn::Generics; + +/// Extend the originally provided `Generics` with those generated by the subsystems, +/// namely `Sub#N`, plus one more `Ctx` which is generic over the subsystem context to use. +pub(crate) fn extra_generics_combine_generics( + orig_generics: &Generics, + subsystems: &[SubSysField], +) -> Result { + let mut combined_generics = orig_generics.clone(); + + let mut ctx_generic = GenericParam::Type( + syn::parse2::(quote! { + Ctx: SubsystemContext, + }) + .unwrap(), + ); + + let mut subsys_generics = subsystems.iter().map(|ssf| { + let subsys_generic_name = ssf.generic.clone(); + GenericParam::Type( + syn::parse2::(quote! { + #subsys_generic_name: Subsystem, + }) + .unwrap(), + ) + }); + combined_generics.params.extend(Some(ctx_generic)); + combined_generics.params.extend(subsys_generics); + Ok(combined_generics) +} + +/// Fields that are _not_ subsystems. +pub(crate) struct BaggageField { + pub(crate) field_name: Ident, + pub(crate) field_ty: Ident, + pub(crate) generic: bool, +} + +/// Generates the wrapper type enum. +pub(crate) fn impl_messages_wrapper_enum( + messages_wrapper: Ident, + subsystems: &[SubSysField], + baggage: &[BaggageField], +) -> Result { + let mut consumes = subsystems.iter().map(|ssf| ssf.consumes.clone().into_iter()).flatten(); + + let msg = "Generated message type wrapper"; + let x = quote! { + #[doc = #msg] + #[derive(Debug, Clone)] + enum #messages_wrapper { + #( + #consumes ( #consumes ), + )* + } + }; + Ok(x) +} + +pub(crate) fn impl_overseer_struct( + overseer_name: Ident, + orig_generics: Generics, + subsystems: &[SubSysField], + baggage: &[BaggageField], +) -> Result { + let mut field_name = &subsystems.iter().map(|ssf| ssf.name.clone()).collect::>(); + + let mut field_ty = &subsystems.iter().map(|ssf| ssf.generic.clone()).collect::>(); + + let mut baggage_name = baggage.iter().map(|bf| bf.field_name.clone()); + let mut baggage_ty = baggage.iter().map(|bf| bf.field_ty.clone()); + + let mut baggage_generic_ty = baggage.iter().filter(|bf| bf.generic).map(|bf| bf.field_ty.clone()); + + let generics = quote! { + < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > + }; + + let where_clause = quote! { + where + Ctx: SubsystemContext, + #( #field_ty : Subsystem )* + }; + + let mut x = quote! { + pub struct #overseer_name #generics { + #( + #field_name: #field_ty, + )* + + #( + #baggage_name: #baggage_ty, + )* + } + }; + + x.extend(crate::builder::impl_builder(overseer_name, subsystems, baggage)?); + + Ok(x) +} + +/// Implement the helper type `ChannelsOut` and `MessagePacket`. +pub(crate) fn impl_channels_out_struct( + subsystems: &[SubSysField], + baggage: &[BaggageField], +) -> Result { + let mut field_name = subsystems.iter().map(|ssf| ssf.name.clone()); + let mut field_ty = subsystems.iter().map(|ssf| ssf.ty.clone()); + let x = quote! { + #[derive(Debug)] + struct MessagePacket { + signals_received: usize, + message: T, + } + + fn make_packet(signals_received: usize, message: T) -> MessagePacket { + MessagePacket { + signals_received, + message, + } + } + + pub struct ChannelsOut { + #( + pub #field_name: ::metered::MeteredSender>, + )* + } + }; + Ok(x) +} + +pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result { + let args = parse_attr(attr)?; + let message_wrapper = args.wrapper_enum_name; + + let span = proc_macro2::Span::call_site(); + let ds = parse2::(orig.clone())?; + match ds.fields { + syn::Fields::Named(named) => { + let overseer_name = ds.ident.clone(); + + // collect the indepedentent subsystem generics + // which need to be carried along, there are the non-generated ones + let mut orig_generics = ds.generics; + + // remove default types + let mut baggage_generic_idents = HashSet::with_capacity(orig_generics.params.len()); + orig_generics.params = orig_generics + .params + .into_iter() + .map(|mut generic| { + match generic { + GenericParam::Type(ref mut param) => { + baggage_generic_idents.insert(param.ident.clone()); + param.eq_token = None; + param.default = None; + } + _ => {} + } + generic + }) + .collect(); + + let (subsystems, baggage) = parse_overseer_struct_field(baggage_generic_idents, named)?; + + let mut additive = impl_overseer_struct(overseer_name.clone(), orig_generics, &subsystems, &baggage)?; + + // additive.extend(impl_messages_wrapper_enum(message_wrapper, &subsystems[..], &baggage[..])?); + // additive.extend(impl_channels_out_struct(&subsystems[..], &baggage[..])?); + additive.extend(impl_replacable_subsystem(overseer_name, &subsystems[..], &baggage[..])); + + // additive.extend(inc::include_static_rs()?); + + Ok(additive) + } + syn::Fields::Unit => Err(Error::new(span, "Must be a struct with named fields. Not an unit struct.")), + syn::Fields::Unnamed(_) => { + Err(Error::new(span, "Must be a struct with named fields. Not an unnamed fields struct.")) + } + } +} diff --git a/node/overseer/overseer-gen/src/inc.rs b/node/overseer/overseer-gen/src/inc.rs new file mode 100644 index 000000000000..984d82a2bf8a --- /dev/null +++ b/node/overseer/overseer-gen/src/inc.rs @@ -0,0 +1,20 @@ +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use std::collections::HashSet; + +use quote::ToTokens; +use syn::AttrStyle; +use syn::Field; +use syn::FieldsNamed; +use syn::Result; +use syn::Variant; + +use super::*; + +pub(crate) fn include_static_rs() -> Result { + use std::str::FromStr; + + let s = include_str!("./inc/static.rs"); + let ts = proc_macro2::TokenStream::from_str(s)?; + Ok(ts) +} diff --git a/node/overseer/overseer-gen/src/inc/static.rs b/node/overseer/overseer-gen/src/inc/static.rs new file mode 100644 index 000000000000..518828aa1ca1 --- /dev/null +++ b/node/overseer/overseer-gen/src/inc/static.rs @@ -0,0 +1,13 @@ +trait MapSubsystem { + type Output; + + fn map_subsystem(&self, sub: T) -> Self::Output; +} + +impl MapSubsystem for F where F: Fn(T) -> U { + type Output = U; + + fn map_subsystem(&self, sub: T) -> U { + (self)(sub) + } +} diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 922b094c2917..283d1055a49c 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -14,385 +14,32 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::collections::HashSet; use proc_macro2::{Span, TokenStream}; use quote::quote; +use std::collections::HashSet; -use syn::{Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, parse2}; -use syn::FieldsNamed; +use quote::ToTokens; use syn::AttrStyle; use syn::Field; +use syn::FieldsNamed; use syn::Variant; -use quote::ToTokens; +use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; + +mod builder; +mod impls; +mod inc; +mod replace; + +use builder::*; +use impls::*; +use inc::*; +use replace::*; #[proc_macro_attribute] pub fn overlord(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream { let attr: TokenStream = attr.into(); let item: TokenStream = item.into(); - impl_overseer_gen(attr, item).unwrap_or_else(|err| err.to_compile_error()).into() -} - -#[derive(Clone)] -struct SubSysField { - /// Name of the field. - name: Ident, - /// Generate generic type name for the `AllSubsystems` type. - generic: Ident, - /// Type of the subsystem. - ty: Ident, - /// Type to be consumed by the subsystem. - consumes: Vec, -} - -fn try_type_to_ident(ty: Type, span: Span) -> Result { - match ty { - Type::Path(path) => path.path.get_ident().cloned() - .ok_or_else(|| { - Error::new(span, "Expected an identifier, but got a path.") - }), - _ => { - Err(Error::new(span, "Type must be a path expression.")) - } - } -} - -struct AttrArgs { - wrapper_enum_name: Ident, - signal_capacity: usize, - message_capacity: usize, -} - -fn parse_attr(_attr: TokenStream) -> Result { - Ok(AttrArgs { - wrapper_enum_name: Ident::new("AllMessages", Span::call_site()), - signal_capacity: 64usize, - message_capacity: 1024usize, - }) -} - -use syn::Token; -use syn::punctuated::Punctuated; -use syn::parse::Parse; -use syn::token::Paren; - -struct SubSystemTag { - attrs: Vec, - paren_token: Paren, - consumes: Punctuated, -} - -impl Parse for SubSystemTag { - fn parse(input: syn::parse::ParseStream) -> Result { - let content; - Ok(Self { - attrs: Attribute::parse_outer(input)?, - paren_token: syn::parenthesized!(content in input), - consumes: content.parse_terminated(Ident::parse)?, - }) - } -} - -/// Creates a list of generic identifiers used for the subsystems -fn parse_overseer_struct_field(baggage_generics: HashSet, fields: FieldsNamed) -> Result<(Vec, Vec)> { - let n = fields.named.len(); - let mut subsystems = Vec::with_capacity(n); - let mut baggage = Vec::with_capacity(n); - for (idx, Field { attrs, vis, ident, ty, .. }) in fields - .named - .into_iter() - .enumerate() { - - let mut consumes = attrs.iter().filter(|attr| { - attr.style == AttrStyle::Outer - }).filter_map(|attr| { - attr.path.get_ident().filter(|ident| { - *ident == "subsystem" - }).map(|_ident| { - let tokens = attr.tokens.clone(); - tokens - }) - }); - - let mut consumes_idents = Vec::with_capacity(attrs.len()); - for tokens in consumes { - let variant = syn::parse2::(dbg!(tokens))?; - consumes_idents.extend(variant.consumes.into_iter()); - } - - let ident = ident.unwrap(); - if !consumes_idents.is_empty() { - subsystems.push(SubSysField { - name: ident, - generic: Ident::new(format!("Sub{}", idx).as_str(), Span::call_site()), - ty: try_type_to_ident(ty, Span::call_site())?, - consumes: consumes_idents, - }); - } else { - baggage.push(BaggageField { - field_name: ident, - field_ty: try_type_to_ident(ty, Span::call_site())?, - generic: false,// XXX FIXME FIXME FIXME - }); - } - } - Ok((subsystems, baggage)) -} - -/// Add a feature to replace one subsystem of the subsystems type. -// fn impl_replace_subsystem() -> Result { -// let orig_generics = vec![]; -// let added_generics = vec![]; -// let added_generics_modded = vec![]; -// let keeper = vec![]; -// let overseer = Ident::new("Xxxx", Span::call_site()); -// let msg = "Xyz"; -// let x = quote!{ -// impl #orig_generics #overseer #orig_generics { -// #[doc = #msg] -// pub fn #fname < NEW > (self, replacement: NEW) -> #strukt_ty #modified_generics { -// #strukt_ty :: #modified_generics { -// #replacable_item: replacement, -// #( -// #keeper: self.#keeper, -// )* -// } -// } -// } -// }; -// Ok(x) -// } - -/// Fields that are _not_ subsystems. -struct BaggageField { - field_name: Ident, - field_ty: Ident, - generic: bool, -} - -/// Implement a builder pattern. -fn impl_builder(name: Ident, baggage: &[BaggageField], subsystems: &[SubSysField]) -> Result { - let builder = Ident::new((name.to_string() + "Builder").as_str(), Span::call_site()); - - let overseer = name.clone(); - let mut orig_generic_ty = &baggage.iter().filter(|b| b.generic).map(|b| b.field_ty.clone()).collect::>(); - let mut orig_generic_name = &baggage.iter().filter(|b| b.generic).map(|b| b.field_name.clone()).collect::>(); - - let mut field_name = &subsystems.iter().map(|x| x.name.clone() ).collect::>(); - let mut field_ty = &subsystems.iter().map(|x| x.ty.clone()).collect::>(); - - // FIXME - let subsystem_generics = Ident::new("FIXME", Span::call_site()); - - let x = quote!{ - - impl #name { - fn builder() -> #builder { - #builder :: default() - } - } - - struct #builder { - #( - #field_name : ::std::option::Option< #field_ty >, - )* - } - - - impl #builder { - #( - fn #field_name (#field_name: #field_ty ) -> #builder { - self.#field_name = Some( #field_name ); - } - )* - } - - impl #builder { - - fn build< #( #orig_generic_name, )* >(mut self, - #( - #orig_generic_name : #orig_generic_ty, - )* - ) -> #overseer < - #( #orig_generic_ty , )*, - SubSystems< #subsystem_generics > - > { - #overseer { - #( - #field_name : self. #field_ty .unwrap(), - )* - } - } - } - }; - Ok(x) -} - -// fn impl_replacable_subsystem(x: Vec<_>) -> Result { -// let msg = "Generated by #[overlord] derive proc-macro."; -// let mut additive = TokenStream::new(); - -// // generate an impl of `fn replace_#name` -// for NameTyTup { field: replacable_item, ty: replacable_item_ty } in replacable_items { -// let keeper = all_fields.iter().filter(|ntt| ntt.field != replacable_item).map(|ntt| ntt.field.clone()); -// let strukt_ty = strukt_ty.clone(); -// let fname = Ident::new(&format!("replace_{}", replacable_item), span); -// // adjust the generics such that the appropriate member type is replaced -// let mut modified_generics = orig_generics.clone(); -// modified_generics.params = modified_generics.params.into_iter().map(|mut generic| { -// match generic { -// GenericParam::Type(ref mut param) => { -// param.eq_token = None; -// param.default = None; -// if match &replacable_item_ty { -// Type::Path(path) => -// path.path.get_ident().filter(|&ident| ident == ¶m.ident).is_some(), -// _ => false -// } { -// param.ident = Ident::new("NEW", span); -// } -// } -// _ => {} -// } -// generic -// }).collect(); - -// #[derive(Clone)] -// struct NameTyTup { -// field: Ident, -// ty: Type, -// } - -// additive.extend(quote! { -// impl #orig_generics #strukt_ty #orig_generics { -// #[doc = #msg] -// pub fn #fname < NEW > (self, replacement: NEW) -> #strukt_ty #modified_generics { -// #strukt_ty :: #modified_generics { -// #replacable_item: replacement, -// #( -// #keeper: self.#keeper, -// )* -// } -// } -// } -// }); -// } - -// Ok(additive) -// } - - -fn impl_messages_wrapper_enum(messages_wrapper: Ident, subsystems: &[SubSysField]) -> Result { - let mut field_ty = subsystems.iter().map(|ssf| { - ssf.ty.clone() - }); - - let msg = "Generated message type wrapper"; - let x = quote!{ - #[doc = #msg] - #[derive(Debug, Clone)] - enum #messages_wrapper { - #( - #field_ty ( #field_ty ), - )* - } - }; - Ok(x) -} - - -fn impl_overseer_struct(overseer_name: Ident, subsystems: &[SubSysField], baggage: &[BaggageField]) -> Result { - let x = quote!{ - struct #overseer_name { - #( - #field_ty: field_name, - )* - - #( - #baggage_ty: baggage_field_name, - )* - } - }; - - x.extend(impl_builder(overseer_name, baggage, subsystems)?); - - Ok(x) -} - -fn impl_channels_out_struct(subsystems: &[SubSysField]) -> Result { - let mut field_name = subsystems.iter().map(|ssf| { - ssf.name.clone() - }); - let mut field_ty = subsystems.iter().map(|ssf| { - ssf.ty.clone() - }); - let x = quote!{ - #[derive(Debug)] - struct MessagePacket { - signals_received: usize, - message: T, - } - - fn make_packet(signals_received: usize, message: T) -> MessagePacket { - MessagePacket { - signals_received, - message, - } - } - - pub struct ChannelsOut { - #( - pub #field_name: ::metered::MeteredSender>, - )* - } - }; - Ok(x) -} - -fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result { - let args = parse_attr(attr)?; - let message_wrapper = args.wrapper_enum_name; - - let span = proc_macro2::Span::call_site(); - let ds = parse2::(orig.clone())?; - match ds.fields { - syn::Fields::Named(named) => { - let overseer_name = ds.ident.clone(); - - // collect the indepedentent subsystem generics - // which need to be carried along, there are the non-generated ones - let mut orig_generics = ds.generics; - - // remove default types - let mut baggage_generics = HashSet::with_capacity(orig_generics.params.len()); - orig_generics.params = orig_generics.params.into_iter().map(|mut generic| { - match generic { - GenericParam::Type(ref mut param) => { - baggage_generics.insert(param.ident.clone()); - param.eq_token = None; - param.default = None; - } - _ => {} - } - generic - }).collect(); - - let (subsystems, bags) = parse_overseer_struct_field(baggage_generics, named)?; - - let mut additive = impl_overseer_struct(&subsystems, &bags); - - additive.extend(impl_messages_wrapper_enum(message_wrapper, &subsystems[..])); - additive.extend(impl_channels_out_struct(&subsystems[..])); - // additive.extend(impl_replace_subsystem(&subsystems[..])); - // additive.extend(impl_(&subsystems[..])); - - - Ok(additive) - } - syn::Fields::Unit => Err(Error::new(span, "Must be a struct with named fields. Not an unit struct.")), - syn::Fields::Unnamed(_) => { - Err(Error::new(span, "Must be a struct with named fields. Not an unnamed fields struct.")) - } - } + impls::impl_overseer_gen(attr, item).unwrap_or_else(|err| err.to_compile_error()).into() } #[cfg(test)] @@ -406,7 +53,7 @@ mod tests { }; let item = quote! { - pub struct Ooooh { + pub struct Ooooh where S: SpawnThat { #[subsystem(Foo)] sub0: FooSubsystem, @@ -420,7 +67,7 @@ mod tests { let output = impl_overseer_gen(attr, item).expect("Simple example always works. qed"); println!("//generated:"); - println!("{:#?}", output); + println!("{}", output); } #[test] diff --git a/node/overseer/overseer-gen/src/replace.rs b/node/overseer/overseer-gen/src/replace.rs new file mode 100644 index 000000000000..b2366ec2273f --- /dev/null +++ b/node/overseer/overseer-gen/src/replace.rs @@ -0,0 +1,82 @@ +//! Replace a subsystem +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use std::collections::HashSet; + +use quote::ToTokens; +use syn::AttrStyle; +use syn::Field; +use syn::FieldsNamed; +use syn::Variant; +use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; + +use super::*; + +pub(crate) fn impl_replacable_subsystem( + name: Ident, + subsystems: &[SubSysField], + baggage: &[BaggageField], +) -> Result { + let msg = "Generated by #[overlord] derive proc-macro."; + + let span = Span::call_site(); + + let mut field_name = &subsystems.iter().map(|x| x.name.clone()).collect::>(); + let mut field_ty = &subsystems.iter().map(|x| x.generic.clone()).collect::>(); + + let mut baggage_generic_ty = &baggage.iter().filter(|b| b.generic).map(|b| b.field_ty.clone()).collect::>(); + + let mut baggage_name = &baggage.iter().map(|x| x.field_name.clone()).collect::>(); + + let generics = quote! { + < Ctx: SubsystemContext, #( #baggage_generic_ty, )* #( #field_ty, )* > + }; + + let where_clause = quote! { + where + #( #field_ty : Subsystem, )* + }; + + let mut additive = TokenStream::new(); + + // generate an impl of `fn replace_#name` + for SubSysField { name: replacable_item, ty: replacable_item_ty, generic, .. } in subsystems { + let keeper = subsystems.iter().filter(|&ssf| ssf.generic != *generic).map(|ssf| ssf.name.clone()); + + let fn_name = Ident::new(&format!("replace_{}", replacable_item), span); + // adjust the generics such that the appropriate member type is replaced + + let new = Ident::new("NEW", span); + let modified_generics = &subsystems + .iter() + .map(|ssf| if ssf.generic != *generic { ssf.generic.clone() } else { new.clone() }) + .collect::>(); + + let modified_generics = quote! { + < Ctx, #( #baggage_generic_ty, )* #( #modified_generics, )* > + }; + + let x: TokenStream = quote! { + impl #generics #name #generics #where_clause { + #[doc = #msg] + pub fn #fn_name < #new > (self, replacement: #new) -> #name #modified_generics + where + #new: Subsystem + { + #name :: #modified_generics { + #replacable_item: replacement, + #( + #keeper: self.#keeper, + )* + #( + #baggage_name: self.#baggage_name, + )* + } + } + } + }; + additive.extend(x); + } + + Ok(additive) +} diff --git a/node/overseer/overseer-gen/tests/ui/err-01.rs b/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs similarity index 100% rename from node/overseer/overseer-gen/tests/ui/err-01.rs rename to node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs diff --git a/node/overseer/overseer-gen/tests/ui/err-02-enum.rs b/node/overseer/overseer-gen/tests/ui/err-02-enum.rs new file mode 100644 index 000000000000..42d5947c0149 --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/err-02-enum.rs @@ -0,0 +1,27 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_gen::overlord; + +struct Msg(u8); + +#[derive(Default, Clone, Copy)] +struct AwesomeSub; + +#[derive(Clone, AllSubsystemsGen)] +#[overlord(Wrapper)] +enum Overseer { + #[subsystem(Msg)] + Sub0(AwesomeSub), +} + +struct Spawner; + +fn main() { + let overseer = Overseer::>::builder() + .sub0(FooSubSys::default()) + .build(Spawner); + + // try to replace one subsystem with another that can not handle `X`. + // since it's missing the trait bound. + let overseer = overseer.replace_sub0(TequilaInABar::default()); +} diff --git a/node/overseer/overseer-gen/tests/ui/gen.rs b/node/overseer/overseer-gen/tests/ui/gen.rs new file mode 100644 index 000000000000..ca56dcbc13ef --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/gen.rs @@ -0,0 +1,80 @@ +pub struct Ooooh { + sub0: Sub0, + yyy: Sub1, + spawner: S, + metrics: Metrics, +} + +impl Ooooh +where + Sub0: Subsystem, + Sub1: Subsystem, +{ + fn builder() -> OooohBuilder { + OooohBuilder::default() + } +} + +struct OooohBuilder { + sub0: ::std::option::Option, + yyy: ::std::option::Option, + spawner: ::std::option::Option, + metrics: ::std::option::Option, +} + +impl OooohBuilder +where + Sub0: Subsystem, + Sub1: Subsystem, +{ + fn sub0(mut self, new: Sub0) -> OooohBuilder { + self.sub0 = Some(new); + self + } + fn yyy(mut self, new: Sub1) -> OooohBuilder { + self.yyy = Some(new); + self + } + fn build(mut self, ctx: Ctx) -> Ooooh { + Ooooh:: { + sub0: self.sub0.unwrap(), + yyy: self.yyy.unwrap(), + spawner: self.spawner.unwrap(), + metrics: self.metrics.unwrap(), + } + } +} + +impl Ooooh +where + Sub0: Subsystem, + Sub1: Subsystem, +{ + #[doc = "Generated by #[overlord] derive proc-macro."] + pub fn replace_sub0(self, replacement: NEW) -> Ooooh + where + NEW: Subsystem, + { + Ooooh:: { + sub0: replacement, + yyy: self.yyy, + spawner: + self.spawner, + metrics: self.metrics + } + } +} + +impl Ooooh +where + Sub0: Subsystem, + Sub1: Subsystem, +{ + #[doc = "Generated by #[overlord] derive proc-macro."] + pub fn replace_yyy(self, replacement: NEW) -> Ooooh + where + NEW: Subsystem, + { + Ooooh:: { yyy: replacement, sub0: self.sub0, spawner: self.spawner, metrics: self.metrics } + } +} diff --git a/node/overseer/overseer-gen/tests/ui/ok-02-boring.rs b/node/overseer/overseer-gen/tests/ui/ok-02-boring.rs new file mode 100644 index 000000000000..ccf999c254b3 --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/ok-02-boring.rs @@ -0,0 +1,26 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_gen::overlord; + + +struct X; + +#[derive(Default, Clone, Copy)] +struct AwesomeSubSys; + +#[derive(Clone, AllSubsystemsGen)] +#[overlord(Wrapper)] +struct Overseer { + #[subsystem(X)] + sub0: AwesomeSubSys, +} + +struct Spawner; + +fn main() { + let overseer = Overseer::>::builder() + .sub0(AwesomeSubSys::default()) + .build(Spawner); + + let overseer = overseer.replace_sub0(TequilaInABar::default()); +} diff --git a/node/overseer/overseer-gen/tests/ui/ok-02.rs b/node/overseer/overseer-gen/tests/ui/ok-02.rs new file mode 100644 index 000000000000..ccf999c254b3 --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/ok-02.rs @@ -0,0 +1,26 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_gen::overlord; + + +struct X; + +#[derive(Default, Clone, Copy)] +struct AwesomeSubSys; + +#[derive(Clone, AllSubsystemsGen)] +#[overlord(Wrapper)] +struct Overseer { + #[subsystem(X)] + sub0: AwesomeSubSys, +} + +struct Spawner; + +fn main() { + let overseer = Overseer::>::builder() + .sub0(AwesomeSubSys::default()) + .build(Spawner); + + let overseer = overseer.replace_sub0(TequilaInABar::default()); +} diff --git a/node/overseer/overseer-gen/tests/ui/ok-03-w-generic.rs b/node/overseer/overseer-gen/tests/ui/ok-03-w-generic.rs new file mode 100644 index 000000000000..833ed4e20c10 --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/ok-03-w-generic.rs @@ -0,0 +1,30 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_gen::overlord; + +trait MetaMeta {} + +#[derive(Debug)] +struct MsgStrukt(u8) + +#[derive(Default, Clone, Copy)] +struct AwesomeSubSys; + +#[derive(Clone, AllSubsystemsGen)] +#[overlord(Wrapper)] +struct Overseer { + #[subsystem(MsgStrukt)] + sub0: AwesomeSubSys, + + something_else: T, +} + +struct Spawner; + +fn main() { + let overseer = Overseer::>::builder() + .sub0(AwesomeSubSys::default()) + .build(Spawner); + + let overseer = overseer.replace_sub0(TequilaInABar::default()); +} diff --git a/node/overseer/overseer-gen/tests/ui/ok-01.rs b/node/overseer/overseer-gen/tests/ui/xxxx.rs similarity index 100% rename from node/overseer/overseer-gen/tests/ui/ok-01.rs rename to node/overseer/overseer-gen/tests/ui/xxxx.rs From cf537e866bfbbeb9a6f13ec952107d34b586efc2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 7 May 2021 18:04:23 +0200 Subject: [PATCH 010/161] parse --- node/overseer/overseer-gen/src/builder.rs | 1 + node/overseer/overseer-gen/src/dispatch.rs | 145 ++++++++++++++++++ node/overseer/overseer-gen/src/impls.rs | 79 ++++++---- .../tests/ui/err-01-replace-w-inadequate.rs | 1 - node/overseer/overseer-gen/tests/ui/ok-02.rs | 26 ---- node/overseer/overseer-gen/tests/ui/xxxx.rs | 133 ---------------- 6 files changed, 194 insertions(+), 191 deletions(-) create mode 100644 node/overseer/overseer-gen/src/dispatch.rs delete mode 100644 node/overseer/overseer-gen/tests/ui/ok-02.rs delete mode 100644 node/overseer/overseer-gen/tests/ui/xxxx.rs diff --git a/node/overseer/overseer-gen/src/builder.rs b/node/overseer/overseer-gen/src/builder.rs index ec8931ecbb39..5038ff64c925 100644 --- a/node/overseer/overseer-gen/src/builder.rs +++ b/node/overseer/overseer-gen/src/builder.rs @@ -46,6 +46,7 @@ pub(crate) fn impl_builder( } } + #[derive(Debug, Clone, Default)] struct #builder #generics { #( #field_name : ::std::option::Option< #field_ty >, diff --git a/node/overseer/overseer-gen/src/dispatch.rs b/node/overseer/overseer-gen/src/dispatch.rs new file mode 100644 index 000000000000..eb56305a7669 --- /dev/null +++ b/node/overseer/overseer-gen/src/dispatch.rs @@ -0,0 +1,145 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use proc_macro2::TokenStream; +use quote::{quote, ToTokens}; +use std::fmt; +use syn::{parse2, Error, Fields, FieldsNamed, FieldsUnnamed, Ident, ItemEnum, Path, Result, Type, Variant}; + +/// An enum variant without base type. +#[derive(Clone)] +struct EnumVariantDispatchWithTy { + // enum ty name + ty: Ident, + // variant + variant: EnumVariantDispatch, +} + +impl fmt::Debug for EnumVariantDispatchWithTy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}::{:?}", self.ty, self.variant) + } +} + +impl ToTokens for EnumVariantDispatchWithTy { + fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { + if let Some(inner) = &self.variant.inner { + let enum_name = &self.ty; + let variant_name = &self.variant.name; + + let quoted = quote! { + #enum_name::#variant_name(#inner::from(event)) + }; + quoted.to_tokens(tokens); + } + } +} + +/// An enum variant without the base type, contains the relevant inner type. +#[derive(Clone)] +struct EnumVariantDispatch { + /// variant name + name: Ident, + /// The inner type for which a `From::from` impl is anticipated from the input type. + /// No code will be generated for this enum variant if `inner` is `None`. + inner: Option, +} + +impl fmt::Debug for EnumVariantDispatch { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}(..)", self.name) + } +} + +fn prepare_enum_variant(variant: &mut Variant) -> Result { + let skip = variant.attrs.iter().find(|attr| attr.path.is_ident("skip")).is_some(); + variant.attrs = variant.attrs.iter().filter(|attr| !attr.path.is_ident("skip")).cloned().collect::>(); + + let variant = variant.clone(); + let span = variant.ident.span(); + let inner = match variant.fields.clone() { + // look for one called inner + Fields::Named(FieldsNamed { brace_token: _, named }) if !skip => named + .iter() + .find_map(|field| { + if let Some(ident) = &field.ident { + if ident == "inner" { + return Some(Some(field.ty.clone())); + } + } + None + }) + .ok_or_else(|| Error::new(span, "To dispatch with struct enum variant, one element must named `inner`"))?, + + // technically, if it has no inner types we cound not require the #[skip] annotation, but better make it consistent + Fields::Unnamed(FieldsUnnamed { paren_token: _, unnamed }) if !skip => unnamed + .first() + .map(|field| Some(field.ty.clone())) + .ok_or_else(|| Error::new(span, "Must be annotated with skip, even if no inner types exist."))?, + _ if skip => None, + Fields::Unit => return Err(Error::new(span, "Must be annotated with #[skip].")), + Fields::Unnamed(_) => { + return Err(Error::new( + span, + "Must be annotated with #[skip] or have in `inner` element which impls `From<_>`.", + )) + } + Fields::Named(_) => { + return Err(Error::new( + span, + "Must be annotated with #[skip] or the first wrapped type must impl `From<_>`.", + )) + } + }; + + Ok(EnumVariantDispatch { name: variant.ident, inner }) +} + +fn impl_wrapper_dispatch_fn(name: Ident, fields: &[SubSysField]) -> Result { + let event_ty = parse2::(attr)?; + + let mut ie = parse2::(item)?; + + let message_enum = ie.ident.clone(); + let variants = ie.variants.iter_mut().try_fold(Vec::::new(), |mut acc, variant| { + let variant = prepare_enum_variant(variant)?; + if variant.inner.is_some() { + acc.push(EnumVariantDispatchWithTy { ty: message_enum.clone(), variant }) + } + Ok::<_, syn::Error>(acc) + })?; + + + let msg = "Generated by #[subsystem_dispatch_gen] proc-macro."; + + let mut x = quote! { + impl #message_enum { + #[doc = #msg] + pub fn dispatch_iter(event: #event_ty) -> impl Iterator + Send { + let mut iter = None.into_iter(); + + #( + let mut iter = iter.chain(std::iter::once(event.focus().ok().map(|event| { + #variants + }))); + )* + iter.filter_map(|x| x) + } + } + }; + + Ok(x) +} diff --git a/node/overseer/overseer-gen/src/impls.rs b/node/overseer/overseer-gen/src/impls.rs index cf1a49475614..f09d43fa7128 100644 --- a/node/overseer/overseer-gen/src/impls.rs +++ b/node/overseer/overseer-gen/src/impls.rs @@ -4,6 +4,7 @@ use std::collections::HashSet; use quote::ToTokens; use syn::AttrStyle; +use syn::Generics; use syn::Field; use syn::FieldsNamed; use syn::Variant; @@ -20,6 +21,8 @@ pub(crate) struct SubSysField { pub(crate) ty: Ident, /// Type to be consumed by the subsystem. pub(crate) consumes: Vec, + /// Consumes is a set of messages, are these to be dispatched? + pub(crate) dispatchable: Vec, } fn try_type_to_ident(ty: Type, span: Span) -> Result { @@ -50,10 +53,25 @@ use syn::punctuated::Punctuated; use syn::token::Paren; use syn::Token; + +pub(crate) struct SubSystemTagInner { + pub(crate) no_dispatch: Option, + pub(crate) consumes: Punctuated, +} + +impl Parse for SubSystemTagInner { + fn parse(input: syn::parse::ParseStream) -> Result { + Ok(Self { + no_dispatch: input.parse().map(|v| true)?, + consumes: input.parse()?, + }) + } +} + pub(crate) struct SubSystemTag { pub(crate) attrs: Vec, pub(crate) paren_token: Paren, - pub(crate) consumes: Punctuated, + pub(crate) inner: SubSystemTagInner, } impl Parse for SubSystemTag { @@ -62,7 +80,7 @@ impl Parse for SubSystemTag { Ok(Self { attrs: Attribute::parse_outer(input)?, paren_token: syn::parenthesized!(content in input), - consumes: content.parse_terminated(Ident::parse)?, + inner: content.parse_terminated(SubSystemTagInner::parse)?, }) } } @@ -108,36 +126,35 @@ pub(crate) fn parse_overseer_struct_field( Ok((subsystems, baggage)) } -use syn::Generics; - -/// Extend the originally provided `Generics` with those generated by the subsystems, -/// namely `Sub#N`, plus one more `Ctx` which is generic over the subsystem context to use. -pub(crate) fn extra_generics_combine_generics( - orig_generics: &Generics, - subsystems: &[SubSysField], -) -> Result { - let mut combined_generics = orig_generics.clone(); - let mut ctx_generic = GenericParam::Type( - syn::parse2::(quote! { - Ctx: SubsystemContext, - }) - .unwrap(), - ); - - let mut subsys_generics = subsystems.iter().map(|ssf| { - let subsys_generic_name = ssf.generic.clone(); - GenericParam::Type( - syn::parse2::(quote! { - #subsys_generic_name: Subsystem, - }) - .unwrap(), - ) - }); - combined_generics.params.extend(Some(ctx_generic)); - combined_generics.params.extend(subsys_generics); - Ok(combined_generics) -} +// /// Extend the originally provided `Generics` with those generated by the subsystems, +// /// namely `Sub#N`, plus one more `Ctx` which is generic over the subsystem context to use. +// pub(crate) fn extra_generics_combine_generics( +// orig_generics: &Generics, +// subsystems: &[SubSysField], +// ) -> Result { +// let mut combined_generics = orig_generics.clone(); + +// let mut ctx_generic = GenericParam::Type( +// syn::parse2::(quote! { +// Ctx: SubsystemContext, +// }) +// .unwrap(), +// ); + +// let mut subsys_generics = subsystems.iter().map(|ssf| { +// let subsys_generic_name = ssf.generic.clone(); +// GenericParam::Type( +// syn::parse2::(quote! { +// #subsys_generic_name: Subsystem, +// }) +// .unwrap(), +// ) +// }); +// combined_generics.params.extend(Some(ctx_generic)); +// combined_generics.params.extend(subsys_generics); +// Ok(combined_generics) +// } /// Fields that are _not_ subsystems. pub(crate) struct BaggageField { diff --git a/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs b/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs index 209efbdc752f..6ca3c10cac87 100644 --- a/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs +++ b/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs @@ -14,7 +14,6 @@ struct AwesomeSubSys; struct TequilaInABar; -#[derive(Clone, AllSubsystemsGen)] #[overlord(Wrapper)] struct Overseer { #[subsystem(X)] diff --git a/node/overseer/overseer-gen/tests/ui/ok-02.rs b/node/overseer/overseer-gen/tests/ui/ok-02.rs deleted file mode 100644 index ccf999c254b3..000000000000 --- a/node/overseer/overseer-gen/tests/ui/ok-02.rs +++ /dev/null @@ -1,26 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_overseer_gen::overlord; - - -struct X; - -#[derive(Default, Clone, Copy)] -struct AwesomeSubSys; - -#[derive(Clone, AllSubsystemsGen)] -#[overlord(Wrapper)] -struct Overseer { - #[subsystem(X)] - sub0: AwesomeSubSys, -} - -struct Spawner; - -fn main() { - let overseer = Overseer::>::builder() - .sub0(AwesomeSubSys::default()) - .build(Spawner); - - let overseer = overseer.replace_sub0(TequilaInABar::default()); -} diff --git a/node/overseer/overseer-gen/tests/ui/xxxx.rs b/node/overseer/overseer-gen/tests/ui/xxxx.rs deleted file mode 100644 index 876895c83aa5..000000000000 --- a/node/overseer/overseer-gen/tests/ui/xxxx.rs +++ /dev/null @@ -1,133 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_overseer_gen::overlord; - - -#[derive(Debug, Clone)] -struct Foo { - pub x: String, -} - -#[derive(Debug, Clone, Copy)] -struct Bar; - -#[derive(Debug, Clone, Copy)] -enum Gap { - X, Y -} - -#[derive(Debug, Clone, Copy)] -struct Arg { - x: &'static [u8], -} - -struct SubsystemA { - -} - - -// generated -struct SubSystems {} - -impl SubSystemPick for SubSystems -where - A: OverseerSubsystem, - V: OverseerSubsystem, -{ - -} - -#[overlord(AllMessages, capacity=1024, signals=64)] -struct Overseer { - #[subsystem(Foo)] - subsystem_a: SubsystemA, - // generic A - - #[subsystem(Bar)] - foo_bar: SubsystemB, - // generic B - - #[subsystem(Gap)] - clown: Clown, - // generic C - - spawner: S, - metrics: Metrics, -} - -impl Overseer { - pub fn subsystems() -> &mut SubSystems { - self.subsystems - } -} - -impl Overseer { - fn builder(metrics: Metrics) -> Self { - OverseerBuilder { - - } - } -} - -const SUBSYSTEM_A: u16 = 0b0001; -const FOO_BAR: u16 = 0b0010; - -struct OverseerBuilder { - // # all members - subsystem_a: Option, - foo_bar: .., - clown: .., -} - -impl OverseerBuilder<0b0111> { - // # for all unset members - fn build(mut self, metrics: Metrics) -> Overseer where S: SubSystemPick { - Overseeer { - subsystems: SubSystems { - subsystem_a: self.subsystem_a.unwrap(), - foo_bar: self.foo_bar.unwrap(), - clown: self.clown.unwrap(), - }, - } - } -} - -impl OverseerBuilder<0u16> { - fn subsystem_a(mut self, subsystem_a: SubsystemA) -> OverseerBuilder { - self.subsystem_a = Some(subsystem_a); - self - } - - fn foo_bar(mut self, foo_bar: FooBar) -> OverseerBuilder { - // unimplemented!() - } - - fn clown(mut self) -> OverseerBuilder { - self.clown = Some(Clown::default()); - } -} - -impl OverseerBuilder { - fn foo_bar(mut self) -> OverseerBuilder { - // unimplemented!() - } - - fn clown(mut self) -> OverseerBuilder { - self.clown = Some(Clown::default()); - } -} - - -impl OverseerBuilder { - fn foo_bar(mut self) -> OverseerBuilder { - // unimplemented!() - } -} - -fn main() { - Overseer::builder() - .subsystem_a(a, b) - .foo_bar() - .build(metrics) - -} From b36d75a9c2bfac758b8a634414d67f143e8e78fa Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 7 May 2021 23:03:21 +0200 Subject: [PATCH 011/161] more changes --- Cargo.lock | 1 + node/overseer/overseer-gen/src/builder.rs | 10 ++- node/overseer/overseer-gen/src/impls.rs | 74 +++++++++-------- .../tests/ui/err-03-subsys-twice.rs | 40 ++++++++++ node/overseer/overseer-gen/tests/ui/gen.rs | 80 ------------------- .../tests/ui/ok-03-no-dispatch.rs | 29 +++++++ ...3-w-generic.rs => ok-03-w-generic copy.rs} | 0 node/overseer/src/subsystems.rs | 0 8 files changed, 119 insertions(+), 115 deletions(-) create mode 100644 node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs delete mode 100644 node/overseer/overseer-gen/tests/ui/gen.rs create mode 100644 node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs rename node/overseer/overseer-gen/tests/ui/{ok-03-w-generic.rs => ok-03-w-generic copy.rs} (100%) create mode 100644 node/overseer/src/subsystems.rs diff --git a/Cargo.lock b/Cargo.lock index 57f1218f1513..85d7622291e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6308,6 +6308,7 @@ name = "polkadot-procmacro-overseer-gen" version = "0.1.0" dependencies = [ "assert_matches", + "proc-macro-crate 1.0.0", "proc-macro2", "quote", "syn", diff --git a/node/overseer/overseer-gen/src/builder.rs b/node/overseer/overseer-gen/src/builder.rs index 5038ff64c925..3751ae99567a 100644 --- a/node/overseer/overseer-gen/src/builder.rs +++ b/node/overseer/overseer-gen/src/builder.rs @@ -64,15 +64,19 @@ pub(crate) fn impl_builder( } )* - fn build(mut self, ctx: Ctx) -> #overseer #generics { - #overseer :: #generics { + fn build(mut self, ctx: Ctx) -> (#overseer #generics, #handler) { + let overseer = #overseer :: #generics { #( #field_name : self. #field_name .unwrap(), )* #( #baggage_name : self. #baggage_name .unwrap(), )* - } + }; + let handler = #handler { + + }; + (overseer, handler) } } }; diff --git a/node/overseer/overseer-gen/src/impls.rs b/node/overseer/overseer-gen/src/impls.rs index f09d43fa7128..45e1c2ed00be 100644 --- a/node/overseer/overseer-gen/src/impls.rs +++ b/node/overseer/overseer-gen/src/impls.rs @@ -9,6 +9,7 @@ use syn::Field; use syn::FieldsNamed; use syn::Variant; use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; +use syn::spanned::Spanned; use super::*; #[derive(Clone)] @@ -22,7 +23,7 @@ pub(crate) struct SubSysField { /// Type to be consumed by the subsystem. pub(crate) consumes: Vec, /// Consumes is a set of messages, are these to be dispatched? - pub(crate) dispatchable: Vec, + pub(crate) no_dispatch: bool, } fn try_type_to_ident(ty: Type, span: Span) -> Result { @@ -54,24 +55,11 @@ use syn::token::Paren; use syn::Token; -pub(crate) struct SubSystemTagInner { - pub(crate) no_dispatch: Option, - pub(crate) consumes: Punctuated, -} - -impl Parse for SubSystemTagInner { - fn parse(input: syn::parse::ParseStream) -> Result { - Ok(Self { - no_dispatch: input.parse().map(|v| true)?, - consumes: input.parse()?, - }) - } -} - pub(crate) struct SubSystemTag { pub(crate) attrs: Vec, pub(crate) paren_token: Paren, - pub(crate) inner: SubSystemTagInner, + pub(crate) no_dispatch: bool, + pub(crate) consumes: Punctuated, } impl Parse for SubSystemTag { @@ -80,7 +68,8 @@ impl Parse for SubSystemTag { Ok(Self { attrs: Attribute::parse_outer(input)?, paren_token: syn::parenthesized!(content in input), - inner: content.parse_terminated(SubSystemTagInner::parse)?, + no_dispatch: false, // FIXME + consumes: content.parse_terminated(Ident::parse)?, }) } } @@ -90,36 +79,57 @@ pub(crate) fn parse_overseer_struct_field( baggage_generics: HashSet, fields: FieldsNamed, ) -> Result<(Vec, Vec)> { + let span = Span::call_site(); let n = fields.named.len(); let mut subsystems = Vec::with_capacity(n); let mut baggage = Vec::with_capacity(n); for (idx, Field { attrs, vis, ident, ty, .. }) in fields.named.into_iter().enumerate() { let mut consumes = attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| { - attr.path.get_ident().filter(|ident| *ident == "subsystem").map(|_ident| { - let tokens = attr.tokens.clone(); - tokens + let span = attr.path.span(); + attr.path.get_ident().filter(|ident| *ident == "subsystem").map(move |_ident| { + let attr_tokens = attr.tokens.clone(); + (attr_tokens, span) }) }); + let ident = ident.ok_or_else(|| { + Error::new(ty.span(), "Missing identifier for member. BUG") + })?; + + if let Some((attr_tokens, span)) = consumes.next() { + if let Some((attr_tokens2, span2)) = consumes.next() { + return Err({ + let mut err = Error::new(span, "The first subsystem annotation is at"); + err.combine( + Error::new(span2, "but another here for the same field.") + ); + err + }) + } + let mut consumes_idents = Vec::with_capacity(attrs.len()); - let mut consumes_idents = Vec::with_capacity(attrs.len()); - for tokens in consumes { - let variant = syn::parse2::(dbg!(tokens))?; + let variant = syn::parse2::(dbg!(attr_tokens.clone()))?; consumes_idents.extend(variant.consumes.into_iter()); - } - let ident = ident.unwrap(); - if !consumes_idents.is_empty() { + if consumes_idents.is_empty() { + return Err( + Error::new(span, "Subsystem must consume at least one message") + ) + } + let no_dispatch = variant.no_dispatch; + subsystems.push(SubSysField { name: ident, generic: Ident::new(format!("Sub{}", idx).as_str(), Span::call_site()), - ty: try_type_to_ident(ty, Span::call_site())?, + ty: try_type_to_ident(ty, span)?, consumes: consumes_idents, + no_dispatch, }); } else { + let field_ty = try_type_to_ident(ty, Span::call_site())?; baggage.push(BaggageField { field_name: ident, - field_ty: try_type_to_ident(ty, Span::call_site())?, - generic: false, // XXX FIXME FIXME FIXME + generic: !baggage_generics.contains(&field_ty), + field_ty, }); } } @@ -292,11 +302,11 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< let mut additive = impl_overseer_struct(overseer_name.clone(), orig_generics, &subsystems, &baggage)?; - // additive.extend(impl_messages_wrapper_enum(message_wrapper, &subsystems[..], &baggage[..])?); - // additive.extend(impl_channels_out_struct(&subsystems[..], &baggage[..])?); + additive.extend(impl_messages_wrapper_enum(message_wrapper, &subsystems[..], &baggage[..])?); + additive.extend(impl_channels_out_struct(&subsystems[..], &baggage[..])?); additive.extend(impl_replacable_subsystem(overseer_name, &subsystems[..], &baggage[..])); - // additive.extend(inc::include_static_rs()?); + additive.extend(inc::include_static_rs()?); Ok(additive) } diff --git a/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs b/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs new file mode 100644 index 000000000000..1718f42b8082 --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs @@ -0,0 +1,40 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_gen::overlord; + +struct X; +struct Z; + +struct Orange; + +#[derive(Default, Clone, Copy)] +struct AwesomeSubSys; + + +#[derive(Default, Clone, Copy)] +struct TequilaInABar; + + +#[overlord(Wrapper)] +struct Overseer { + #[subsystem(X)] + #[subsystem(Z)] + sub0: AwesomeSubSys, + + #[subsystem(Orange)] + shots_of: TequilaInABar + + other: Stuff, +} + +struct Spawner; + +fn main() { + let overseer = Overseer::>::builder() + .sub0(FooSubSys::default()) + .build(Spawner); + + // try to replace one subsystem with another that can not handle `X`. + // since it's missing the trait bound. + let overseer = overseer.replace_sub0(TequilaInABar::default()); +} diff --git a/node/overseer/overseer-gen/tests/ui/gen.rs b/node/overseer/overseer-gen/tests/ui/gen.rs deleted file mode 100644 index ca56dcbc13ef..000000000000 --- a/node/overseer/overseer-gen/tests/ui/gen.rs +++ /dev/null @@ -1,80 +0,0 @@ -pub struct Ooooh { - sub0: Sub0, - yyy: Sub1, - spawner: S, - metrics: Metrics, -} - -impl Ooooh -where - Sub0: Subsystem, - Sub1: Subsystem, -{ - fn builder() -> OooohBuilder { - OooohBuilder::default() - } -} - -struct OooohBuilder { - sub0: ::std::option::Option, - yyy: ::std::option::Option, - spawner: ::std::option::Option, - metrics: ::std::option::Option, -} - -impl OooohBuilder -where - Sub0: Subsystem, - Sub1: Subsystem, -{ - fn sub0(mut self, new: Sub0) -> OooohBuilder { - self.sub0 = Some(new); - self - } - fn yyy(mut self, new: Sub1) -> OooohBuilder { - self.yyy = Some(new); - self - } - fn build(mut self, ctx: Ctx) -> Ooooh { - Ooooh:: { - sub0: self.sub0.unwrap(), - yyy: self.yyy.unwrap(), - spawner: self.spawner.unwrap(), - metrics: self.metrics.unwrap(), - } - } -} - -impl Ooooh -where - Sub0: Subsystem, - Sub1: Subsystem, -{ - #[doc = "Generated by #[overlord] derive proc-macro."] - pub fn replace_sub0(self, replacement: NEW) -> Ooooh - where - NEW: Subsystem, - { - Ooooh:: { - sub0: replacement, - yyy: self.yyy, - spawner: - self.spawner, - metrics: self.metrics - } - } -} - -impl Ooooh -where - Sub0: Subsystem, - Sub1: Subsystem, -{ - #[doc = "Generated by #[overlord] derive proc-macro."] - pub fn replace_yyy(self, replacement: NEW) -> Ooooh - where - NEW: Subsystem, - { - Ooooh:: { yyy: replacement, sub0: self.sub0, spawner: self.spawner, metrics: self.metrics } - } -} diff --git a/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs b/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs new file mode 100644 index 000000000000..af99446b9de6 --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs @@ -0,0 +1,29 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_gen::overlord; + +#[derive(Debug)] +struct MsgStrukt(u8); + +#[derive(Default, Clone, Copy)] +struct AwesomeSubSys; + +#[overlord(Wrapper)] +struct Overseer { + #[subsystem(no_dispatch, MsgStrukt)] + sub0: AwesomeSubSys, + + something_else: T, +} + +#[derive(Debug, Clone, Copy)] +struct DummySpawner; + +fn main() { + let overseer = Overseer::::builder() + .sub0(AwesomeSubSys::default()) + .spawner(DummySpawner) + .build(Ctx); + + let overseer = overseer.replace_sub0(TequilaInABar::default()); +} diff --git a/node/overseer/overseer-gen/tests/ui/ok-03-w-generic.rs b/node/overseer/overseer-gen/tests/ui/ok-03-w-generic copy.rs similarity index 100% rename from node/overseer/overseer-gen/tests/ui/ok-03-w-generic.rs rename to node/overseer/overseer-gen/tests/ui/ok-03-w-generic copy.rs diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs new file mode 100644 index 000000000000..e69de29bb2d1 From d86938ea3517a9b2adf2e9c48bc13a64080cbc0b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 10 May 2021 12:06:10 +0200 Subject: [PATCH 012/161] shuffle --- node/overseer/overseer-gen/src/builder.rs | 84 -- .../overseer-gen/src/impl_channels_out.rs | 100 ++ .../overseer-gen/src/impl_overseer.rs | 224 ++++ .../src/{replace.rs => impl_replace.rs} | 0 node/overseer/overseer-gen/src/impls.rs | 72 +- node/overseer/overseer-gen/src/inc/static.rs | 47 + node/overseer/overseer-gen/src/lib.rs | 10 +- node/overseer/src/lib.rs | 1011 +---------------- node/overseer/src/tests.rs | 1001 ++++++++++++++++ 9 files changed, 1382 insertions(+), 1167 deletions(-) delete mode 100644 node/overseer/overseer-gen/src/builder.rs create mode 100644 node/overseer/overseer-gen/src/impl_channels_out.rs create mode 100644 node/overseer/overseer-gen/src/impl_overseer.rs rename node/overseer/overseer-gen/src/{replace.rs => impl_replace.rs} (100%) create mode 100644 node/overseer/src/tests.rs diff --git a/node/overseer/overseer-gen/src/builder.rs b/node/overseer/overseer-gen/src/builder.rs deleted file mode 100644 index 3751ae99567a..000000000000 --- a/node/overseer/overseer-gen/src/builder.rs +++ /dev/null @@ -1,84 +0,0 @@ -use proc_macro2::{Span, TokenStream}; -use quote::quote; -use std::collections::HashSet; - -use quote::ToTokens; -use syn::AttrStyle; -use syn::Field; -use syn::FieldsNamed; -use syn::Variant; -use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; - -use super::*; - -/// Implement a builder pattern. -pub(crate) fn impl_builder( - name: Ident, - subsystems: &[SubSysField], - baggage: &[BaggageField], -) -> Result { - let builder = Ident::new((name.to_string() + "Builder").as_str(), Span::call_site()); - - let overseer = name.clone(); - - let mut field_name = &subsystems.iter().map(|x| x.name.clone()).collect::>(); - let mut field_ty = &subsystems.iter().map(|x| x.generic.clone()).collect::>(); - - let mut baggage_generic_ty = &baggage.iter().filter(|b| b.generic).map(|b| b.field_ty.clone()).collect::>(); - - let mut baggage_name = &baggage.iter().map(|x| x.field_name.clone()).collect::>(); - let mut baggage_ty = &baggage.iter().map(|x| x.field_ty.clone()).collect::>(); - - let generics = quote! { - < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > - }; - - let where_clause = quote! { - where - #( #field_ty : Subsystem, )* - }; - - let x = quote! { - - impl #generics #name #generics #where_clause { - fn builder() -> #builder { - #builder :: default() - } - } - - #[derive(Debug, Clone, Default)] - struct #builder #generics { - #( - #field_name : ::std::option::Option< #field_ty >, - )* - #( - #baggage_name : ::std::option::Option< #baggage_name >, - )* - } - - impl #generics #builder #generics #where_clause { - #( - fn #field_name (mut self, new: #field_ty ) -> #builder { - self.#field_name = Some( new ); - self - } - )* - - fn build(mut self, ctx: Ctx) -> (#overseer #generics, #handler) { - let overseer = #overseer :: #generics { - #( - #field_name : self. #field_name .unwrap(), - )* - #( - #baggage_name : self. #baggage_name .unwrap(), - )* - }; - let handler = #handler { - - }; - (overseer, handler) - } - } - }; - Ok(x) -} diff --git a/node/overseer/overseer-gen/src/impl_channels_out.rs b/node/overseer/overseer-gen/src/impl_channels_out.rs new file mode 100644 index 000000000000..ca35a3a6e9cc --- /dev/null +++ b/node/overseer/overseer-gen/src/impl_channels_out.rs @@ -0,0 +1,100 @@ +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use std::collections::HashSet; + +use quote::ToTokens; +use syn::AttrStyle; +use syn::Generics; +use syn::Field; +use syn::FieldsNamed; +use syn::Variant; +use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; +use syn::spanned::Spanned; + +use super::*; + +/// Implement the helper type `ChannelsOut` and `MessagePacket`. +pub(crate) fn impl_channels_out_struct( + wrapper: &Ident, + subsystems: &[SubSysField], + baggage: &[BaggageField], +) -> Result { + let mut channel_name = subsystems.iter().map(|ssf| + ssf.name.clone()); + let mut channel_name_unbounded = subsystems.iter().map(|ssf| + Ident::new(ssf.name.span(), ssf.name.to_string() + "_unbounded") + ); + let mut field_ty = subsystems.iter().map(|ssf| ssf.ty.clone()); + let ts = quote! { + #[derive(Debug)] + struct MessagePacket { + signals_received: usize, + message: T, + } + + fn make_packet(signals_received: usize, message: T) -> MessagePacket { + MessagePacket { + signals_received, + message, + } + } + + pub struct ChannelsOut { + #( + pub #channel_name: ::metered::MeteredSender>, + )* + #( + pub #channel_name_unbounded: ::metered::UnboundedMeteredSender>, + )* + } + + impl ChannelsOut { + async pub fn send_and_log_error( + &mut self, + signals_received: usize, + message: #wrapper, + ) { + let res = match message { + #( + #wrapper :: #field_ty (msg) => { + self. #channel_name .send(make_packet(signals_received, msg)).await + }, + ) + }; + + if res.is_err() { + tracing::debug!( + target: LOG_TARGET, + "Failed to send a message to another subsystem", + ); + } + } + + + pub fn send_unbounded_and_log_error( + &self, + signals_received: usize, + message: AllMessages, + ) { + let res = match message { + #( + #wrapper :: #field_ty (msg) => { + self. #channel_name_unbounded .send( + make_packet(signals_received, msg) + ).await + }, + ) + }; + + if res.is_err() { + tracing::debug!( + target: LOG_TARGET, + "Failed to send a message to another subsystem", + ); + } + } + } + + }; + Ok(ts) +} diff --git a/node/overseer/overseer-gen/src/impl_overseer.rs b/node/overseer/overseer-gen/src/impl_overseer.rs new file mode 100644 index 000000000000..1f8fb68f828f --- /dev/null +++ b/node/overseer/overseer-gen/src/impl_overseer.rs @@ -0,0 +1,224 @@ +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use std::collections::HashSet; + +use quote::ToTokens; +use syn::AttrStyle; +use syn::Field; +use syn::FieldsNamed; +use syn::Variant; +use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; + +use super::*; + +pub(crate) fn impl_overseer_struct( + overseer_name: Ident, + orig_generics: Generics, + subsystems: &[SubSysField], + baggage: &[BaggageField], +) -> Result { + let mut field_name = &subsystems.iter().map(|ssf| ssf.name.clone()).collect::>(); + + let mut field_ty = &subsystems.iter().map(|ssf| ssf.generic.clone()).collect::>(); + + let mut baggage_name = baggage.iter().map(|bf| bf.field_name.clone()); + let mut baggage_ty = baggage.iter().map(|bf| bf.field_ty.clone()); + + let mut baggage_generic_ty = baggage.iter().filter(|bf| bf.generic).map(|bf| bf.field_ty.clone()); + + let generics = quote! { + < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > + }; + + let where_clause = quote! { + where + Ctx: SubsystemContext, + #( #field_ty : Subsystem )* + }; + + let mut x = quote! { + pub struct #overseer_name #generics { + #( + #field_name: #field_ty, + )* + + #( + #baggage_name: #baggage_ty, + )* + } + + impl #generics #overseer_name #generics { + pub async fn stop(mut self) -> { + #( + let _ = self. #field_name .send_signal(OverseerSignal::Conclude).await + )* + loop { + select! { + _ = self.running_subsystems.next() => { + if self.running_subsystems.is_empty() { + break; + } + }, + _ = stop_delay => break, + complete => break, + } + } + } + } + + async pub fn broadcast_signal(&mut self, signal: OverseerSignal) -> SubsystemResult<()> { + #( + self. #field_name .send_signal(signal.clone()).await, + )* + let _ = signal; + + Ok(()) + } + + async pub fn route_message(&mut self, msg: #wrapper) -> SubsystemResult<()> { + match msg { + #( + #field_ty (msg) => self. #field_name .send_message(msg).await?, + )* + } + Ok(()) + } + }; + + x.extend(crate::builder::impl_builder(overseer_name, subsystems, baggage)?); + + Ok(x) +} + +/// Implement a builder pattern. +pub(crate) fn impl_builder( + name: Ident, + subsystems: &[SubSysField], + baggage: &[BaggageField], +) -> Result { + let builder = Ident::new((name.to_string() + "Builder").as_str(), Span::call_site()); + + let overseer = name.clone(); + let handler = Ident::new(&(overseer.to_string() + "Handler"), overseer.span()); + + let mut field_name = &subsystems.iter().map(|x| x.name.clone()).collect::>(); + let mut field_ty = &subsystems.iter().map(|x| x.generic.clone()).collect::>(); + + let mut channel_name = subsystems.iter().map(|ssf| + ssf.name.clone()); + let mut channel_name_unbounded = subsystems.iter().map(|ssf| + Ident::new(ssf.name.span(), ssf.name.to_string() + "_unbounded") + ); + let mut channel_name_tx = subsystems.iter().map(|ssf| + Ident::new(ssf.name.span(), ssf.name.to_string() + "_tx") + ); + let mut channel_name_unbounded_tx = subsystems.iter().map(|ssf| + Ident::new(ssf.name.span(), ssf.name.to_string() + "_unbounded_tx") + ); + let mut channel_name_rx = subsystems.iter().map(|ssf| + Ident::new(ssf.name.span(), ssf.name.to_string() + "_rx") + ); + let mut channel_name_unbounded_rx = subsystems.iter().map(|ssf| + Ident::new(ssf.name.span(), ssf.name.to_string() + "_unbounded_rx") + ); + + let mut baggage_generic_ty = &baggage.iter().filter(|b| b.generic).map(|b| b.field_ty.clone()).collect::>(); + + let mut baggage_name = &baggage.iter().map(|x| x.field_name.clone()).collect::>(); + let mut baggage_ty = &baggage.iter().map(|x| x.field_ty.clone()).collect::>(); + + let generics = quote! { + < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > + }; + + let where_clause = quote! { + where + #( #field_ty : Subsystem, )* + }; + + let x = quote! { + + impl #generics #name #generics #where_clause { + fn builder() -> #builder { + #builder :: default() + } + } + + #[derive(Debug, Clone, Default)] + struct #builder #generics { + #( + #field_name : ::std::option::Option< #field_ty >, + )* + #( + #baggage_name : ::std::option::Option< #baggage_name >, + )* + } + + impl #generics #builder #generics #where_clause { + #( + fn #field_name (mut self, new: #field_ty ) -> #builder { + self.#field_name = Some( new ); + self + } + )* + + fn build(mut self, ctx: Ctx) -> (#overseer #generics, #handler) { + let overseer = #overseer :: #generics { + #( + #field_name : self. #field_name .unwrap(), + )* + #( + #baggage_name : self. #baggage_name .unwrap(), + )* + }; + + const CHANNEL_CAPACITY: usize = 1024; + const SIGNAL_CHANNEL_CAPACITY: usize = 64; + let (events_tx, events_rx) = #crate::metered::channel(SIGNAL_CHANNEL_CAPACITY); + + let handler = #handler { + events_tx: events_tx.clone(), + }; + + + let metrics = ::register(prometheus_registry)?; + + let (to_overseer_tx, to_overseer_rx) = metered::unbounded(); + + let mut running_subsystems = FuturesUnordered::new(); + + + let channels_out = { + #( + let (#channel_name_tx, #channel_name_rx) = ::metered::channel::>(CHANNEL_CAPACITY); + )* + + #( + let (#channel_name_unbounded_tx, #channel_name_unbounded_rx) = ::metered::unbounded::>(); + )* + + + let channels_out = ChannelsOut { + #( + pub #channel_name: #channel_name_tx .clone(), + )* + #( + pub #channel_name_unbounded: #channel_name_tx_unbounded .clone(), + )* + } + + ChannelsOut { + // #( #bounded ) + // #( #unbounded ) + }; + } + + + // #( #launch subsystem ) + + (overseer, handler) + } + } + }; + Ok(x) +} diff --git a/node/overseer/overseer-gen/src/replace.rs b/node/overseer/overseer-gen/src/impl_replace.rs similarity index 100% rename from node/overseer/overseer-gen/src/replace.rs rename to node/overseer/overseer-gen/src/impl_replace.rs diff --git a/node/overseer/overseer-gen/src/impls.rs b/node/overseer/overseer-gen/src/impls.rs index 45e1c2ed00be..389eea723f0b 100644 --- a/node/overseer/overseer-gen/src/impls.rs +++ b/node/overseer/overseer-gen/src/impls.rs @@ -12,6 +12,8 @@ use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, use syn::spanned::Spanned; use super::*; + + #[derive(Clone)] pub(crate) struct SubSysField { /// Name of the field. @@ -194,77 +196,7 @@ pub(crate) fn impl_messages_wrapper_enum( Ok(x) } -pub(crate) fn impl_overseer_struct( - overseer_name: Ident, - orig_generics: Generics, - subsystems: &[SubSysField], - baggage: &[BaggageField], -) -> Result { - let mut field_name = &subsystems.iter().map(|ssf| ssf.name.clone()).collect::>(); - - let mut field_ty = &subsystems.iter().map(|ssf| ssf.generic.clone()).collect::>(); - - let mut baggage_name = baggage.iter().map(|bf| bf.field_name.clone()); - let mut baggage_ty = baggage.iter().map(|bf| bf.field_ty.clone()); - - let mut baggage_generic_ty = baggage.iter().filter(|bf| bf.generic).map(|bf| bf.field_ty.clone()); - - let generics = quote! { - < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > - }; - - let where_clause = quote! { - where - Ctx: SubsystemContext, - #( #field_ty : Subsystem )* - }; - - let mut x = quote! { - pub struct #overseer_name #generics { - #( - #field_name: #field_ty, - )* - - #( - #baggage_name: #baggage_ty, - )* - } - }; - x.extend(crate::builder::impl_builder(overseer_name, subsystems, baggage)?); - - Ok(x) -} - -/// Implement the helper type `ChannelsOut` and `MessagePacket`. -pub(crate) fn impl_channels_out_struct( - subsystems: &[SubSysField], - baggage: &[BaggageField], -) -> Result { - let mut field_name = subsystems.iter().map(|ssf| ssf.name.clone()); - let mut field_ty = subsystems.iter().map(|ssf| ssf.ty.clone()); - let x = quote! { - #[derive(Debug)] - struct MessagePacket { - signals_received: usize, - message: T, - } - - fn make_packet(signals_received: usize, message: T) -> MessagePacket { - MessagePacket { - signals_received, - message, - } - } - - pub struct ChannelsOut { - #( - pub #field_name: ::metered::MeteredSender>, - )* - } - }; - Ok(x) -} pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result { let args = parse_attr(attr)?; diff --git a/node/overseer/overseer-gen/src/inc/static.rs b/node/overseer/overseer-gen/src/inc/static.rs index 518828aa1ca1..ee9dd3450897 100644 --- a/node/overseer/overseer-gen/src/inc/static.rs +++ b/node/overseer/overseer-gen/src/inc/static.rs @@ -11,3 +11,50 @@ impl MapSubsystem for F where F: Fn(T) -> U { (self)(sub) } } + +type SubsystemIncomingMessages = ::futures::stream::Select< + ::metered::MeteredReceiver>, + ::metered::UnboundedMeteredReceiver>, +>; + + + +#[derive(Debug, Default, Clone)] +struct SignalsReceived(Arc); + +impl SignalsReceived { + fn load(&self) -> usize { + self.0.load(atomic::Ordering::SeqCst) + } + + fn inc(&self) { + self.0.fetch_add(1, atomic::Ordering::SeqCst); + } +} + +/// A sender from subsystems to other subsystems. +#[derive(Debug, Clone)] +pub struct OverseerSubsystemSender { + channels: ChannelsOut, + signals_received: SignalsReceived, +} + +#[async_trait::async_trait] +impl SubsystemSender for OverseerSubsystemSender { + async fn send_message(&mut self, msg: AllMessages) { + self.channels.send_and_log_error(self.signals_received.load(), msg).await; + } + + async fn send_messages(&mut self, msgs: T) + where T: IntoIterator + Send, T::IntoIter: Send + { + // This can definitely be optimized if necessary. + for msg in msgs { + self.send_message(msg).await; + } + } + + fn send_unbounded_message(&mut self, msg: AllMessages) { + self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); + } +} diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 283d1055a49c..ddebffa05cab 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -25,15 +25,17 @@ use syn::FieldsNamed; use syn::Variant; use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; -mod builder; mod impls; +mod impl_overseer; +mod impl_replace; +mod impl_channels_out; mod inc; -mod replace; -use builder::*; use impls::*; +use impl_overseer::*; +use impl_replace::*; +use impl_channels_out::*; use inc::*; -use replace::*; #[proc_macro_attribute] pub fn overlord(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream { diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 6a674fb29e35..af21a83f3455 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -1308,12 +1308,7 @@ where /// /// The [`Subsystems`] may be any type as long as they implement an expected interface. /// Here, we create a mock validation subsystem and a few dummy ones and start the `Overseer` with them. - /// For the sake of simplicity the termination of the example is done with a timeout. - /// ``` - /// # use std::time::Duration; - /// # use futures::{executor, pin_mut, select, FutureExt}; - /// # use futures_timer::Delay; - /// # use polkadot_overseer::{Overseer, HeadSupportsParachains, AllSubsystems}; + /// For the sake of siroute_messagerseer::{Overseer, HeadSupportsParachains, AllSubsystems}; /// # use polkadot_primitives::v1::Hash; /// # use polkadot_subsystem::{ /// # Subsystem, DummySubsystem, SpawnedSubsystem, SubsystemContext, @@ -2237,1006 +2232,4 @@ fn spawn( } #[cfg(test)] -mod tests { - use std::sync::atomic; - use std::collections::HashMap; - use futures::{executor, pin_mut, select, FutureExt, pending}; - - use polkadot_primitives::v1::{CollatorPair, CandidateHash}; - use polkadot_subsystem::{messages::RuntimeApiRequest, messages::NetworkBridgeEvent, jaeger}; - use polkadot_node_primitives::{CollationResult, CollationGenerationConfig, PoV, BlockData}; - use polkadot_node_network_protocol::{PeerId, UnifiedReputationChange}; - use polkadot_node_subsystem_util::metered; - - use sp_core::crypto::Pair as _; - use assert_matches::assert_matches; - - use super::*; - - struct TestSubsystem1(metered::MeteredSender); - - impl Subsystem for TestSubsystem1 - where C: SubsystemContext - { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let mut sender = self.0; - SpawnedSubsystem { - name: "test-subsystem-1", - future: Box::pin(async move { - let mut i = 0; - loop { - match ctx.recv().await { - Ok(FromOverseer::Communication { .. }) => { - let _ = sender.send(i).await; - i += 1; - continue; - } - Ok(FromOverseer::Signal(OverseerSignal::Conclude)) => return Ok(()), - Err(_) => return Ok(()), - _ => (), - } - } - }), - } - } - } - - struct TestSubsystem2(metered::MeteredSender); - - impl Subsystem for TestSubsystem2 - where C: SubsystemContext - { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let sender = self.0.clone(); - SpawnedSubsystem { - name: "test-subsystem-2", - future: Box::pin(async move { - let _sender = sender; - let mut c: usize = 0; - loop { - if c < 10 { - let (tx, _) = oneshot::channel(); - ctx.send_message( - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - Default::default(), - PoV { - block_data: BlockData(Vec::new()), - }.into(), - tx, - ) - ) - ).await; - c += 1; - continue; - } - match ctx.try_recv().await { - Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => { - break; - } - Ok(Some(_)) => { - continue; - } - Err(_) => return Ok(()), - _ => (), - } - pending!(); - } - - Ok(()) - }), - } - } - } - - struct ReturnOnStart; - - impl Subsystem for ReturnOnStart - where C: SubsystemContext - { - fn start(self, mut _ctx: C) -> SpawnedSubsystem { - SpawnedSubsystem { - name: "test-subsystem-4", - future: Box::pin(async move { - // Do nothing and exit. - Ok(()) - }), - } - } - } - - struct MockSupportsParachains; - - impl HeadSupportsParachains for MockSupportsParachains { - fn head_supports_parachains(&self, _head: &Hash) -> bool { - true - } - } - - // Checks that a minimal configuration of two jobs can run and exchange messages. - #[test] - fn overseer_works() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let (s1_tx, s1_rx) = metered::channel::(64); - let (s2_tx, s2_rx) = metered::channel::(64); - - let mut s1_rx = s1_rx.fuse(); - let mut s2_rx = s2_rx.fuse(); - - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_validation(TestSubsystem1(s1_tx)) - .replace_candidate_backing(TestSubsystem2(s2_tx)); - - let (overseer, mut handler) = Overseer::new( - vec![], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - let overseer_fut = overseer.run().fuse(); - - pin_mut!(overseer_fut); - - let mut s1_results = Vec::new(); - let mut s2_results = Vec::new(); - - loop { - select! { - _ = overseer_fut => break, - s1_next = s1_rx.next() => { - match s1_next { - Some(msg) => { - s1_results.push(msg); - if s1_results.len() == 10 { - handler.stop().await; - } - } - None => break, - } - }, - s2_next = s2_rx.next() => { - match s2_next { - Some(_) => s2_results.push(s2_next), - None => break, - } - }, - complete => break, - } - } - - assert_eq!(s1_results, (0..10).collect::>()); - }); - } - - // Checks activated/deactivated metrics are updated properly. - #[test] - fn overseer_metrics_work() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let first_block_hash = [1; 32].into(); - let second_block_hash = [2; 32].into(); - let third_block_hash = [3; 32].into(); - - let first_block = BlockInfo { - hash: first_block_hash, - parent_hash: [0; 32].into(), - number: 1, - }; - let second_block = BlockInfo { - hash: second_block_hash, - parent_hash: first_block_hash, - number: 2, - }; - let third_block = BlockInfo { - hash: third_block_hash, - parent_hash: second_block_hash, - number: 3, - }; - - let all_subsystems = AllSubsystems::<()>::dummy(); - let registry = prometheus::Registry::new(); - let (overseer, mut handler) = Overseer::new( - vec![first_block], - all_subsystems, - Some(®istry), - MockSupportsParachains, - spawner, - ).unwrap(); - let overseer_fut = overseer.run().fuse(); - - pin_mut!(overseer_fut); - - handler.block_imported(second_block).await; - handler.block_imported(third_block).await; - handler.send_msg(AllMessages::CandidateValidation(test_candidate_validation_msg())).await; - handler.stop().await; - - select! { - res = overseer_fut => { - assert!(res.is_ok()); - let metrics = extract_metrics(®istry); - assert_eq!(metrics["activated"], 3); - assert_eq!(metrics["deactivated"], 2); - assert_eq!(metrics["relayed"], 1); - }, - complete => (), - } - }); - } - - fn extract_metrics(registry: &prometheus::Registry) -> HashMap<&'static str, u64> { - let gather = registry.gather(); - assert_eq!(gather[0].get_name(), "parachain_activated_heads_total"); - assert_eq!(gather[1].get_name(), "parachain_deactivated_heads_total"); - assert_eq!(gather[2].get_name(), "parachain_messages_relayed_total"); - let activated = gather[0].get_metric()[0].get_counter().get_value() as u64; - let deactivated = gather[1].get_metric()[0].get_counter().get_value() as u64; - let relayed = gather[2].get_metric()[0].get_counter().get_value() as u64; - let mut result = HashMap::new(); - result.insert("activated", activated); - result.insert("deactivated", deactivated); - result.insert("relayed", relayed); - result - } - - // Spawn a subsystem that immediately exits. - // - // Should immediately conclude the overseer itself. - #[test] - fn overseer_ends_on_subsystem_exit() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_backing(ReturnOnStart); - let (overseer, _handle) = Overseer::new( - vec![], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - - overseer.run().await.unwrap(); - }) - } - - struct TestSubsystem5(metered::MeteredSender); - - impl Subsystem for TestSubsystem5 - where C: SubsystemContext - { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let mut sender = self.0.clone(); - - SpawnedSubsystem { - name: "test-subsystem-5", - future: Box::pin(async move { - loop { - match ctx.try_recv().await { - Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => break, - Ok(Some(FromOverseer::Signal(s))) => { - sender.send(s).await.unwrap(); - continue; - }, - Ok(Some(_)) => continue, - Err(_) => break, - _ => (), - } - pending!(); - } - - Ok(()) - }), - } - } - } - - struct TestSubsystem6(metered::MeteredSender); - - impl Subsystem for TestSubsystem6 - where C: SubsystemContext - { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let mut sender = self.0.clone(); - - SpawnedSubsystem { - name: "test-subsystem-6", - future: Box::pin(async move { - loop { - match ctx.try_recv().await { - Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => break, - Ok(Some(FromOverseer::Signal(s))) => { - sender.send(s).await.unwrap(); - continue; - }, - Ok(Some(_)) => continue, - Err(_) => break, - _ => (), - } - pending!(); - } - - Ok(()) - }), - } - } - } - - // Tests that starting with a defined set of leaves and receiving - // notifications on imported blocks triggers expected `StartWork` and `StopWork` heartbeats. - #[test] - fn overseer_start_stop_works() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let first_block_hash = [1; 32].into(); - let second_block_hash = [2; 32].into(); - let third_block_hash = [3; 32].into(); - - let first_block = BlockInfo { - hash: first_block_hash, - parent_hash: [0; 32].into(), - number: 1, - }; - let second_block = BlockInfo { - hash: second_block_hash, - parent_hash: first_block_hash, - number: 2, - }; - let third_block = BlockInfo { - hash: third_block_hash, - parent_hash: second_block_hash, - number: 3, - }; - - let (tx_5, mut rx_5) = metered::channel(64); - let (tx_6, mut rx_6) = metered::channel(64); - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_validation(TestSubsystem5(tx_5)) - .replace_candidate_backing(TestSubsystem6(tx_6)); - let (overseer, mut handler) = Overseer::new( - vec![first_block], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - - let overseer_fut = overseer.run().fuse(); - pin_mut!(overseer_fut); - - let mut ss5_results = Vec::new(); - let mut ss6_results = Vec::new(); - - handler.block_imported(second_block).await; - handler.block_imported(third_block).await; - - let expected_heartbeats = vec![ - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: first_block_hash, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - })), - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: [ActivatedLeaf { - hash: second_block_hash, - number: 2, - span: Arc::new(jaeger::Span::Disabled), - }].as_ref().into(), - deactivated: [first_block_hash].as_ref().into(), - }), - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: [ActivatedLeaf { - hash: third_block_hash, - number: 3, - span: Arc::new(jaeger::Span::Disabled), - }].as_ref().into(), - deactivated: [second_block_hash].as_ref().into(), - }), - ]; - - loop { - select! { - res = overseer_fut => { - assert!(res.is_ok()); - break; - }, - res = rx_5.next() => { - if let Some(res) = res { - ss5_results.push(res); - } - } - res = rx_6.next() => { - if let Some(res) = res { - ss6_results.push(res); - } - } - complete => break, - } - - if ss5_results.len() == expected_heartbeats.len() && - ss6_results.len() == expected_heartbeats.len() { - handler.stop().await; - } - } - - assert_eq!(ss5_results, expected_heartbeats); - assert_eq!(ss6_results, expected_heartbeats); - }); - } - - // Tests that starting with a defined set of leaves and receiving - // notifications on imported blocks triggers expected `StartWork` and `StopWork` heartbeats. - #[test] - fn overseer_finalize_works() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let first_block_hash = [1; 32].into(); - let second_block_hash = [2; 32].into(); - let third_block_hash = [3; 32].into(); - - let first_block = BlockInfo { - hash: first_block_hash, - parent_hash: [0; 32].into(), - number: 1, - }; - let second_block = BlockInfo { - hash: second_block_hash, - parent_hash: [42; 32].into(), - number: 2, - }; - let third_block = BlockInfo { - hash: third_block_hash, - parent_hash: second_block_hash, - number: 3, - }; - - let (tx_5, mut rx_5) = metered::channel(64); - let (tx_6, mut rx_6) = metered::channel(64); - - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_validation(TestSubsystem5(tx_5)) - .replace_candidate_backing(TestSubsystem6(tx_6)); - - // start with two forks of different height. - let (overseer, mut handler) = Overseer::new( - vec![first_block, second_block], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - - let overseer_fut = overseer.run().fuse(); - pin_mut!(overseer_fut); - - let mut ss5_results = Vec::new(); - let mut ss6_results = Vec::new(); - - // this should stop work on both forks we started with earlier. - handler.block_finalized(third_block).await; - - let expected_heartbeats = vec![ - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: [ - ActivatedLeaf { - hash: first_block_hash, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - }, - ActivatedLeaf { - hash: second_block_hash, - number: 2, - span: Arc::new(jaeger::Span::Disabled), - }, - ].as_ref().into(), - ..Default::default() - }), - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - deactivated: [first_block_hash, second_block_hash].as_ref().into(), - ..Default::default() - }), - OverseerSignal::BlockFinalized(third_block_hash, 3), - ]; - - loop { - select! { - res = overseer_fut => { - assert!(res.is_ok()); - break; - }, - res = rx_5.next() => { - if let Some(res) = res { - ss5_results.push(res); - } - } - res = rx_6.next() => { - if let Some(res) = res { - ss6_results.push(res); - } - } - complete => break, - } - - if ss5_results.len() == expected_heartbeats.len() && ss6_results.len() == expected_heartbeats.len() { - handler.stop().await; - } - } - - assert_eq!(ss5_results.len(), expected_heartbeats.len()); - assert_eq!(ss6_results.len(), expected_heartbeats.len()); - - // Notifications on finality for multiple blocks at once - // may be received in different orders. - for expected in expected_heartbeats { - assert!(ss5_results.contains(&expected)); - assert!(ss6_results.contains(&expected)); - } - }); - } - - #[test] - fn do_not_send_empty_leaves_update_on_block_finalization() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let imported_block = BlockInfo { - hash: Hash::random(), - parent_hash: Hash::random(), - number: 1, - }; - - let finalized_block = BlockInfo { - hash: Hash::random(), - parent_hash: Hash::random(), - number: 1, - }; - - let (tx_5, mut rx_5) = metered::channel(64); - - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_backing(TestSubsystem6(tx_5)); - - let (overseer, mut handler) = Overseer::new( - Vec::new(), - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - - let overseer_fut = overseer.run().fuse(); - pin_mut!(overseer_fut); - - let mut ss5_results = Vec::new(); - - handler.block_finalized(finalized_block.clone()).await; - handler.block_imported(imported_block.clone()).await; - - let expected_heartbeats = vec![ - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: [ - ActivatedLeaf { - hash: imported_block.hash, - number: imported_block.number, - span: Arc::new(jaeger::Span::Disabled) - } - ].as_ref().into(), - ..Default::default() - }), - OverseerSignal::BlockFinalized(finalized_block.hash, 1), - ]; - - loop { - select! { - res = overseer_fut => { - assert!(res.is_ok()); - break; - }, - res = rx_5.next() => { - if let Some(res) = dbg!(res) { - ss5_results.push(res); - } - } - } - - if ss5_results.len() == expected_heartbeats.len() { - handler.stop().await; - } - } - - assert_eq!(ss5_results.len(), expected_heartbeats.len()); - - for expected in expected_heartbeats { - assert!(ss5_results.contains(&expected)); - } - }); - } - - #[derive(Clone)] - struct CounterSubsystem { - stop_signals_received: Arc, - signals_received: Arc, - msgs_received: Arc, - } - - impl CounterSubsystem { - fn new( - stop_signals_received: Arc, - signals_received: Arc, - msgs_received: Arc, - ) -> Self { - Self { - stop_signals_received, - signals_received, - msgs_received, - } - } - } - - impl Subsystem for CounterSubsystem - where - C: SubsystemContext, - M: Send, - { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - SpawnedSubsystem { - name: "counter-subsystem", - future: Box::pin(async move { - loop { - match ctx.try_recv().await { - Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => { - self.stop_signals_received.fetch_add(1, atomic::Ordering::SeqCst); - break; - }, - Ok(Some(FromOverseer::Signal(_))) => { - self.signals_received.fetch_add(1, atomic::Ordering::SeqCst); - continue; - }, - Ok(Some(FromOverseer::Communication { .. })) => { - self.msgs_received.fetch_add(1, atomic::Ordering::SeqCst); - continue; - }, - Err(_) => (), - _ => (), - } - pending!(); - } - - Ok(()) - }), - } - } - } - - fn test_candidate_validation_msg() -> CandidateValidationMessage { - let (sender, _) = oneshot::channel(); - let pov = Arc::new(PoV { block_data: BlockData(Vec::new()) }); - CandidateValidationMessage::ValidateFromChainState(Default::default(), pov, sender) - } - - fn test_candidate_backing_msg() -> CandidateBackingMessage { - let (sender, _) = oneshot::channel(); - CandidateBackingMessage::GetBackedCandidates(Default::default(), Vec::new(), sender) - } - - fn test_candidate_selection_msg() -> CandidateSelectionMessage { - CandidateSelectionMessage::default() - } - - fn test_chain_api_msg() -> ChainApiMessage { - let (sender, _) = oneshot::channel(); - ChainApiMessage::FinalizedBlockNumber(sender) - } - - fn test_collator_generation_msg() -> CollationGenerationMessage { - CollationGenerationMessage::Initialize(CollationGenerationConfig { - key: CollatorPair::generate().0, - collator: Box::new(|_, _| TestCollator.boxed()), - para_id: Default::default(), - }) - } - struct TestCollator; - - impl Future for TestCollator { - type Output = Option; - - fn poll(self: Pin<&mut Self>, _cx: &mut futures::task::Context) -> Poll { - panic!("at the Disco") - } - } - - impl Unpin for TestCollator {} - - fn test_collator_protocol_msg() -> CollatorProtocolMessage { - CollatorProtocolMessage::CollateOn(Default::default()) - } - - fn test_network_bridge_event() -> NetworkBridgeEvent { - NetworkBridgeEvent::PeerDisconnected(PeerId::random()) - } - - fn test_statement_distribution_msg() -> StatementDistributionMessage { - StatementDistributionMessage::NetworkBridgeUpdateV1(test_network_bridge_event()) - } - - fn test_availability_recovery_msg() -> AvailabilityRecoveryMessage { - let (sender, _) = oneshot::channel(); - AvailabilityRecoveryMessage::RecoverAvailableData( - Default::default(), - Default::default(), - None, - sender, - ) - } - - fn test_bitfield_distribution_msg() -> BitfieldDistributionMessage { - BitfieldDistributionMessage::NetworkBridgeUpdateV1(test_network_bridge_event()) - } - - fn test_provisioner_msg() -> ProvisionerMessage { - let (sender, _) = oneshot::channel(); - ProvisionerMessage::RequestInherentData(Default::default(), sender) - } - - fn test_runtime_api_msg() -> RuntimeApiMessage { - let (sender, _) = oneshot::channel(); - RuntimeApiMessage::Request(Default::default(), RuntimeApiRequest::Validators(sender)) - } - - fn test_availability_store_msg() -> AvailabilityStoreMessage { - let (sender, _) = oneshot::channel(); - AvailabilityStoreMessage::QueryAvailableData(CandidateHash(Default::default()), sender) - } - - fn test_network_bridge_msg() -> NetworkBridgeMessage { - NetworkBridgeMessage::ReportPeer(PeerId::random(), UnifiedReputationChange::BenefitMinor("")) - } - - fn test_approval_distribution_msg() -> ApprovalDistributionMessage { - ApprovalDistributionMessage::NewBlocks(Default::default()) - } - - fn test_approval_voting_msg() -> ApprovalVotingMessage { - let (sender, _) = oneshot::channel(); - ApprovalVotingMessage::ApprovedAncestor(Default::default(), 0, sender) - } - - // Checks that `stop`, `broadcast_signal` and `broadcast_message` are implemented correctly. - #[test] - fn overseer_all_subsystems_receive_signals_and_messages() { - const NUM_SUBSYSTEMS: usize = 18; - // -3 for BitfieldSigning, GossipSupport and AvailabilityDistribution - const NUM_SUBSYSTEMS_MESSAGED: usize = NUM_SUBSYSTEMS - 3; - - let spawner = sp_core::testing::TaskExecutor::new(); - executor::block_on(async move { - let stop_signals_received = Arc::new(atomic::AtomicUsize::new(0)); - let signals_received = Arc::new(atomic::AtomicUsize::new(0)); - let msgs_received = Arc::new(atomic::AtomicUsize::new(0)); - - let subsystem = CounterSubsystem::new( - stop_signals_received.clone(), - signals_received.clone(), - msgs_received.clone(), - ); - - let all_subsystems = AllSubsystems { - candidate_validation: subsystem.clone(), - candidate_backing: subsystem.clone(), - candidate_selection: subsystem.clone(), - collation_generation: subsystem.clone(), - collator_protocol: subsystem.clone(), - statement_distribution: subsystem.clone(), - availability_distribution: subsystem.clone(), - availability_recovery: subsystem.clone(), - bitfield_signing: subsystem.clone(), - bitfield_distribution: subsystem.clone(), - provisioner: subsystem.clone(), - runtime_api: subsystem.clone(), - availability_store: subsystem.clone(), - network_bridge: subsystem.clone(), - chain_api: subsystem.clone(), - approval_distribution: subsystem.clone(), - approval_voting: subsystem.clone(), - gossip_support: subsystem.clone(), - }; - let (overseer, mut handler) = Overseer::new( - vec![], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - let overseer_fut = overseer.run().fuse(); - - pin_mut!(overseer_fut); - - // send a signal to each subsystem - handler.block_imported(BlockInfo { - hash: Default::default(), - parent_hash: Default::default(), - number: Default::default(), - }).await; - - // send a msg to each subsystem - // except for BitfieldSigning and GossipSupport as the messages are not instantiable - handler.send_msg(AllMessages::CandidateValidation(test_candidate_validation_msg())).await; - handler.send_msg(AllMessages::CandidateBacking(test_candidate_backing_msg())).await; - handler.send_msg(AllMessages::CandidateSelection(test_candidate_selection_msg())).await; - handler.send_msg(AllMessages::CollationGeneration(test_collator_generation_msg())).await; - handler.send_msg(AllMessages::CollatorProtocol(test_collator_protocol_msg())).await; - handler.send_msg(AllMessages::StatementDistribution(test_statement_distribution_msg())).await; - handler.send_msg(AllMessages::AvailabilityRecovery(test_availability_recovery_msg())).await; - // handler.send_msg(AllMessages::BitfieldSigning(test_bitfield_signing_msg())).await; - // handler.send_msg(AllMessages::GossipSupport(test_bitfield_signing_msg())).await; - handler.send_msg(AllMessages::BitfieldDistribution(test_bitfield_distribution_msg())).await; - handler.send_msg(AllMessages::Provisioner(test_provisioner_msg())).await; - handler.send_msg(AllMessages::RuntimeApi(test_runtime_api_msg())).await; - handler.send_msg(AllMessages::AvailabilityStore(test_availability_store_msg())).await; - handler.send_msg(AllMessages::NetworkBridge(test_network_bridge_msg())).await; - handler.send_msg(AllMessages::ChainApi(test_chain_api_msg())).await; - handler.send_msg(AllMessages::ApprovalDistribution(test_approval_distribution_msg())).await; - handler.send_msg(AllMessages::ApprovalVoting(test_approval_voting_msg())).await; - - // Wait until all subsystems have received. Otherwise the messages might race against - // the conclude signal. - loop { - match (&mut overseer_fut).timeout(Duration::from_millis(100)).await { - None => { - let r = msgs_received.load(atomic::Ordering::SeqCst); - if r < NUM_SUBSYSTEMS_MESSAGED { - Delay::new(Duration::from_millis(100)).await; - } else if r > NUM_SUBSYSTEMS_MESSAGED { - panic!("too many messages received??"); - } else { - break - } - } - Some(_) => panic!("exited too early"), - } - } - - // send a stop signal to each subsystems - handler.stop().await; - - let res = overseer_fut.await; - assert_eq!(stop_signals_received.load(atomic::Ordering::SeqCst), NUM_SUBSYSTEMS); - assert_eq!(signals_received.load(atomic::Ordering::SeqCst), NUM_SUBSYSTEMS); - assert_eq!(msgs_received.load(atomic::Ordering::SeqCst), NUM_SUBSYSTEMS_MESSAGED); - - assert!(res.is_ok()); - }); - } - - #[test] - fn context_holds_onto_message_until_enough_signals_received() { - let (candidate_validation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (candidate_backing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (candidate_selection_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (statement_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (availability_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (availability_recovery_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (bitfield_signing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (bitfield_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (provisioner_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (runtime_api_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (availability_store_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (network_bridge_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (chain_api_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (collator_protocol_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (collation_generation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (approval_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (approval_voting_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (gossip_support_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - - let (candidate_validation_unbounded_tx, _) = metered::unbounded(); - let (candidate_backing_unbounded_tx, _) = metered::unbounded(); - let (candidate_selection_unbounded_tx, _) = metered::unbounded(); - let (statement_distribution_unbounded_tx, _) = metered::unbounded(); - let (availability_distribution_unbounded_tx, _) = metered::unbounded(); - let (availability_recovery_unbounded_tx, _) = metered::unbounded(); - let (bitfield_signing_unbounded_tx, _) = metered::unbounded(); - let (bitfield_distribution_unbounded_tx, _) = metered::unbounded(); - let (provisioner_unbounded_tx, _) = metered::unbounded(); - let (runtime_api_unbounded_tx, _) = metered::unbounded(); - let (availability_store_unbounded_tx, _) = metered::unbounded(); - let (network_bridge_unbounded_tx, _) = metered::unbounded(); - let (chain_api_unbounded_tx, _) = metered::unbounded(); - let (collator_protocol_unbounded_tx, _) = metered::unbounded(); - let (collation_generation_unbounded_tx, _) = metered::unbounded(); - let (approval_distribution_unbounded_tx, _) = metered::unbounded(); - let (approval_voting_unbounded_tx, _) = metered::unbounded(); - let (gossip_support_unbounded_tx, _) = metered::unbounded(); - - let channels_out = ChannelsOut { - candidate_validation: candidate_validation_bounded_tx.clone(), - candidate_backing: candidate_backing_bounded_tx.clone(), - candidate_selection: candidate_selection_bounded_tx.clone(), - statement_distribution: statement_distribution_bounded_tx.clone(), - availability_distribution: availability_distribution_bounded_tx.clone(), - availability_recovery: availability_recovery_bounded_tx.clone(), - bitfield_signing: bitfield_signing_bounded_tx.clone(), - bitfield_distribution: bitfield_distribution_bounded_tx.clone(), - provisioner: provisioner_bounded_tx.clone(), - runtime_api: runtime_api_bounded_tx.clone(), - availability_store: availability_store_bounded_tx.clone(), - network_bridge: network_bridge_bounded_tx.clone(), - chain_api: chain_api_bounded_tx.clone(), - collator_protocol: collator_protocol_bounded_tx.clone(), - collation_generation: collation_generation_bounded_tx.clone(), - approval_distribution: approval_distribution_bounded_tx.clone(), - approval_voting: approval_voting_bounded_tx.clone(), - gossip_support: gossip_support_bounded_tx.clone(), - - candidate_validation_unbounded: candidate_validation_unbounded_tx.clone(), - candidate_backing_unbounded: candidate_backing_unbounded_tx.clone(), - candidate_selection_unbounded: candidate_selection_unbounded_tx.clone(), - statement_distribution_unbounded: statement_distribution_unbounded_tx.clone(), - availability_distribution_unbounded: availability_distribution_unbounded_tx.clone(), - availability_recovery_unbounded: availability_recovery_unbounded_tx.clone(), - bitfield_signing_unbounded: bitfield_signing_unbounded_tx.clone(), - bitfield_distribution_unbounded: bitfield_distribution_unbounded_tx.clone(), - provisioner_unbounded: provisioner_unbounded_tx.clone(), - runtime_api_unbounded: runtime_api_unbounded_tx.clone(), - availability_store_unbounded: availability_store_unbounded_tx.clone(), - network_bridge_unbounded: network_bridge_unbounded_tx.clone(), - chain_api_unbounded: chain_api_unbounded_tx.clone(), - collator_protocol_unbounded: collator_protocol_unbounded_tx.clone(), - collation_generation_unbounded: collation_generation_unbounded_tx.clone(), - approval_distribution_unbounded: approval_distribution_unbounded_tx.clone(), - approval_voting_unbounded: approval_voting_unbounded_tx.clone(), - gossip_support_unbounded: gossip_support_unbounded_tx.clone(), - }; - - let (mut signal_tx, signal_rx) = metered::channel(CHANNEL_CAPACITY); - let (mut bounded_tx, bounded_rx) = metered::channel(CHANNEL_CAPACITY); - let (unbounded_tx, unbounded_rx) = metered::unbounded(); - let (to_overseer_tx, _to_overseer_rx) = metered::unbounded(); - - let mut ctx = OverseerSubsystemContext::<()>::new_unmetered( - signal_rx, - stream::select(bounded_rx, unbounded_rx), - channels_out, - to_overseer_tx, - ); - - assert_eq!(ctx.signals_received.load(), 0); - - let test_fut = async move { - signal_tx.send(OverseerSignal::Conclude).await.unwrap(); - assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Signal(OverseerSignal::Conclude)); - - assert_eq!(ctx.signals_received.load(), 1); - bounded_tx.send(MessagePacket { - signals_received: 2, - message: (), - }).await.unwrap(); - unbounded_tx.unbounded_send(MessagePacket { - signals_received: 2, - message: (), - }).unwrap(); - - match poll!(ctx.recv()) { - Poll::Pending => {} - Poll::Ready(_) => panic!("ready too early"), - }; - - assert!(ctx.pending_incoming.is_some()); - - signal_tx.send(OverseerSignal::Conclude).await.unwrap(); - assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Signal(OverseerSignal::Conclude)); - assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Communication { msg: () }); - assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Communication { msg: () }); - assert!(ctx.pending_incoming.is_none()); - }; - - futures::executor::block_on(test_fut); - } -} +mod tests; diff --git a/node/overseer/src/tests.rs b/node/overseer/src/tests.rs new file mode 100644 index 000000000000..0fb65958c078 --- /dev/null +++ b/node/overseer/src/tests.rs @@ -0,0 +1,1001 @@ +use std::sync::atomic; +use std::collections::HashMap; +use futures::{executor, pin_mut, select, FutureExt, pending}; + +use polkadot_primitives::v1::{CollatorPair, CandidateHash}; +use polkadot_subsystem::{messages::RuntimeApiRequest, messages::NetworkBridgeEvent, jaeger}; +use polkadot_node_primitives::{CollationResult, CollationGenerationConfig, PoV, BlockData}; +use polkadot_node_network_protocol::{PeerId, UnifiedReputationChange}; +use polkadot_node_subsystem_util::metered; + +use sp_core::crypto::Pair as _; +use assert_matches::assert_matches; + +use super::*; + +struct TestSubsystem1(metered::MeteredSender); + +impl Subsystem for TestSubsystem1 + where C: SubsystemContext +{ + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let mut sender = self.0; + SpawnedSubsystem { + name: "test-subsystem-1", + future: Box::pin(async move { + let mut i = 0; + loop { + match ctx.recv().await { + Ok(FromOverseer::Communication { .. }) => { + let _ = sender.send(i).await; + i += 1; + continue; + } + Ok(FromOverseer::Signal(OverseerSignal::Conclude)) => return Ok(()), + Err(_) => return Ok(()), + _ => (), + } + } + }), + } + } +} + +struct TestSubsystem2(metered::MeteredSender); + +impl Subsystem for TestSubsystem2 + where C: SubsystemContext +{ + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let sender = self.0.clone(); + SpawnedSubsystem { + name: "test-subsystem-2", + future: Box::pin(async move { + let _sender = sender; + let mut c: usize = 0; + loop { + if c < 10 { + let (tx, _) = oneshot::channel(); + ctx.send_message( + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromChainState( + Default::default(), + PoV { + block_data: BlockData(Vec::new()), + }.into(), + tx, + ) + ) + ).await; + c += 1; + continue; + } + match ctx.try_recv().await { + Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => { + break; + } + Ok(Some(_)) => { + continue; + } + Err(_) => return Ok(()), + _ => (), + } + pending!(); + } + + Ok(()) + }), + } + } +} + +struct ReturnOnStart; + +impl Subsystem for ReturnOnStart + where C: SubsystemContext +{ + fn start(self, mut _ctx: C) -> SpawnedSubsystem { + SpawnedSubsystem { + name: "test-subsystem-4", + future: Box::pin(async move { + // Do nothing and exit. + Ok(()) + }), + } + } +} + +struct MockSupportsParachains; + +impl HeadSupportsParachains for MockSupportsParachains { + fn head_supports_parachains(&self, _head: &Hash) -> bool { + true + } +} + +// Checks that a minimal configuration of two jobs can run and exchange messages. +#[test] +fn overseer_works() { + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let (s1_tx, s1_rx) = metered::channel::(64); + let (s2_tx, s2_rx) = metered::channel::(64); + + let mut s1_rx = s1_rx.fuse(); + let mut s2_rx = s2_rx.fuse(); + + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_validation(TestSubsystem1(s1_tx)) + .replace_candidate_backing(TestSubsystem2(s2_tx)); + + let (overseer, mut handler) = Overseer::new( + vec![], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + let overseer_fut = overseer.run().fuse(); + + pin_mut!(overseer_fut); + + let mut s1_results = Vec::new(); + let mut s2_results = Vec::new(); + + loop { + select! { + _ = overseer_fut => break, + s1_next = s1_rx.next() => { + match s1_next { + Some(msg) => { + s1_results.push(msg); + if s1_results.len() == 10 { + handler.stop().await; + } + } + None => break, + } + }, + s2_next = s2_rx.next() => { + match s2_next { + Some(_) => s2_results.push(s2_next), + None => break, + } + }, + complete => break, + } + } + + assert_eq!(s1_results, (0..10).collect::>()); + }); +} + +// Checks activated/deactivated metrics are updated properly. +#[test] +fn overseer_metrics_work() { + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let first_block_hash = [1; 32].into(); + let second_block_hash = [2; 32].into(); + let third_block_hash = [3; 32].into(); + + let first_block = BlockInfo { + hash: first_block_hash, + parent_hash: [0; 32].into(), + number: 1, + }; + let second_block = BlockInfo { + hash: second_block_hash, + parent_hash: first_block_hash, + number: 2, + }; + let third_block = BlockInfo { + hash: third_block_hash, + parent_hash: second_block_hash, + number: 3, + }; + + let all_subsystems = AllSubsystems::<()>::dummy(); + let registry = prometheus::Registry::new(); + let (overseer, mut handler) = Overseer::new( + vec![first_block], + all_subsystems, + Some(®istry), + MockSupportsParachains, + spawner, + ).unwrap(); + let overseer_fut = overseer.run().fuse(); + + pin_mut!(overseer_fut); + + handler.block_imported(second_block).await; + handler.block_imported(third_block).await; + handler.send_msg(AllMessages::CandidateValidation(test_candidate_validation_msg())).await; + handler.stop().await; + + select! { + res = overseer_fut => { + assert!(res.is_ok()); + let metrics = extract_metrics(®istry); + assert_eq!(metrics["activated"], 3); + assert_eq!(metrics["deactivated"], 2); + assert_eq!(metrics["relayed"], 1); + }, + complete => (), + } + }); +} + +fn extract_metrics(registry: &prometheus::Registry) -> HashMap<&'static str, u64> { + let gather = registry.gather(); + assert_eq!(gather[0].get_name(), "parachain_activated_heads_total"); + assert_eq!(gather[1].get_name(), "parachain_deactivated_heads_total"); + assert_eq!(gather[2].get_name(), "parachain_messages_relayed_total"); + let activated = gather[0].get_metric()[0].get_counter().get_value() as u64; + let deactivated = gather[1].get_metric()[0].get_counter().get_value() as u64; + let relayed = gather[2].get_metric()[0].get_counter().get_value() as u64; + let mut result = HashMap::new(); + result.insert("activated", activated); + result.insert("deactivated", deactivated); + result.insert("relayed", relayed); + result +} + +// Spawn a subsystem that immediately exits. +// +// Should immediately conclude the overseer itself. +#[test] +fn overseer_ends_on_subsystem_exit() { + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_backing(ReturnOnStart); + let (overseer, _handle) = Overseer::new( + vec![], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + + overseer.run().await.unwrap(); + }) +} + +struct TestSubsystem5(metered::MeteredSender); + +impl Subsystem for TestSubsystem5 + where C: SubsystemContext +{ + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let mut sender = self.0.clone(); + + SpawnedSubsystem { + name: "test-subsystem-5", + future: Box::pin(async move { + loop { + match ctx.try_recv().await { + Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => break, + Ok(Some(FromOverseer::Signal(s))) => { + sender.send(s).await.unwrap(); + continue; + }, + Ok(Some(_)) => continue, + Err(_) => break, + _ => (), + } + pending!(); + } + + Ok(()) + }), + } + } +} + +struct TestSubsystem6(metered::MeteredSender); + +impl Subsystem for TestSubsystem6 + where C: SubsystemContext +{ + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let mut sender = self.0.clone(); + + SpawnedSubsystem { + name: "test-subsystem-6", + future: Box::pin(async move { + loop { + match ctx.try_recv().await { + Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => break, + Ok(Some(FromOverseer::Signal(s))) => { + sender.send(s).await.unwrap(); + continue; + }, + Ok(Some(_)) => continue, + Err(_) => break, + _ => (), + } + pending!(); + } + + Ok(()) + }), + } + } +} + +// Tests that starting with a defined set of leaves and receiving +// notifications on imported blocks triggers expected `StartWork` and `StopWork` heartbeats. +#[test] +fn overseer_start_stop_works() { + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let first_block_hash = [1; 32].into(); + let second_block_hash = [2; 32].into(); + let third_block_hash = [3; 32].into(); + + let first_block = BlockInfo { + hash: first_block_hash, + parent_hash: [0; 32].into(), + number: 1, + }; + let second_block = BlockInfo { + hash: second_block_hash, + parent_hash: first_block_hash, + number: 2, + }; + let third_block = BlockInfo { + hash: third_block_hash, + parent_hash: second_block_hash, + number: 3, + }; + + let (tx_5, mut rx_5) = metered::channel(64); + let (tx_6, mut rx_6) = metered::channel(64); + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_validation(TestSubsystem5(tx_5)) + .replace_candidate_backing(TestSubsystem6(tx_6)); + let (overseer, mut handler) = Overseer::new( + vec![first_block], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + + let overseer_fut = overseer.run().fuse(); + pin_mut!(overseer_fut); + + let mut ss5_results = Vec::new(); + let mut ss6_results = Vec::new(); + + handler.block_imported(second_block).await; + handler.block_imported(third_block).await; + + let expected_heartbeats = vec![ + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: first_block_hash, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: [ActivatedLeaf { + hash: second_block_hash, + number: 2, + span: Arc::new(jaeger::Span::Disabled), + }].as_ref().into(), + deactivated: [first_block_hash].as_ref().into(), + }), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: [ActivatedLeaf { + hash: third_block_hash, + number: 3, + span: Arc::new(jaeger::Span::Disabled), + }].as_ref().into(), + deactivated: [second_block_hash].as_ref().into(), + }), + ]; + + loop { + select! { + res = overseer_fut => { + assert!(res.is_ok()); + break; + }, + res = rx_5.next() => { + if let Some(res) = res { + ss5_results.push(res); + } + } + res = rx_6.next() => { + if let Some(res) = res { + ss6_results.push(res); + } + } + complete => break, + } + + if ss5_results.len() == expected_heartbeats.len() && + ss6_results.len() == expected_heartbeats.len() { + handler.stop().await; + } + } + + assert_eq!(ss5_results, expected_heartbeats); + assert_eq!(ss6_results, expected_heartbeats); + }); +} + +// Tests that starting with a defined set of leaves and receiving +// notifications on imported blocks triggers expected `StartWork` and `StopWork` heartbeats. +#[test] +fn overseer_finalize_works() { + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let first_block_hash = [1; 32].into(); + let second_block_hash = [2; 32].into(); + let third_block_hash = [3; 32].into(); + + let first_block = BlockInfo { + hash: first_block_hash, + parent_hash: [0; 32].into(), + number: 1, + }; + let second_block = BlockInfo { + hash: second_block_hash, + parent_hash: [42; 32].into(), + number: 2, + }; + let third_block = BlockInfo { + hash: third_block_hash, + parent_hash: second_block_hash, + number: 3, + }; + + let (tx_5, mut rx_5) = metered::channel(64); + let (tx_6, mut rx_6) = metered::channel(64); + + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_validation(TestSubsystem5(tx_5)) + .replace_candidate_backing(TestSubsystem6(tx_6)); + + // start with two forks of different height. + let (overseer, mut handler) = Overseer::new( + vec![first_block, second_block], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + + let overseer_fut = overseer.run().fuse(); + pin_mut!(overseer_fut); + + let mut ss5_results = Vec::new(); + let mut ss6_results = Vec::new(); + + // this should stop work on both forks we started with earlier. + handler.block_finalized(third_block).await; + + let expected_heartbeats = vec![ + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: [ + ActivatedLeaf { + hash: first_block_hash, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + }, + ActivatedLeaf { + hash: second_block_hash, + number: 2, + span: Arc::new(jaeger::Span::Disabled), + }, + ].as_ref().into(), + ..Default::default() + }), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + deactivated: [first_block_hash, second_block_hash].as_ref().into(), + ..Default::default() + }), + OverseerSignal::BlockFinalized(third_block_hash, 3), + ]; + + loop { + select! { + res = overseer_fut => { + assert!(res.is_ok()); + break; + }, + res = rx_5.next() => { + if let Some(res) = res { + ss5_results.push(res); + } + } + res = rx_6.next() => { + if let Some(res) = res { + ss6_results.push(res); + } + } + complete => break, + } + + if ss5_results.len() == expected_heartbeats.len() && ss6_results.len() == expected_heartbeats.len() { + handler.stop().await; + } + } + + assert_eq!(ss5_results.len(), expected_heartbeats.len()); + assert_eq!(ss6_results.len(), expected_heartbeats.len()); + + // Notifications on finality for multiple blocks at once + // may be received in different orders. + for expected in expected_heartbeats { + assert!(ss5_results.contains(&expected)); + assert!(ss6_results.contains(&expected)); + } + }); +} + +#[test] +fn do_not_send_empty_leaves_update_on_block_finalization() { + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let imported_block = BlockInfo { + hash: Hash::random(), + parent_hash: Hash::random(), + number: 1, + }; + + let finalized_block = BlockInfo { + hash: Hash::random(), + parent_hash: Hash::random(), + number: 1, + }; + + let (tx_5, mut rx_5) = metered::channel(64); + + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_backing(TestSubsystem6(tx_5)); + + let (overseer, mut handler) = Overseer::new( + Vec::new(), + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + + let overseer_fut = overseer.run().fuse(); + pin_mut!(overseer_fut); + + let mut ss5_results = Vec::new(); + + handler.block_finalized(finalized_block.clone()).await; + handler.block_imported(imported_block.clone()).await; + + let expected_heartbeats = vec![ + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: [ + ActivatedLeaf { + hash: imported_block.hash, + number: imported_block.number, + span: Arc::new(jaeger::Span::Disabled) + } + ].as_ref().into(), + ..Default::default() + }), + OverseerSignal::BlockFinalized(finalized_block.hash, 1), + ]; + + loop { + select! { + res = overseer_fut => { + assert!(res.is_ok()); + break; + }, + res = rx_5.next() => { + if let Some(res) = dbg!(res) { + ss5_results.push(res); + } + } + } + + if ss5_results.len() == expected_heartbeats.len() { + handler.stop().await; + } + } + + assert_eq!(ss5_results.len(), expected_heartbeats.len()); + + for expected in expected_heartbeats { + assert!(ss5_results.contains(&expected)); + } + }); +} + +#[derive(Clone)] +struct CounterSubsystem { + stop_signals_received: Arc, + signals_received: Arc, + msgs_received: Arc, +} + +impl CounterSubsystem { + fn new( + stop_signals_received: Arc, + signals_received: Arc, + msgs_received: Arc, + ) -> Self { + Self { + stop_signals_received, + signals_received, + msgs_received, + } + } +} + +impl Subsystem for CounterSubsystem + where + C: SubsystemContext, + M: Send, +{ + fn start(self, mut ctx: C) -> SpawnedSubsystem { + SpawnedSubsystem { + name: "counter-subsystem", + future: Box::pin(async move { + loop { + match ctx.try_recv().await { + Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => { + self.stop_signals_received.fetch_add(1, atomic::Ordering::SeqCst); + break; + }, + Ok(Some(FromOverseer::Signal(_))) => { + self.signals_received.fetch_add(1, atomic::Ordering::SeqCst); + continue; + }, + Ok(Some(FromOverseer::Communication { .. })) => { + self.msgs_received.fetch_add(1, atomic::Ordering::SeqCst); + continue; + }, + Err(_) => (), + _ => (), + } + pending!(); + } + + Ok(()) + }), + } + } +} + +fn test_candidate_validation_msg() -> CandidateValidationMessage { + let (sender, _) = oneshot::channel(); + let pov = Arc::new(PoV { block_data: BlockData(Vec::new()) }); + CandidateValidationMessage::ValidateFromChainState(Default::default(), pov, sender) +} + +fn test_candidate_backing_msg() -> CandidateBackingMessage { + let (sender, _) = oneshot::channel(); + CandidateBackingMessage::GetBackedCandidates(Default::default(), Vec::new(), sender) +} + +fn test_candidate_selection_msg() -> CandidateSelectionMessage { + CandidateSelectionMessage::default() +} + +fn test_chain_api_msg() -> ChainApiMessage { + let (sender, _) = oneshot::channel(); + ChainApiMessage::FinalizedBlockNumber(sender) +} + +fn test_collator_generation_msg() -> CollationGenerationMessage { + CollationGenerationMessage::Initialize(CollationGenerationConfig { + key: CollatorPair::generate().0, + collator: Box::new(|_, _| TestCollator.boxed()), + para_id: Default::default(), + }) +} +struct TestCollator; + +impl Future for TestCollator { + type Output = Option; + + fn poll(self: Pin<&mut Self>, _cx: &mut futures::task::Context) -> Poll { + panic!("at the Disco") + } +} + +impl Unpin for TestCollator {} + +fn test_collator_protocol_msg() -> CollatorProtocolMessage { + CollatorProtocolMessage::CollateOn(Default::default()) +} + +fn test_network_bridge_event() -> NetworkBridgeEvent { + NetworkBridgeEvent::PeerDisconnected(PeerId::random()) +} + +fn test_statement_distribution_msg() -> StatementDistributionMessage { + StatementDistributionMessage::NetworkBridgeUpdateV1(test_network_bridge_event()) +} + +fn test_availability_recovery_msg() -> AvailabilityRecoveryMessage { + let (sender, _) = oneshot::channel(); + AvailabilityRecoveryMessage::RecoverAvailableData( + Default::default(), + Default::default(), + None, + sender, + ) +} + +fn test_bitfield_distribution_msg() -> BitfieldDistributionMessage { + BitfieldDistributionMessage::NetworkBridgeUpdateV1(test_network_bridge_event()) +} + +fn test_provisioner_msg() -> ProvisionerMessage { + let (sender, _) = oneshot::channel(); + ProvisionerMessage::RequestInherentData(Default::default(), sender) +} + +fn test_runtime_api_msg() -> RuntimeApiMessage { + let (sender, _) = oneshot::channel(); + RuntimeApiMessage::Request(Default::default(), RuntimeApiRequest::Validators(sender)) +} + +fn test_availability_store_msg() -> AvailabilityStoreMessage { + let (sender, _) = oneshot::channel(); + AvailabilityStoreMessage::QueryAvailableData(CandidateHash(Default::default()), sender) +} + +fn test_network_bridge_msg() -> NetworkBridgeMessage { + NetworkBridgeMessage::ReportPeer(PeerId::random(), UnifiedReputationChange::BenefitMinor("")) +} + +fn test_approval_distribution_msg() -> ApprovalDistributionMessage { + ApprovalDistributionMessage::NewBlocks(Default::default()) +} + +fn test_approval_voting_msg() -> ApprovalVotingMessage { + let (sender, _) = oneshot::channel(); + ApprovalVotingMessage::ApprovedAncestor(Default::default(), 0, sender) +} + +// Checks that `stop`, `broadcast_signal` and `broadcast_message` are implemented correctly. +#[test] +fn overseer_all_subsystems_receive_signals_and_messages() { + const NUM_SUBSYSTEMS: usize = 18; + // -3 for BitfieldSigning, GossipSupport and AvailabilityDistribution + const NUM_SUBSYSTEMS_MESSAGED: usize = NUM_SUBSYSTEMS - 3; + + let spawner = sp_core::testing::TaskExecutor::new(); + executor::block_on(async move { + let stop_signals_received = Arc::new(atomic::AtomicUsize::new(0)); + let signals_received = Arc::new(atomic::AtomicUsize::new(0)); + let msgs_received = Arc::new(atomic::AtomicUsize::new(0)); + + let subsystem = CounterSubsystem::new( + stop_signals_received.clone(), + signals_received.clone(), + msgs_received.clone(), + ); + + let all_subsystems = AllSubsystems { + candidate_validation: subsystem.clone(), + candidate_backing: subsystem.clone(), + candidate_selection: subsystem.clone(), + collation_generation: subsystem.clone(), + collator_protocol: subsystem.clone(), + statement_distribution: subsystem.clone(), + availability_distribution: subsystem.clone(), + availability_recovery: subsystem.clone(), + bitfield_signing: subsystem.clone(), + bitfield_distribution: subsystem.clone(), + provisioner: subsystem.clone(), + runtime_api: subsystem.clone(), + availability_store: subsystem.clone(), + network_bridge: subsystem.clone(), + chain_api: subsystem.clone(), + approval_distribution: subsystem.clone(), + approval_voting: subsystem.clone(), + gossip_support: subsystem.clone(), + }; + let (overseer, mut handler) = Overseer::new( + vec![], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + let overseer_fut = overseer.run().fuse(); + + pin_mut!(overseer_fut); + + // send a signal to each subsystem + handler.block_imported(BlockInfo { + hash: Default::default(), + parent_hash: Default::default(), + number: Default::default(), + }).await; + + // send a msg to each subsystem + // except for BitfieldSigning and GossipSupport as the messages are not instantiable + handler.send_msg(AllMessages::CandidateValidation(test_candidate_validation_msg())).await; + handler.send_msg(AllMessages::CandidateBacking(test_candidate_backing_msg())).await; + handler.send_msg(AllMessages::CandidateSelection(test_candidate_selection_msg())).await; + handler.send_msg(AllMessages::CollationGeneration(test_collator_generation_msg())).await; + handler.send_msg(AllMessages::CollatorProtocol(test_collator_protocol_msg())).await; + handler.send_msg(AllMessages::StatementDistribution(test_statement_distribution_msg())).await; + handler.send_msg(AllMessages::AvailabilityRecovery(test_availability_recovery_msg())).await; + // handler.send_msg(AllMessages::BitfieldSigning(test_bitfield_signing_msg())).await; + // handler.send_msg(AllMessages::GossipSupport(test_bitfield_signing_msg())).await; + handler.send_msg(AllMessages::BitfieldDistribution(test_bitfield_distribution_msg())).await; + handler.send_msg(AllMessages::Provisioner(test_provisioner_msg())).await; + handler.send_msg(AllMessages::RuntimeApi(test_runtime_api_msg())).await; + handler.send_msg(AllMessages::AvailabilityStore(test_availability_store_msg())).await; + handler.send_msg(AllMessages::NetworkBridge(test_network_bridge_msg())).await; + handler.send_msg(AllMessages::ChainApi(test_chain_api_msg())).await; + handler.send_msg(AllMessages::ApprovalDistribution(test_approval_distribution_msg())).await; + handler.send_msg(AllMessages::ApprovalVoting(test_approval_voting_msg())).await; + + // Wait until all subsystems have received. Otherwise the messages might race against + // the conclude signal. + loop { + match (&mut overseer_fut).timeout(Duration::from_millis(100)).await { + None => { + let r = msgs_received.load(atomic::Ordering::SeqCst); + if r < NUM_SUBSYSTEMS_MESSAGED { + Delay::new(Duration::from_millis(100)).await; + } else if r > NUM_SUBSYSTEMS_MESSAGED { + panic!("too many messages received??"); + } else { + break + } + } + Some(_) => panic!("exited too early"), + } + } + + // send a stop signal to each subsystems + handler.stop().await; + + let res = overseer_fut.await; + assert_eq!(stop_signals_received.load(atomic::Ordering::SeqCst), NUM_SUBSYSTEMS); + assert_eq!(signals_received.load(atomic::Ordering::SeqCst), NUM_SUBSYSTEMS); + assert_eq!(msgs_received.load(atomic::Ordering::SeqCst), NUM_SUBSYSTEMS_MESSAGED); + + assert!(res.is_ok()); + }); +} + +#[test] +fn context_holds_onto_message_until_enough_signals_received() { + let (candidate_validation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (candidate_backing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (candidate_selection_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (statement_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (availability_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (availability_recovery_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (bitfield_signing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (bitfield_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (provisioner_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (runtime_api_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (availability_store_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (network_bridge_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (chain_api_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (collator_protocol_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (collation_generation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (approval_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (approval_voting_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (gossip_support_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + + let (candidate_validation_unbounded_tx, _) = metered::unbounded(); + let (candidate_backing_unbounded_tx, _) = metered::unbounded(); + let (candidate_selection_unbounded_tx, _) = metered::unbounded(); + let (statement_distribution_unbounded_tx, _) = metered::unbounded(); + let (availability_distribution_unbounded_tx, _) = metered::unbounded(); + let (availability_recovery_unbounded_tx, _) = metered::unbounded(); + let (bitfield_signing_unbounded_tx, _) = metered::unbounded(); + let (bitfield_distribution_unbounded_tx, _) = metered::unbounded(); + let (provisioner_unbounded_tx, _) = metered::unbounded(); + let (runtime_api_unbounded_tx, _) = metered::unbounded(); + let (availability_store_unbounded_tx, _) = metered::unbounded(); + let (network_bridge_unbounded_tx, _) = metered::unbounded(); + let (chain_api_unbounded_tx, _) = metered::unbounded(); + let (collator_protocol_unbounded_tx, _) = metered::unbounded(); + let (collation_generation_unbounded_tx, _) = metered::unbounded(); + let (approval_distribution_unbounded_tx, _) = metered::unbounded(); + let (approval_voting_unbounded_tx, _) = metered::unbounded(); + let (gossip_support_unbounded_tx, _) = metered::unbounded(); + + let channels_out = ChannelsOut { + candidate_validation: candidate_validation_bounded_tx.clone(), + candidate_backing: candidate_backing_bounded_tx.clone(), + candidate_selection: candidate_selection_bounded_tx.clone(), + statement_distribution: statement_distribution_bounded_tx.clone(), + availability_distribution: availability_distribution_bounded_tx.clone(), + availability_recovery: availability_recovery_bounded_tx.clone(), + bitfield_signing: bitfield_signing_bounded_tx.clone(), + bitfield_distribution: bitfield_distribution_bounded_tx.clone(), + provisioner: provisioner_bounded_tx.clone(), + runtime_api: runtime_api_bounded_tx.clone(), + availability_store: availability_store_bounded_tx.clone(), + network_bridge: network_bridge_bounded_tx.clone(), + chain_api: chain_api_bounded_tx.clone(), + collator_protocol: collator_protocol_bounded_tx.clone(), + collation_generation: collation_generation_bounded_tx.clone(), + approval_distribution: approval_distribution_bounded_tx.clone(), + approval_voting: approval_voting_bounded_tx.clone(), + gossip_support: gossip_support_bounded_tx.clone(), + + candidate_validation_unbounded: candidate_validation_unbounded_tx.clone(), + candidate_backing_unbounded: candidate_backing_unbounded_tx.clone(), + candidate_selection_unbounded: candidate_selection_unbounded_tx.clone(), + statement_distribution_unbounded: statement_distribution_unbounded_tx.clone(), + availability_distribution_unbounded: availability_distribution_unbounded_tx.clone(), + availability_recovery_unbounded: availability_recovery_unbounded_tx.clone(), + bitfield_signing_unbounded: bitfield_signing_unbounded_tx.clone(), + bitfield_distribution_unbounded: bitfield_distribution_unbounded_tx.clone(), + provisioner_unbounded: provisioner_unbounded_tx.clone(), + runtime_api_unbounded: runtime_api_unbounded_tx.clone(), + availability_store_unbounded: availability_store_unbounded_tx.clone(), + network_bridge_unbounded: network_bridge_unbounded_tx.clone(), + chain_api_unbounded: chain_api_unbounded_tx.clone(), + collator_protocol_unbounded: collator_protocol_unbounded_tx.clone(), + collation_generation_unbounded: collation_generation_unbounded_tx.clone(), + approval_distribution_unbounded: approval_distribution_unbounded_tx.clone(), + approval_voting_unbounded: approval_voting_unbounded_tx.clone(), + gossip_support_unbounded: gossip_support_unbounded_tx.clone(), + }; + + let (mut signal_tx, signal_rx) = metered::channel(CHANNEL_CAPACITY); + let (mut bounded_tx, bounded_rx) = metered::channel(CHANNEL_CAPACITY); + let (unbounded_tx, unbounded_rx) = metered::unbounded(); + let (to_overseer_tx, _to_overseer_rx) = metered::unbounded(); + + let mut ctx = OverseerSubsystemContext::<()>::new_unmetered( + signal_rx, + stream::select(bounded_rx, unbounded_rx), + channels_out, + to_overseer_tx, + ); + + assert_eq!(ctx.signals_received.load(), 0); + + let test_fut = async move { + signal_tx.send(OverseerSignal::Conclude).await.unwrap(); + assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Signal(OverseerSignal::Conclude)); + + assert_eq!(ctx.signals_received.load(), 1); + bounded_tx.send(MessagePacket { + signals_received: 2, + message: (), + }).await.unwrap(); + unbounded_tx.unbounded_send(MessagePacket { + signals_received: 2, + message: (), + }).unwrap(); + + match poll!(ctx.recv()) { + Poll::Pending => {} + Poll::Ready(_) => panic!("ready too early"), + }; + + assert!(ctx.pending_incoming.is_some()); + + signal_tx.send(OverseerSignal::Conclude).await.unwrap(); + assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Signal(OverseerSignal::Conclude)); + assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Communication { msg: () }); + assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Communication { msg: () }); + assert!(ctx.pending_incoming.is_none()); + }; + + futures::executor::block_on(test_fut); +} From d484fde9924bf772df3cec4842a3019d19a22dba Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 10 May 2021 15:01:58 +0200 Subject: [PATCH 013/161] foo --- .../overseer-gen/src/impl_channels_out.rs | 34 ++++---- .../overseer-gen/src/impl_overseer.rs | 81 +++++++++---------- node/overseer/overseer-gen/src/impls.rs | 8 +- node/overseer/overseer-gen/src/inc/static.rs | 23 ++++++ .../ui/{ok-02-boring.rs => ok-01-boring.rs} | 0 ...3-w-generic copy.rs => ok-02-w-generic.rs} | 2 +- 6 files changed, 85 insertions(+), 63 deletions(-) rename node/overseer/overseer-gen/tests/ui/{ok-02-boring.rs => ok-01-boring.rs} (100%) rename node/overseer/overseer-gen/tests/ui/{ok-03-w-generic copy.rs => ok-02-w-generic.rs} (96%) diff --git a/node/overseer/overseer-gen/src/impl_channels_out.rs b/node/overseer/overseer-gen/src/impl_channels_out.rs index ca35a3a6e9cc..7babd84c8176 100644 --- a/node/overseer/overseer-gen/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/src/impl_channels_out.rs @@ -15,17 +15,18 @@ use super::*; /// Implement the helper type `ChannelsOut` and `MessagePacket`. pub(crate) fn impl_channels_out_struct( - wrapper: &Ident, + message_wrapper: &Ident, subsystems: &[SubSysField], baggage: &[BaggageField], ) -> Result { - let mut channel_name = subsystems.iter().map(|ssf| - ssf.name.clone()); - let mut channel_name_unbounded = subsystems.iter().map(|ssf| - Ident::new(ssf.name.span(), ssf.name.to_string() + "_unbounded") - ); - let mut field_ty = subsystems.iter().map(|ssf| ssf.ty.clone()); - let ts = quote! { + let mut channel_name = &subsystems.iter().map(|ssf| ssf.name.clone()).collect::>(); + let mut channel_name_unbounded = &subsystems.iter().map(|ssf| + Ident::new( &(ssf.name.to_string() + "_unbounded"), ssf.name.span()) + ).collect::>(); + + let mut field_ty = &subsystems.iter().map(|ssf| ssf.ty.clone()).collect::>(); + + let ts = quote! { #[derive(Debug)] struct MessagePacket { signals_received: usize, @@ -52,14 +53,16 @@ pub(crate) fn impl_channels_out_struct( async pub fn send_and_log_error( &mut self, signals_received: usize, - message: #wrapper, + message: #message_wrapper, ) { let res = match message { #( - #wrapper :: #field_ty (msg) => { - self. #channel_name .send(make_packet(signals_received, msg)).await + #message_wrapper :: #field_ty (msg) => { + self. #channel_name .send( + make_packet(signals_received, msg) + ).await }, - ) + )* }; if res.is_err() { @@ -70,7 +73,6 @@ pub(crate) fn impl_channels_out_struct( } } - pub fn send_unbounded_and_log_error( &self, signals_received: usize, @@ -78,16 +80,16 @@ pub(crate) fn impl_channels_out_struct( ) { let res = match message { #( - #wrapper :: #field_ty (msg) => { + #message_wrapper :: #field_ty (msg) => { self. #channel_name_unbounded .send( make_packet(signals_received, msg) ).await }, - ) + )* }; if res.is_err() { - tracing::debug!( + ::tracing::debug!( target: LOG_TARGET, "Failed to send a message to another subsystem", ); diff --git a/node/overseer/overseer-gen/src/impl_overseer.rs b/node/overseer/overseer-gen/src/impl_overseer.rs index 1f8fb68f828f..7f4d9e14e11f 100644 --- a/node/overseer/overseer-gen/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/src/impl_overseer.rs @@ -7,12 +7,15 @@ use syn::AttrStyle; use syn::Field; use syn::FieldsNamed; use syn::Variant; +use syn::Generics; use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; use super::*; + pub(crate) fn impl_overseer_struct( - overseer_name: Ident, + overseer_name: &Ident, + message_wrapper: &Ident, orig_generics: Generics, subsystems: &[SubSysField], baggage: &[BaggageField], @@ -21,10 +24,10 @@ pub(crate) fn impl_overseer_struct( let mut field_ty = &subsystems.iter().map(|ssf| ssf.generic.clone()).collect::>(); - let mut baggage_name = baggage.iter().map(|bf| bf.field_name.clone()); - let mut baggage_ty = baggage.iter().map(|bf| bf.field_ty.clone()); + let mut baggage_name = &baggage.iter().map(|bf| bf.field_name.clone()).collect::>(); + let mut baggage_ty = &baggage.iter().map(|bf| bf.field_ty.clone()).collect::>(); - let mut baggage_generic_ty = baggage.iter().filter(|bf| bf.generic).map(|bf| bf.field_ty.clone()); + let mut baggage_generic_ty = &baggage.iter().filter(|bf| bf.generic).map(|bf| bf.field_ty.clone()).collect::>(); let generics = quote! { < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > @@ -75,7 +78,7 @@ pub(crate) fn impl_overseer_struct( Ok(()) } - async pub fn route_message(&mut self, msg: #wrapper) -> SubsystemResult<()> { + async pub fn route_message(&mut self, msg: #message_wrapper) -> SubsystemResult<()> { match msg { #( #field_ty (msg) => self. #field_name .send_message(msg).await?, @@ -85,14 +88,14 @@ pub(crate) fn impl_overseer_struct( } }; - x.extend(crate::builder::impl_builder(overseer_name, subsystems, baggage)?); + x.extend(impl_builder(overseer_name, subsystems, baggage)?); Ok(x) } /// Implement a builder pattern. pub(crate) fn impl_builder( - name: Ident, + name: &Ident, subsystems: &[SubSysField], baggage: &[BaggageField], ) -> Result { @@ -101,31 +104,31 @@ pub(crate) fn impl_builder( let overseer = name.clone(); let handler = Ident::new(&(overseer.to_string() + "Handler"), overseer.span()); - let mut field_name = &subsystems.iter().map(|x| x.name.clone()).collect::>(); - let mut field_ty = &subsystems.iter().map(|x| x.generic.clone()).collect::>(); + let field_name = &subsystems.iter().map(|x| x.name.clone()).collect::>(); + let field_ty = &subsystems.iter().map(|x| x.generic.clone()).collect::>(); - let mut channel_name = subsystems.iter().map(|ssf| + let channel_name = subsystems.iter().map(|ssf| ssf.name.clone()); - let mut channel_name_unbounded = subsystems.iter().map(|ssf| - Ident::new(ssf.name.span(), ssf.name.to_string() + "_unbounded") - ); - let mut channel_name_tx = subsystems.iter().map(|ssf| - Ident::new(ssf.name.span(), ssf.name.to_string() + "_tx") - ); - let mut channel_name_unbounded_tx = subsystems.iter().map(|ssf| - Ident::new(ssf.name.span(), ssf.name.to_string() + "_unbounded_tx") - ); - let mut channel_name_rx = subsystems.iter().map(|ssf| - Ident::new(ssf.name.span(), ssf.name.to_string() + "_rx") - ); - let mut channel_name_unbounded_rx = subsystems.iter().map(|ssf| - Ident::new(ssf.name.span(), ssf.name.to_string() + "_unbounded_rx") - ); - - let mut baggage_generic_ty = &baggage.iter().filter(|b| b.generic).map(|b| b.field_ty.clone()).collect::>(); - - let mut baggage_name = &baggage.iter().map(|x| x.field_name.clone()).collect::>(); - let mut baggage_ty = &baggage.iter().map(|x| x.field_ty.clone()).collect::>(); + let channel_name_unbounded = subsystems.iter().map(|ssf| + Ident::new(&(ssf.name.to_string() + "_unbounded"), ssf.name.span()) + ).collect::>(); + let channel_name_tx = &subsystems.iter().map(|ssf| + Ident::new(&(ssf.name.to_string() + "_tx"), ssf.name.span()) + ).collect::>(); + let channel_name_unbounded_tx = &subsystems.iter().map(|ssf| + Ident::new(&(ssf.name.to_string() + "_unbounded_tx"), ssf.name.span()) + ).collect::>(); + let channel_name_rx = &subsystems.iter().map(|ssf| + Ident::new(&(ssf.name.to_string() + "_rx"), ssf.name.span()) + ).collect::>(); + let channel_name_unbounded_rx = &subsystems.iter().map(|ssf| + Ident::new(&(ssf.name.to_string() + "_unbounded_rx"), ssf.name.span()) + ).collect::>(); + + let baggage_generic_ty = &baggage.iter().filter(|b| b.generic).map(|b| b.field_ty.clone()).collect::>(); + + let baggage_name = &baggage.iter().map(|x| x.field_name.clone()).collect::>(); + let baggage_ty = &baggage.iter().map(|x| x.field_ty.clone()).collect::>(); let generics = quote! { < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > @@ -136,7 +139,7 @@ pub(crate) fn impl_builder( #( #field_ty : Subsystem, )* }; - let x = quote! { + let ts = quote! { impl #generics #name #generics #where_clause { fn builder() -> #builder { @@ -174,7 +177,7 @@ pub(crate) fn impl_builder( const CHANNEL_CAPACITY: usize = 1024; const SIGNAL_CHANNEL_CAPACITY: usize = 64; - let (events_tx, events_rx) = #crate::metered::channel(SIGNAL_CHANNEL_CAPACITY); + let (events_tx, events_rx) = ::metered::channel(SIGNAL_CHANNEL_CAPACITY); let handler = #handler { events_tx: events_tx.clone(), @@ -197,20 +200,14 @@ pub(crate) fn impl_builder( let (#channel_name_unbounded_tx, #channel_name_unbounded_rx) = ::metered::unbounded::>(); )* - - let channels_out = ChannelsOut { + ChannelsOut { #( - pub #channel_name: #channel_name_tx .clone(), + channel_name: #channel_name_tx .clone(), )* #( - pub #channel_name_unbounded: #channel_name_tx_unbounded .clone(), + #channel_name_unbounded: #channel_name_unbounded_tx .clone(), )* } - - ChannelsOut { - // #( #bounded ) - // #( #unbounded ) - }; } @@ -220,5 +217,5 @@ pub(crate) fn impl_builder( } } }; - Ok(x) + Ok(ts) } diff --git a/node/overseer/overseer-gen/src/impls.rs b/node/overseer/overseer-gen/src/impls.rs index 389eea723f0b..735893f49f29 100644 --- a/node/overseer/overseer-gen/src/impls.rs +++ b/node/overseer/overseer-gen/src/impls.rs @@ -177,7 +177,7 @@ pub(crate) struct BaggageField { /// Generates the wrapper type enum. pub(crate) fn impl_messages_wrapper_enum( - messages_wrapper: Ident, + messages_wrapper: &Ident, subsystems: &[SubSysField], baggage: &[BaggageField], ) -> Result { @@ -232,10 +232,10 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< let (subsystems, baggage) = parse_overseer_struct_field(baggage_generic_idents, named)?; - let mut additive = impl_overseer_struct(overseer_name.clone(), orig_generics, &subsystems, &baggage)?; + let mut additive = impl_overseer_struct(&overseer_name, &message_wrapper, orig_generics, &subsystems, &baggage)?; - additive.extend(impl_messages_wrapper_enum(message_wrapper, &subsystems[..], &baggage[..])?); - additive.extend(impl_channels_out_struct(&subsystems[..], &baggage[..])?); + additive.extend(impl_messages_wrapper_enum(&message_wrapper, &subsystems[..], &baggage[..])?); + additive.extend(impl_channels_out_struct(&message_wrapper, &subsystems[..], &baggage[..])?); additive.extend(impl_replacable_subsystem(overseer_name, &subsystems[..], &baggage[..])); additive.extend(inc::include_static_rs()?); diff --git a/node/overseer/overseer-gen/src/inc/static.rs b/node/overseer/overseer-gen/src/inc/static.rs index ee9dd3450897..421ab3e2d911 100644 --- a/node/overseer/overseer-gen/src/inc/static.rs +++ b/node/overseer/overseer-gen/src/inc/static.rs @@ -58,3 +58,26 @@ impl SubsystemSender for OverseerSubsystemSender { self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); } } + +#[derive(Clone)] +struct SubsystemMeters { + bounded: metered::Meter, + unbounded: metered::Meter, + signals: metered::Meter, +} + +impl SubsystemMeters { + fn read(&self) -> SubsystemMeterReadouts { + SubsystemMeterReadouts { + bounded: self.bounded.read(), + unbounded: self.unbounded.read(), + signals: self.signals.read(), + } + } +} + +struct SubsystemMeterReadouts { + bounded: metered::Readout, + unbounded: metered::Readout, + signals: metered::Readout, +} diff --git a/node/overseer/overseer-gen/tests/ui/ok-02-boring.rs b/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs similarity index 100% rename from node/overseer/overseer-gen/tests/ui/ok-02-boring.rs rename to node/overseer/overseer-gen/tests/ui/ok-01-boring.rs diff --git a/node/overseer/overseer-gen/tests/ui/ok-03-w-generic copy.rs b/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs similarity index 96% rename from node/overseer/overseer-gen/tests/ui/ok-03-w-generic copy.rs rename to node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs index 833ed4e20c10..92de18c0602e 100644 --- a/node/overseer/overseer-gen/tests/ui/ok-03-w-generic copy.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs @@ -5,7 +5,7 @@ use polkadot_procmacro_overseer_gen::overlord; trait MetaMeta {} #[derive(Debug)] -struct MsgStrukt(u8) +struct MsgStrukt(u8); #[derive(Default, Clone, Copy)] struct AwesomeSubSys; From a6bfc20875c9df705bf514612ed9289b4f8b560b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 10 May 2021 17:22:05 +0200 Subject: [PATCH 014/161] refactor --- .../overseer-gen/src/impl_channels_out.rs | 28 +-- .../overseer-gen/src/impl_message_wrapper.rs | 25 +++ .../overseer-gen/src/impl_overseer.rs | 33 ++-- .../overseer/overseer-gen/src/impl_replace.rs | 28 +-- node/overseer/overseer-gen/src/inc.rs | 19 +- node/overseer/overseer-gen/src/lib.rs | 80 ++++++-- .../overseer-gen/src/{impls.rs => parse.rs} | 181 ++++++++---------- .../overseer-gen/tests/ui/ok-02-w-generic.rs | 4 +- 8 files changed, 209 insertions(+), 189 deletions(-) create mode 100644 node/overseer/overseer-gen/src/impl_message_wrapper.rs rename node/overseer/overseer-gen/src/{impls.rs => parse.rs} (62%) diff --git a/node/overseer/overseer-gen/src/impl_channels_out.rs b/node/overseer/overseer-gen/src/impl_channels_out.rs index 7babd84c8176..8c9235effff6 100644 --- a/node/overseer/overseer-gen/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/src/impl_channels_out.rs @@ -1,30 +1,20 @@ -use proc_macro2::{Span, TokenStream}; -use quote::quote; -use std::collections::HashSet; -use quote::ToTokens; -use syn::AttrStyle; -use syn::Generics; -use syn::Field; -use syn::FieldsNamed; -use syn::Variant; -use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; -use syn::spanned::Spanned; +use quote::quote; +use syn::{Ident, Result}; use super::*; /// Implement the helper type `ChannelsOut` and `MessagePacket`. pub(crate) fn impl_channels_out_struct( - message_wrapper: &Ident, - subsystems: &[SubSysField], - baggage: &[BaggageField], + info: &OverseerInfo, ) -> Result { - let mut channel_name = &subsystems.iter().map(|ssf| ssf.name.clone()).collect::>(); - let mut channel_name_unbounded = &subsystems.iter().map(|ssf| - Ident::new( &(ssf.name.to_string() + "_unbounded"), ssf.name.span()) - ).collect::>(); - let mut field_ty = &subsystems.iter().map(|ssf| ssf.ty.clone()).collect::>(); + let message_wrapper = info.message_wrapper.clone(); + + let channel_name = &info.channels(""); + let channel_name_unbounded = &info.channels("_unbounded"); + + let mut consumes = &info.subsystem.iter().map(|ssf| ssf.consumes.clone()).collect::>(); let ts = quote! { #[derive(Debug)] diff --git a/node/overseer/overseer-gen/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/src/impl_message_wrapper.rs new file mode 100644 index 000000000000..6bd7a2148e2f --- /dev/null +++ b/node/overseer/overseer-gen/src/impl_message_wrapper.rs @@ -0,0 +1,25 @@ +use quote::quote; +use syn::{Ident, Result}; + +use super::*; + +/// Generates the wrapper type enum. +pub(crate) fn impl_message_wrapper_enum( + info: &OverseerInfo, +) -> Result { + let consumes = info.subsystems.iter().map(|ssf| ssf.consumes.clone()).flatten(); + + let message_wrapper = info.message_wrapper; + + let msg = "Generated message type wrapper"; + let x = quote! { + #[doc = #msg] + #[derive(Debug, Clone)] + enum #messages_wrapper { + #( + #consumes ( #consumes ), + )* + } + }; + Ok(x) +} diff --git a/node/overseer/overseer-gen/src/impl_overseer.rs b/node/overseer/overseer-gen/src/impl_overseer.rs index 7f4d9e14e11f..89a44fba8e74 100644 --- a/node/overseer/overseer-gen/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/src/impl_overseer.rs @@ -1,39 +1,28 @@ -use proc_macro2::{Span, TokenStream}; +use proc_macro2::{Span}; use quote::quote; -use std::collections::HashSet; - -use quote::ToTokens; -use syn::AttrStyle; -use syn::Field; -use syn::FieldsNamed; -use syn::Variant; use syn::Generics; -use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; +use syn::{Ident, Result}; use super::*; pub(crate) fn impl_overseer_struct( - overseer_name: &Ident, - message_wrapper: &Ident, - orig_generics: Generics, - subsystems: &[SubSysField], - baggage: &[BaggageField], + info: &OverseerInfo, ) -> Result { - let mut field_name = &subsystems.iter().map(|ssf| ssf.name.clone()).collect::>(); + let field_name = &info.subsystem_names(); - let mut field_ty = &subsystems.iter().map(|ssf| ssf.generic.clone()).collect::>(); + let field_ty = &info.subsystem_generic_types(); - let mut baggage_name = &baggage.iter().map(|bf| bf.field_name.clone()).collect::>(); - let mut baggage_ty = &baggage.iter().map(|bf| bf.field_ty.clone()).collect::>(); + let baggage_name = &info.baggage_names(); + let baggage_ty = &info.baggage_types(); - let mut baggage_generic_ty = &baggage.iter().filter(|bf| bf.generic).map(|bf| bf.field_ty.clone()).collect::>(); + let baggage_generic_ty = &info.baggage_generic_types(); let generics = quote! { < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > }; - let where_clause = quote! { + let _where_clause = quote! { where Ctx: SubsystemContext, #( #field_ty : Subsystem )* @@ -107,7 +96,7 @@ pub(crate) fn impl_builder( let field_name = &subsystems.iter().map(|x| x.name.clone()).collect::>(); let field_ty = &subsystems.iter().map(|x| x.generic.clone()).collect::>(); - let channel_name = subsystems.iter().map(|ssf| + let _channel_name = subsystems.iter().map(|ssf| ssf.name.clone()); let channel_name_unbounded = subsystems.iter().map(|ssf| Ident::new(&(ssf.name.to_string() + "_unbounded"), ssf.name.span()) @@ -128,7 +117,7 @@ pub(crate) fn impl_builder( let baggage_generic_ty = &baggage.iter().filter(|b| b.generic).map(|b| b.field_ty.clone()).collect::>(); let baggage_name = &baggage.iter().map(|x| x.field_name.clone()).collect::>(); - let baggage_ty = &baggage.iter().map(|x| x.field_ty.clone()).collect::>(); + let _baggage_ty = &baggage.iter().map(|x| x.field_ty.clone()).collect::>(); let generics = quote! { < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > diff --git a/node/overseer/overseer-gen/src/impl_replace.rs b/node/overseer/overseer-gen/src/impl_replace.rs index b2366ec2273f..ee732f130c29 100644 --- a/node/overseer/overseer-gen/src/impl_replace.rs +++ b/node/overseer/overseer-gen/src/impl_replace.rs @@ -1,47 +1,37 @@ //! Replace a subsystem use proc_macro2::{Span, TokenStream}; use quote::quote; -use std::collections::HashSet; - -use quote::ToTokens; -use syn::AttrStyle; -use syn::Field; -use syn::FieldsNamed; -use syn::Variant; -use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; +use syn::{Ident, Result}; use super::*; pub(crate) fn impl_replacable_subsystem( - name: Ident, - subsystems: &[SubSysField], - baggage: &[BaggageField], + info: OverseerInfo, ) -> Result { let msg = "Generated by #[overlord] derive proc-macro."; let span = Span::call_site(); - let mut field_name = &subsystems.iter().map(|x| x.name.clone()).collect::>(); - let mut field_ty = &subsystems.iter().map(|x| x.generic.clone()).collect::>(); - - let mut baggage_generic_ty = &baggage.iter().filter(|b| b.generic).map(|b| b.field_ty.clone()).collect::>(); + let field_ty = &info.subsystem_generic_types(); + let baggage_generic_ty = &info.baggage_generic_types(); - let mut baggage_name = &baggage.iter().map(|x| x.field_name.clone()).collect::>(); + let baggage_name = &info.baggage_names(); let generics = quote! { - < Ctx: SubsystemContext, #( #baggage_generic_ty, )* #( #field_ty, )* > + < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > }; let where_clause = quote! { where + Ctx: SubsystemContext, #( #field_ty : Subsystem, )* }; let mut additive = TokenStream::new(); // generate an impl of `fn replace_#name` - for SubSysField { name: replacable_item, ty: replacable_item_ty, generic, .. } in subsystems { - let keeper = subsystems.iter().filter(|&ssf| ssf.generic != *generic).map(|ssf| ssf.name.clone()); + for SubSysField { name: replacable_item, ty: _replacable_item_ty, generic, .. } in subsystems { + let keeper = info.subsystems.iter().filter(|&ssf| ssf.generic != *generic).map(|ssf| ssf.name.clone()); let fn_name = Ident::new(&format!("replace_{}", replacable_item), span); // adjust the generics such that the appropriate member type is replaced diff --git a/node/overseer/overseer-gen/src/inc.rs b/node/overseer/overseer-gen/src/inc.rs index 984d82a2bf8a..38e0c4fd8244 100644 --- a/node/overseer/overseer-gen/src/inc.rs +++ b/node/overseer/overseer-gen/src/inc.rs @@ -1,15 +1,12 @@ -use proc_macro2::{Span, TokenStream}; -use quote::quote; -use std::collections::HashSet; - -use quote::ToTokens; -use syn::AttrStyle; -use syn::Field; -use syn::FieldsNamed; -use syn::Result; -use syn::Variant; -use super::*; + + + + + + + +use syn::Result; pub(crate) fn include_static_rs() -> Result { use std::str::FromStr; diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index ddebffa05cab..1082cdc9c93a 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -14,39 +14,93 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use proc_macro2::{Span, TokenStream}; -use quote::quote; +use proc_macro2::TokenStream; +use syn::{parse2, Error, GenericParam, Result}; use std::collections::HashSet; -use quote::ToTokens; -use syn::AttrStyle; -use syn::Field; -use syn::FieldsNamed; -use syn::Variant; -use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; - -mod impls; +mod parse; mod impl_overseer; mod impl_replace; mod impl_channels_out; +mod impl_message_wrapper; mod inc; -use impls::*; +use parse::*; use impl_overseer::*; use impl_replace::*; use impl_channels_out::*; -use inc::*; +use impl_message_wrapper::*; #[proc_macro_attribute] pub fn overlord(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream { let attr: TokenStream = attr.into(); let item: TokenStream = item.into(); - impls::impl_overseer_gen(attr, item).unwrap_or_else(|err| err.to_compile_error()).into() + impl_overseer_gen(attr, item).unwrap_or_else(|err| err.to_compile_error()).into() +} + +pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result { + let args = parse_attr(attr)?; + let message_wrapper = args.wrapper_enum_name; + + let span = proc_macro2::Span::call_site(); + let ds = parse2::(orig.clone())?; + match ds.fields { + syn::Fields::Named(named) => { + let overseer_name = ds.ident.clone(); + + // collect the indepedentent subsystem generics + // which need to be carried along, there are the non-generated ones + let mut orig_generics = ds.generics; + + // remove default types + let mut baggage_generic_idents = HashSet::with_capacity(orig_generics.params.len()); + orig_generics.params = orig_generics + .params + .into_iter() + .map(|mut generic| { + match generic { + GenericParam::Type(ref mut param) => { + baggage_generic_idents.insert(param.ident.clone()); + param.eq_token = None; + param.default = None; + } + _ => {} + } + generic + }) + .collect(); + + let (subsystem, baggage) = parse_overseer_struct_field(baggage_generic_idents, named)?; + let info = OverseerInfo { + subsystem, + baggage, + overseer_name, + message_wrapper, + }; + + let mut additive = impl_overseer_struct(orig_generics, &info)?; + + additive.extend(impl_message_wrapper_enum(&info)?); + additive.extend(impl_channels_out_struct(&info)?); + additive.extend(impl_replacable_subsystem(&info)); + + additive.extend(inc::include_static_rs()?); + + Ok(additive) + } + syn::Fields::Unit => Err(Error::new(span, "Must be a struct with named fields. Not an unit struct.")), + syn::Fields::Unnamed(_) => { + Err(Error::new(span, "Must be a struct with named fields. Not an unnamed fields struct.")) + } + } } + + #[cfg(test)] mod tests { use super::*; + use quote::quote; #[test] fn basic() { diff --git a/node/overseer/overseer-gen/src/impls.rs b/node/overseer/overseer-gen/src/parse.rs similarity index 62% rename from node/overseer/overseer-gen/src/impls.rs rename to node/overseer/overseer-gen/src/parse.rs index 735893f49f29..0ee841813540 100644 --- a/node/overseer/overseer-gen/src/impls.rs +++ b/node/overseer/overseer-gen/src/parse.rs @@ -1,19 +1,21 @@ use proc_macro2::{Span, TokenStream}; -use quote::quote; use std::collections::HashSet; - -use quote::ToTokens; use syn::AttrStyle; -use syn::Generics; +use syn::punctuated::Punctuated; +use syn::parse::Parse; +use syn::token::Paren; +use syn::Token; use syn::Field; use syn::FieldsNamed; -use syn::Variant; -use syn::{parse2, Attribute, Error, GenericParam, Ident, PathArguments, Result, Type, TypeParam, WhereClause}; +use syn::Result; +use syn::Ident; use syn::spanned::Spanned; +use syn::Error; +use syn::Attribute; +use syn::Type; -use super::*; - - +/// A field of the struct annotated with +/// `#[subsystem(no_dispatch, A | B | C)]` #[derive(Clone)] pub(crate) struct SubSysField { /// Name of the field. @@ -23,7 +25,7 @@ pub(crate) struct SubSysField { /// Type of the subsystem. pub(crate) ty: Ident, /// Type to be consumed by the subsystem. - pub(crate) consumes: Vec, + pub(crate) consumes: Ident, /// Consumes is a set of messages, are these to be dispatched? pub(crate) no_dispatch: bool, } @@ -37,13 +39,14 @@ fn try_type_to_ident(ty: Type, span: Span) -> Result { } } +/// Attribute arguments pub(crate) struct AttrArgs { pub(crate) wrapper_enum_name: Ident, pub(crate) signal_capacity: usize, pub(crate) message_capacity: usize, } -fn parse_attr(_attr: TokenStream) -> Result { +pub(crate) fn parse_attr(_attr: TokenStream) -> Result { Ok(AttrArgs { wrapper_enum_name: Ident::new("AllMessages", Span::call_site()), signal_capacity: 64usize, @@ -51,12 +54,6 @@ fn parse_attr(_attr: TokenStream) -> Result { }) } -use syn::parse::Parse; -use syn::punctuated::Punctuated; -use syn::token::Paren; -use syn::Token; - - pub(crate) struct SubSystemTag { pub(crate) attrs: Vec, pub(crate) paren_token: Paren, @@ -76,16 +73,73 @@ impl Parse for SubSystemTag { } } + +/// Fields that are _not_ subsystems. +pub(crate) struct BaggageField { + pub(crate) field_name: Ident, + pub(crate) field_ty: Ident, + pub(crate) generic: bool, +} + + +#[derive(Clone, Debug)] +pub(crate) struct OverseerInfo { + /// Fields annotated with `#[subsystem(..)]`. + pub(crate) subsystems: Vec, + /// Fields that do not define a subsystem, + /// but are mere baggage. + pub(crate) baggage: Vec, + /// Name of the wrapping enum for all messages, defaults to `AllMessages`. + pub(crate) message_wrapper: Ident, + /// Name of the overseer struct, used as a prefix for + /// almost all generated types. + pub(crate) overseer_name: Ident, +} + +impl OverseerInfo { + pub(crate) fn subsystems(&self) -> &[SubSysField] { + self.subsystems.as_slice() + } + + pub(crate) fn subsystem_names(&self) -> Vec { + self.subsystems.iter().map(|ssf| ssf.name.clone()).collect::>() + } + pub(crate) fn subsystem_types(&self) -> Vec { + self.subsystems.iter().map(|ssf| ssf.ty.clone()).collect::>() + } + + pub(crate) fn baggage_names(&self) -> Vec { + self.baggage.iter().map(|bag| bag.field_name.clone()).collect::>() + } + pub(crate) fn baggage_types(&self) -> Vec { + self.baggage.iter().map(|bag| bag.field_ty.clone()).collect::>() + } + + + pub(crate) fn subsystem_generic_types(&self) -> Vec { + self.subsystem.iter().map(|sff| sff.generic.clone()).collect::>() + } + + pub(crate) fn baggage_generic_ty(&self) -> Vec { + self.baggage.iter().filter(|bag| bag.generic).map(|bag| bag.field_ty.clone()).collect::>() + } + + pub(crate) fn channels(&self, suffix: &'static str) -> Vec { + self.subsystems.iter().map(|ssf| Ident::new(ssf.name.to_string() + suffix, ssf.name.span()).collect::>() + } +} + + /// Creates a list of generic identifiers used for the subsystems pub(crate) fn parse_overseer_struct_field( baggage_generics: HashSet, fields: FieldsNamed, ) -> Result<(Vec, Vec)> { - let span = Span::call_site(); + let _span = Span::call_site(); let n = fields.named.len(); let mut subsystems = Vec::with_capacity(n); let mut baggage = Vec::with_capacity(n); - for (idx, Field { attrs, vis, ident, ty, .. }) in fields.named.into_iter().enumerate() { + for (idx, Field { attrs, vis: _, ident, ty, .. }) in fields.named.into_iter().enumerate() { let mut consumes = attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| { let span = attr.path.span(); attr.path.get_ident().filter(|ident| *ident == "subsystem").map(move |_ident| { @@ -98,7 +152,7 @@ pub(crate) fn parse_overseer_struct_field( })?; if let Some((attr_tokens, span)) = consumes.next() { - if let Some((attr_tokens2, span2)) = consumes.next() { + if let Some((_attr_tokens2, span2)) = consumes.next() { return Err({ let mut err = Error::new(span, "The first subsystem annotation is at"); err.combine( @@ -110,8 +164,12 @@ pub(crate) fn parse_overseer_struct_field( let mut consumes_idents = Vec::with_capacity(attrs.len()); let variant = syn::parse2::(dbg!(attr_tokens.clone()))?; + if variant.consumes.len() != 1 { + return Err(Error::new(attr_tokens.span(), "Currently only exactly one consumes message is supported.")) + } consumes_idents.extend(variant.consumes.into_iter()); + if consumes_idents.is_empty() { return Err( Error::new(span, "Subsystem must consume at least one message") @@ -123,7 +181,7 @@ pub(crate) fn parse_overseer_struct_field( name: ident, generic: Ident::new(format!("Sub{}", idx).as_str(), Span::call_site()), ty: try_type_to_ident(ty, span)?, - consumes: consumes_idents, + consumes: consumes_idents[0], no_dispatch, }); } else { @@ -167,84 +225,3 @@ pub(crate) fn parse_overseer_struct_field( // combined_generics.params.extend(subsys_generics); // Ok(combined_generics) // } - -/// Fields that are _not_ subsystems. -pub(crate) struct BaggageField { - pub(crate) field_name: Ident, - pub(crate) field_ty: Ident, - pub(crate) generic: bool, -} - -/// Generates the wrapper type enum. -pub(crate) fn impl_messages_wrapper_enum( - messages_wrapper: &Ident, - subsystems: &[SubSysField], - baggage: &[BaggageField], -) -> Result { - let mut consumes = subsystems.iter().map(|ssf| ssf.consumes.clone().into_iter()).flatten(); - - let msg = "Generated message type wrapper"; - let x = quote! { - #[doc = #msg] - #[derive(Debug, Clone)] - enum #messages_wrapper { - #( - #consumes ( #consumes ), - )* - } - }; - Ok(x) -} - - - -pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result { - let args = parse_attr(attr)?; - let message_wrapper = args.wrapper_enum_name; - - let span = proc_macro2::Span::call_site(); - let ds = parse2::(orig.clone())?; - match ds.fields { - syn::Fields::Named(named) => { - let overseer_name = ds.ident.clone(); - - // collect the indepedentent subsystem generics - // which need to be carried along, there are the non-generated ones - let mut orig_generics = ds.generics; - - // remove default types - let mut baggage_generic_idents = HashSet::with_capacity(orig_generics.params.len()); - orig_generics.params = orig_generics - .params - .into_iter() - .map(|mut generic| { - match generic { - GenericParam::Type(ref mut param) => { - baggage_generic_idents.insert(param.ident.clone()); - param.eq_token = None; - param.default = None; - } - _ => {} - } - generic - }) - .collect(); - - let (subsystems, baggage) = parse_overseer_struct_field(baggage_generic_idents, named)?; - - let mut additive = impl_overseer_struct(&overseer_name, &message_wrapper, orig_generics, &subsystems, &baggage)?; - - additive.extend(impl_messages_wrapper_enum(&message_wrapper, &subsystems[..], &baggage[..])?); - additive.extend(impl_channels_out_struct(&message_wrapper, &subsystems[..], &baggage[..])?); - additive.extend(impl_replacable_subsystem(overseer_name, &subsystems[..], &baggage[..])); - - additive.extend(inc::include_static_rs()?); - - Ok(additive) - } - syn::Fields::Unit => Err(Error::new(span, "Must be a struct with named fields. Not an unit struct.")), - syn::Fields::Unnamed(_) => { - Err(Error::new(span, "Must be a struct with named fields. Not an unnamed fields struct.")) - } - } -} diff --git a/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs b/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs index 92de18c0602e..c8136343faaf 100644 --- a/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs @@ -10,7 +10,7 @@ struct MsgStrukt(u8); #[derive(Default, Clone, Copy)] struct AwesomeSubSys; -#[derive(Clone, AllSubsystemsGen)] +#[derive(Clone)] #[overlord(Wrapper)] struct Overseer { #[subsystem(MsgStrukt)] @@ -25,6 +25,4 @@ fn main() { let overseer = Overseer::>::builder() .sub0(AwesomeSubSys::default()) .build(Spawner); - - let overseer = overseer.replace_sub0(TequilaInABar::default()); } From 4f839be13723b683d053c75219de670b409e321f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 10 May 2021 17:58:50 +0200 Subject: [PATCH 015/161] refactor --- .../overseer-gen/src/impl_channels_out.rs | 23 +++--- .../overseer-gen/src/impl_message_wrapper.rs | 6 +- .../overseer-gen/src/impl_overseer.rs | 78 ++++++++----------- .../overseer/overseer-gen/src/impl_replace.rs | 15 ++-- node/overseer/overseer-gen/src/inc.rs | 8 -- node/overseer/overseer-gen/src/lib.rs | 15 ++-- node/overseer/overseer-gen/src/parse.rs | 21 +++-- 7 files changed, 78 insertions(+), 88 deletions(-) diff --git a/node/overseer/overseer-gen/src/impl_channels_out.rs b/node/overseer/overseer-gen/src/impl_channels_out.rs index 8c9235effff6..a4070c31add4 100644 --- a/node/overseer/overseer-gen/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/src/impl_channels_out.rs @@ -11,10 +11,10 @@ pub(crate) fn impl_channels_out_struct( let message_wrapper = info.message_wrapper.clone(); - let channel_name = &info.channels(""); - let channel_name_unbounded = &info.channels("_unbounded"); + let channel_name = &info.channel_names(""); + let channel_name_unbounded = &info.channel_names("_unbounded"); - let mut consumes = &info.subsystem.iter().map(|ssf| ssf.consumes.clone()).collect::>(); + let consumes = &info.consumes(); let ts = quote! { #[derive(Debug)] @@ -32,10 +32,10 @@ pub(crate) fn impl_channels_out_struct( pub struct ChannelsOut { #( - pub #channel_name: ::metered::MeteredSender>, + pub #channel_name: ::metered::MeteredSender>, )* #( - pub #channel_name_unbounded: ::metered::UnboundedMeteredSender>, + pub #channel_name_unbounded: ::metered::UnboundedMeteredSender>, )* } @@ -47,7 +47,7 @@ pub(crate) fn impl_channels_out_struct( ) { let res = match message { #( - #message_wrapper :: #field_ty (msg) => { + #message_wrapper :: #consumes (msg) => { self. #channel_name .send( make_packet(signals_received, msg) ).await @@ -56,10 +56,11 @@ pub(crate) fn impl_channels_out_struct( }; if res.is_err() { - tracing::debug!( - target: LOG_TARGET, - "Failed to send a message to another subsystem", - ); + // XXX FIXME + // tracing::debug!( + // target: LOG_TARGET, + // "Failed to send a message to another subsystem", + // ); } } @@ -70,7 +71,7 @@ pub(crate) fn impl_channels_out_struct( ) { let res = match message { #( - #message_wrapper :: #field_ty (msg) => { + #message_wrapper :: #consumes (msg) => { self. #channel_name_unbounded .send( make_packet(signals_received, msg) ).await diff --git a/node/overseer/overseer-gen/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/src/impl_message_wrapper.rs index 6bd7a2148e2f..6d9ffb086dd8 100644 --- a/node/overseer/overseer-gen/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/src/impl_message_wrapper.rs @@ -7,15 +7,15 @@ use super::*; pub(crate) fn impl_message_wrapper_enum( info: &OverseerInfo, ) -> Result { - let consumes = info.subsystems.iter().map(|ssf| ssf.consumes.clone()).flatten(); + let consumes = info.consumes(); - let message_wrapper = info.message_wrapper; + let message_wrapper = &info.message_wrapper; let msg = "Generated message type wrapper"; let x = quote! { #[doc = #msg] #[derive(Debug, Clone)] - enum #messages_wrapper { + enum #message_wrapper { #( #consumes ( #consumes ), )* diff --git a/node/overseer/overseer-gen/src/impl_overseer.rs b/node/overseer/overseer-gen/src/impl_overseer.rs index 89a44fba8e74..9ab47a0686a1 100644 --- a/node/overseer/overseer-gen/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/src/impl_overseer.rs @@ -9,6 +9,8 @@ use super::*; pub(crate) fn impl_overseer_struct( info: &OverseerInfo, ) -> Result { + let message_wrapper = &info.message_wrapper.clone(); + let overseer_name = info.overseer_name.clone(); let field_name = &info.subsystem_names(); let field_ty = &info.subsystem_generic_types(); @@ -28,7 +30,7 @@ pub(crate) fn impl_overseer_struct( #( #field_ty : Subsystem )* }; - let mut x = quote! { + let mut ts = quote! { pub struct #overseer_name #generics { #( #field_name: #field_ty, @@ -40,9 +42,9 @@ pub(crate) fn impl_overseer_struct( } impl #generics #overseer_name #generics { - pub async fn stop(mut self) -> { + pub async fn stop(mut self) { #( - let _ = self. #field_name .send_signal(OverseerSignal::Conclude).await + let _ = self. #field_name .send_signal(OverseerSignal::Conclude).await; )* loop { select! { @@ -58,16 +60,16 @@ pub(crate) fn impl_overseer_struct( } } - async pub fn broadcast_signal(&mut self, signal: OverseerSignal) -> SubsystemResult<()> { + pub async fn broadcast_signal(&mut self, signal: OverseerSignal) -> SubsystemResult<()> { #( - self. #field_name .send_signal(signal.clone()).await, + self. #field_name .send_signal(signal.clone()).await; )* let _ = signal; Ok(()) } - async pub fn route_message(&mut self, msg: #message_wrapper) -> SubsystemResult<()> { + pub async fn route_message(&mut self, msg: #message_wrapper) -> SubsystemResult<()> { match msg { #( #field_ty (msg) => self. #field_name .send_message(msg).await?, @@ -77,47 +79,31 @@ pub(crate) fn impl_overseer_struct( } }; - x.extend(impl_builder(overseer_name, subsystems, baggage)?); - - Ok(x) + ts.extend(impl_builder(info)?); + Ok(ts) } /// Implement a builder pattern. pub(crate) fn impl_builder( - name: &Ident, - subsystems: &[SubSysField], - baggage: &[BaggageField], + info: &OverseerInfo, ) -> Result { - let builder = Ident::new((name.to_string() + "Builder").as_str(), Span::call_site()); - - let overseer = name.clone(); - let handler = Ident::new(&(overseer.to_string() + "Handler"), overseer.span()); - - let field_name = &subsystems.iter().map(|x| x.name.clone()).collect::>(); - let field_ty = &subsystems.iter().map(|x| x.generic.clone()).collect::>(); - - let _channel_name = subsystems.iter().map(|ssf| - ssf.name.clone()); - let channel_name_unbounded = subsystems.iter().map(|ssf| - Ident::new(&(ssf.name.to_string() + "_unbounded"), ssf.name.span()) - ).collect::>(); - let channel_name_tx = &subsystems.iter().map(|ssf| - Ident::new(&(ssf.name.to_string() + "_tx"), ssf.name.span()) - ).collect::>(); - let channel_name_unbounded_tx = &subsystems.iter().map(|ssf| - Ident::new(&(ssf.name.to_string() + "_unbounded_tx"), ssf.name.span()) - ).collect::>(); - let channel_name_rx = &subsystems.iter().map(|ssf| - Ident::new(&(ssf.name.to_string() + "_rx"), ssf.name.span()) - ).collect::>(); - let channel_name_unbounded_rx = &subsystems.iter().map(|ssf| - Ident::new(&(ssf.name.to_string() + "_unbounded_rx"), ssf.name.span()) - ).collect::>(); - - let baggage_generic_ty = &baggage.iter().filter(|b| b.generic).map(|b| b.field_ty.clone()).collect::>(); - - let baggage_name = &baggage.iter().map(|x| x.field_name.clone()).collect::>(); - let _baggage_ty = &baggage.iter().map(|x| x.field_ty.clone()).collect::>(); + let overseer_name = info.overseer_name.clone(); + let builder = Ident::new(&(overseer_name.to_string() + "Builder"), overseer_name.span()); + let handler = Ident::new(&(overseer_name.to_string() + "Handler"), overseer_name.span()); + + let field_name = &info.subsystem_names(); + let field_ty = &info.subsystem_generic_types(); + + let channel_name_unbounded = &info.channel_names("_unbounded"); + + let channel_name_tx = &info.channel_names("_tx"); + let channel_name_unbounded_tx = &info.channel_names("_unbounded_tx"); + + let channel_name_rx = &info.channel_names("_rx"); + let channel_name_unbounded_rx = &info.channel_names("_unbounded_rx"); + + let baggage_generic_ty = &info.baggage_generic_types(); + let baggage_name = &info.baggage_names(); let generics = quote! { < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > @@ -130,7 +116,7 @@ pub(crate) fn impl_builder( let ts = quote! { - impl #generics #name #generics #where_clause { + impl #generics #overseer_name #generics #where_clause { fn builder() -> #builder { #builder :: default() } @@ -154,8 +140,8 @@ pub(crate) fn impl_builder( } )* - fn build(mut self, ctx: Ctx) -> (#overseer #generics, #handler) { - let overseer = #overseer :: #generics { + fn build(mut self, ctx: Ctx) -> (#overseer_name #generics, #handler) { + let overseer = #overseer_name :: #generics { #( #field_name : self. #field_name .unwrap(), )* @@ -173,8 +159,6 @@ pub(crate) fn impl_builder( }; - let metrics = ::register(prometheus_registry)?; - let (to_overseer_tx, to_overseer_rx) = metered::unbounded(); let mut running_subsystems = FuturesUnordered::new(); diff --git a/node/overseer/overseer-gen/src/impl_replace.rs b/node/overseer/overseer-gen/src/impl_replace.rs index ee732f130c29..f93edba440b9 100644 --- a/node/overseer/overseer-gen/src/impl_replace.rs +++ b/node/overseer/overseer-gen/src/impl_replace.rs @@ -6,11 +6,12 @@ use syn::{Ident, Result}; use super::*; pub(crate) fn impl_replacable_subsystem( - info: OverseerInfo, + info: &OverseerInfo, ) -> Result { let msg = "Generated by #[overlord] derive proc-macro."; let span = Span::call_site(); + let overseer_name = &info.overseer_name; let field_ty = &info.subsystem_generic_types(); let baggage_generic_ty = &info.baggage_generic_types(); @@ -30,14 +31,14 @@ pub(crate) fn impl_replacable_subsystem( let mut additive = TokenStream::new(); // generate an impl of `fn replace_#name` - for SubSysField { name: replacable_item, ty: _replacable_item_ty, generic, .. } in subsystems { + for SubSysField { name: replacable_item, ty: _replacable_item_ty, generic, .. } in info.subsystems.iter() { let keeper = info.subsystems.iter().filter(|&ssf| ssf.generic != *generic).map(|ssf| ssf.name.clone()); let fn_name = Ident::new(&format!("replace_{}", replacable_item), span); // adjust the generics such that the appropriate member type is replaced let new = Ident::new("NEW", span); - let modified_generics = &subsystems + let modified_generics = &info.subsystems .iter() .map(|ssf| if ssf.generic != *generic { ssf.generic.clone() } else { new.clone() }) .collect::>(); @@ -47,13 +48,13 @@ pub(crate) fn impl_replacable_subsystem( }; let x: TokenStream = quote! { - impl #generics #name #generics #where_clause { + impl #generics #overseer_name #generics #where_clause { #[doc = #msg] - pub fn #fn_name < #new > (self, replacement: #new) -> #name #modified_generics + pub fn #fn_name < #new > (self, replacement: #new) -> #overseer_name #modified_generics where - #new: Subsystem + #new: Subsystem, { - #name :: #modified_generics { + #overseer_name :: #modified_generics { #replacable_item: replacement, #( #keeper: self.#keeper, diff --git a/node/overseer/overseer-gen/src/inc.rs b/node/overseer/overseer-gen/src/inc.rs index 38e0c4fd8244..7a2b189feac1 100644 --- a/node/overseer/overseer-gen/src/inc.rs +++ b/node/overseer/overseer-gen/src/inc.rs @@ -1,11 +1,3 @@ - - - - - - - - use syn::Result; pub(crate) fn include_static_rs() -> Result { diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 1082cdc9c93a..ca2ea0fbbc7b 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -70,19 +70,19 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< }) .collect(); - let (subsystem, baggage) = parse_overseer_struct_field(baggage_generic_idents, named)?; + let (subsystems, baggage) = parse_overseer_struct_field(baggage_generic_idents, named)?; let info = OverseerInfo { - subsystem, + subsystems, baggage, overseer_name, message_wrapper, }; - let mut additive = impl_overseer_struct(orig_generics, &info)?; + let mut additive = impl_overseer_struct(&info)?; additive.extend(impl_message_wrapper_enum(&info)?); additive.extend(impl_channels_out_struct(&info)?); - additive.extend(impl_replacable_subsystem(&info)); + additive.extend(impl_replacable_subsystem(&info)?); additive.extend(inc::include_static_rs()?); @@ -113,8 +113,11 @@ mod tests { #[subsystem(Foo)] sub0: FooSubsystem, - #[subsystem(Bar | Twain)] - yyy: BarSubsystem, + #[subsystem(Bar)] + yyy: BaersBuyBilliardBalls, + + #[subsystem(Twain)] + fff: Beeeeep, spawner: S, metrics: Metrics, diff --git a/node/overseer/overseer-gen/src/parse.rs b/node/overseer/overseer-gen/src/parse.rs index 0ee841813540..306fc294075a 100644 --- a/node/overseer/overseer-gen/src/parse.rs +++ b/node/overseer/overseer-gen/src/parse.rs @@ -16,7 +16,7 @@ use syn::Type; /// A field of the struct annotated with /// `#[subsystem(no_dispatch, A | B | C)]` -#[derive(Clone)] +#[derive(Clone, Debug)] pub(crate) struct SubSysField { /// Name of the field. pub(crate) name: Ident, @@ -75,6 +75,7 @@ impl Parse for SubSystemTag { /// Fields that are _not_ subsystems. +#[derive(Debug, Clone)] pub(crate) struct BaggageField { pub(crate) field_name: Ident, pub(crate) field_ty: Ident, @@ -117,15 +118,23 @@ impl OverseerInfo { pub(crate) fn subsystem_generic_types(&self) -> Vec { - self.subsystem.iter().map(|sff| sff.generic.clone()).collect::>() + self.subsystems.iter().map(|sff| sff.generic.clone()).collect::>() } - pub(crate) fn baggage_generic_ty(&self) -> Vec { + pub(crate) fn baggage_generic_types(&self) -> Vec { self.baggage.iter().filter(|bag| bag.generic).map(|bag| bag.field_ty.clone()).collect::>() } - pub(crate) fn channels(&self, suffix: &'static str) -> Vec { - self.subsystems.iter().map(|ssf| Ident::new(ssf.name.to_string() + suffix, ssf.name.span()).collect::>() + pub(crate) fn channel_names(&self, suffix: &'static str) -> Vec { + self.subsystems.iter() + .map(|ssf| Ident::new(&(ssf.name.to_string() + suffix), ssf.name.span())) + .collect::>() + } + + pub(crate) fn consumes(&self) -> Vec { + self.subsystems.iter() + .map(|ssf| ssf.consumes.clone()) + .collect::>() } } @@ -181,7 +190,7 @@ pub(crate) fn parse_overseer_struct_field( name: ident, generic: Ident::new(format!("Sub{}", idx).as_str(), Span::call_site()), ty: try_type_to_ident(ty, span)?, - consumes: consumes_idents[0], + consumes: consumes_idents[0].clone(), no_dispatch, }); } else { From a6e9e272b7b6b69b191161275c510c1a2b6cbfc1 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 11 May 2021 09:03:55 +0200 Subject: [PATCH 016/161] file shuffle --- Cargo.lock | 32 ++-- Cargo.toml | 1 + node/overseer/overseer-gen/Cargo.toml | 15 +- .../overseer-gen/proc-macro/Cargo.toml | 22 +++ .../{ => proc-macro}/src/dispatch.rs | 0 .../{ => proc-macro}/src/impl_channels_out.rs | 11 +- .../src/impl_message_wrapper.rs | 0 .../{ => proc-macro}/src/impl_overseer.rs | 1 - .../{ => proc-macro}/src/impl_replace.rs | 0 .../overseer-gen/{ => proc-macro}/src/inc.rs | 0 .../{ => proc-macro}/src/inc/static.rs | 0 .../overseer-gen/proc-macro/src/lib.rs | 138 +++++++++++++++++ .../{ => proc-macro}/src/parse.rs | 0 .../tests/ui/err-01-replace-w-inadequate.rs | 0 .../{ => proc-macro}/tests/ui/err-02-enum.rs | 0 .../tests/ui/err-03-subsys-twice.rs | 0 .../{ => proc-macro}/tests/ui/ok-01-boring.rs | 0 .../tests/ui/ok-02-w-generic.rs | 0 .../tests/ui/ok-03-no-dispatch.rs | 0 node/overseer/overseer-gen/src/lib.rs | 140 +----------------- node/overseer/subsystems-gen/Cargo.toml | 3 + 21 files changed, 195 insertions(+), 168 deletions(-) create mode 100644 node/overseer/overseer-gen/proc-macro/Cargo.toml rename node/overseer/overseer-gen/{ => proc-macro}/src/dispatch.rs (100%) rename node/overseer/overseer-gen/{ => proc-macro}/src/impl_channels_out.rs (89%) rename node/overseer/overseer-gen/{ => proc-macro}/src/impl_message_wrapper.rs (100%) rename node/overseer/overseer-gen/{ => proc-macro}/src/impl_overseer.rs (99%) rename node/overseer/overseer-gen/{ => proc-macro}/src/impl_replace.rs (100%) rename node/overseer/overseer-gen/{ => proc-macro}/src/inc.rs (100%) rename node/overseer/overseer-gen/{ => proc-macro}/src/inc/static.rs (100%) create mode 100644 node/overseer/overseer-gen/proc-macro/src/lib.rs rename node/overseer/overseer-gen/{ => proc-macro}/src/parse.rs (100%) rename node/overseer/overseer-gen/{ => proc-macro}/tests/ui/err-01-replace-w-inadequate.rs (100%) rename node/overseer/overseer-gen/{ => proc-macro}/tests/ui/err-02-enum.rs (100%) rename node/overseer/overseer-gen/{ => proc-macro}/tests/ui/err-03-subsys-twice.rs (100%) rename node/overseer/overseer-gen/{ => proc-macro}/tests/ui/ok-01-boring.rs (100%) rename node/overseer/overseer-gen/{ => proc-macro}/tests/ui/ok-02-w-generic.rs (100%) rename node/overseer/overseer-gen/{ => proc-macro}/tests/ui/ok-03-no-dispatch.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 85d7622291e4..9461c3f09ae7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6259,6 +6259,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "polkadot-overseer-gen" +version = "0.1.0" +dependencies = [ + "polkadot-overseer-gen-proc-macro", + "tracing", +] + +[[package]] +name = "polkadot-overseer-gen-proc-macro" +version = "0.1.0" +dependencies = [ + "assert_matches", + "proc-macro-crate 1.0.0", + "proc-macro2", + "quote", + "syn", + "trybuild", +] + [[package]] name = "polkadot-parachain" version = "0.9.1" @@ -6303,18 +6323,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "polkadot-procmacro-overseer-gen" -version = "0.1.0" -dependencies = [ - "assert_matches", - "proc-macro-crate 1.0.0", - "proc-macro2", - "quote", - "syn", - "trybuild", -] - [[package]] name = "polkadot-procmacro-overseer-subsystems-gen" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 61438e1b96b7..becf65a89301 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ members = [ "node/network/gossip-support", "node/overseer", "node/overseer/overseer-gen", + "node/overseer/overseer-gen/proc-macro", "node/overseer/subsystems-gen", "node/primitives", "node/service", diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index 21f7b82224fd..aa57102ef6e8 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -1,19 +1,10 @@ [package] -name = "polkadot-procmacro-overseer-gen" +name = "polkadot-overseer-gen" version = "0.1.0" authors = ["Parity Technologies "] edition = "2018" description = "Generate an overseer including builder pattern and message wrapper from a single struct." -[lib] -proc-macro = true - [dependencies] -syn = { version = "1.0.60", features = ["full", "extra-traits"] } -quote = "1.0.9" -proc-macro2 = "1.0.26" -proc-macro-crate = "1.0.0" -assert_matches = "1.5.0" - -[dev-dependencies] -trybuild = "1.0.41" +polkadot-overseer-gen-proc-macro = { path = "./proc-macro" } +tracing = "0.1" diff --git a/node/overseer/overseer-gen/proc-macro/Cargo.toml b/node/overseer/overseer-gen/proc-macro/Cargo.toml new file mode 100644 index 000000000000..29dd07ae65f3 --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "polkadot-overseer-gen-proc-macro" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" +description = "Generate an overseer including builder pattern and message wrapper from a single annotated struct definition." + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0.60", features = ["full", "extra-traits"] } +quote = "1.0.9" +proc-macro2 = "1.0.26" +proc-macro-crate = "1.0.0" +assert_matches = "1.5.0" + +[dev-dependencies] +trybuild = "1.0.41" diff --git a/node/overseer/overseer-gen/src/dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/dispatch.rs similarity index 100% rename from node/overseer/overseer-gen/src/dispatch.rs rename to node/overseer/overseer-gen/proc-macro/src/dispatch.rs diff --git a/node/overseer/overseer-gen/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs similarity index 89% rename from node/overseer/overseer-gen/src/impl_channels_out.rs rename to node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index a4070c31add4..f5741abbda5d 100644 --- a/node/overseer/overseer-gen/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -40,7 +40,7 @@ pub(crate) fn impl_channels_out_struct( } impl ChannelsOut { - async pub fn send_and_log_error( + pub async fn send_and_log_error( &mut self, signals_received: usize, message: #message_wrapper, @@ -80,10 +80,11 @@ pub(crate) fn impl_channels_out_struct( }; if res.is_err() { - ::tracing::debug!( - target: LOG_TARGET, - "Failed to send a message to another subsystem", - ); + // FIXME + // tracing::debug!( + // target: LOG_TARGET, + // "Failed to send a message to another subsystem", + // ); } } } diff --git a/node/overseer/overseer-gen/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs similarity index 100% rename from node/overseer/overseer-gen/src/impl_message_wrapper.rs rename to node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs diff --git a/node/overseer/overseer-gen/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs similarity index 99% rename from node/overseer/overseer-gen/src/impl_overseer.rs rename to node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 9ab47a0686a1..eca3cfd4c7fc 100644 --- a/node/overseer/overseer-gen/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -183,7 +183,6 @@ pub(crate) fn impl_builder( } } - // #( #launch subsystem ) (overseer, handler) diff --git a/node/overseer/overseer-gen/src/impl_replace.rs b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs similarity index 100% rename from node/overseer/overseer-gen/src/impl_replace.rs rename to node/overseer/overseer-gen/proc-macro/src/impl_replace.rs diff --git a/node/overseer/overseer-gen/src/inc.rs b/node/overseer/overseer-gen/proc-macro/src/inc.rs similarity index 100% rename from node/overseer/overseer-gen/src/inc.rs rename to node/overseer/overseer-gen/proc-macro/src/inc.rs diff --git a/node/overseer/overseer-gen/src/inc/static.rs b/node/overseer/overseer-gen/proc-macro/src/inc/static.rs similarity index 100% rename from node/overseer/overseer-gen/src/inc/static.rs rename to node/overseer/overseer-gen/proc-macro/src/inc/static.rs diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs new file mode 100644 index 000000000000..ca2ea0fbbc7b --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -0,0 +1,138 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use proc_macro2::TokenStream; +use syn::{parse2, Error, GenericParam, Result}; +use std::collections::HashSet; + +mod parse; +mod impl_overseer; +mod impl_replace; +mod impl_channels_out; +mod impl_message_wrapper; +mod inc; + +use parse::*; +use impl_overseer::*; +use impl_replace::*; +use impl_channels_out::*; +use impl_message_wrapper::*; + +#[proc_macro_attribute] +pub fn overlord(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream { + let attr: TokenStream = attr.into(); + let item: TokenStream = item.into(); + impl_overseer_gen(attr, item).unwrap_or_else(|err| err.to_compile_error()).into() +} + +pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result { + let args = parse_attr(attr)?; + let message_wrapper = args.wrapper_enum_name; + + let span = proc_macro2::Span::call_site(); + let ds = parse2::(orig.clone())?; + match ds.fields { + syn::Fields::Named(named) => { + let overseer_name = ds.ident.clone(); + + // collect the indepedentent subsystem generics + // which need to be carried along, there are the non-generated ones + let mut orig_generics = ds.generics; + + // remove default types + let mut baggage_generic_idents = HashSet::with_capacity(orig_generics.params.len()); + orig_generics.params = orig_generics + .params + .into_iter() + .map(|mut generic| { + match generic { + GenericParam::Type(ref mut param) => { + baggage_generic_idents.insert(param.ident.clone()); + param.eq_token = None; + param.default = None; + } + _ => {} + } + generic + }) + .collect(); + + let (subsystems, baggage) = parse_overseer_struct_field(baggage_generic_idents, named)?; + let info = OverseerInfo { + subsystems, + baggage, + overseer_name, + message_wrapper, + }; + + let mut additive = impl_overseer_struct(&info)?; + + additive.extend(impl_message_wrapper_enum(&info)?); + additive.extend(impl_channels_out_struct(&info)?); + additive.extend(impl_replacable_subsystem(&info)?); + + additive.extend(inc::include_static_rs()?); + + Ok(additive) + } + syn::Fields::Unit => Err(Error::new(span, "Must be a struct with named fields. Not an unit struct.")), + syn::Fields::Unnamed(_) => { + Err(Error::new(span, "Must be a struct with named fields. Not an unnamed fields struct.")) + } + } +} + + + +#[cfg(test)] +mod tests { + use super::*; + use quote::quote; + + #[test] + fn basic() { + let attr = quote! { + overloard(AllMessages, x, y) + }; + + let item = quote! { + pub struct Ooooh where S: SpawnThat { + #[subsystem(Foo)] + sub0: FooSubsystem, + + #[subsystem(Bar)] + yyy: BaersBuyBilliardBalls, + + #[subsystem(Twain)] + fff: Beeeeep, + + spawner: S, + metrics: Metrics, + } + }; + + let output = impl_overseer_gen(attr, item).expect("Simple example always works. qed"); + println!("//generated:"); + println!("{}", output); + } + + #[test] + fn ui() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/err-*.rs"); + t.pass("tests/ui/ok-*.rs"); + } +} diff --git a/node/overseer/overseer-gen/src/parse.rs b/node/overseer/overseer-gen/proc-macro/src/parse.rs similarity index 100% rename from node/overseer/overseer-gen/src/parse.rs rename to node/overseer/overseer-gen/proc-macro/src/parse.rs diff --git a/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs b/node/overseer/overseer-gen/proc-macro/tests/ui/err-01-replace-w-inadequate.rs similarity index 100% rename from node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs rename to node/overseer/overseer-gen/proc-macro/tests/ui/err-01-replace-w-inadequate.rs diff --git a/node/overseer/overseer-gen/tests/ui/err-02-enum.rs b/node/overseer/overseer-gen/proc-macro/tests/ui/err-02-enum.rs similarity index 100% rename from node/overseer/overseer-gen/tests/ui/err-02-enum.rs rename to node/overseer/overseer-gen/proc-macro/tests/ui/err-02-enum.rs diff --git a/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs b/node/overseer/overseer-gen/proc-macro/tests/ui/err-03-subsys-twice.rs similarity index 100% rename from node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs rename to node/overseer/overseer-gen/proc-macro/tests/ui/err-03-subsys-twice.rs diff --git a/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs b/node/overseer/overseer-gen/proc-macro/tests/ui/ok-01-boring.rs similarity index 100% rename from node/overseer/overseer-gen/tests/ui/ok-01-boring.rs rename to node/overseer/overseer-gen/proc-macro/tests/ui/ok-01-boring.rs diff --git a/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs b/node/overseer/overseer-gen/proc-macro/tests/ui/ok-02-w-generic.rs similarity index 100% rename from node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs rename to node/overseer/overseer-gen/proc-macro/tests/ui/ok-02-w-generic.rs diff --git a/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs b/node/overseer/overseer-gen/proc-macro/tests/ui/ok-03-no-dispatch.rs similarity index 100% rename from node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs rename to node/overseer/overseer-gen/proc-macro/tests/ui/ok-03-no-dispatch.rs diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index ca2ea0fbbc7b..f304d246fda9 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -1,138 +1,2 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use proc_macro2::TokenStream; -use syn::{parse2, Error, GenericParam, Result}; -use std::collections::HashSet; - -mod parse; -mod impl_overseer; -mod impl_replace; -mod impl_channels_out; -mod impl_message_wrapper; -mod inc; - -use parse::*; -use impl_overseer::*; -use impl_replace::*; -use impl_channels_out::*; -use impl_message_wrapper::*; - -#[proc_macro_attribute] -pub fn overlord(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream { - let attr: TokenStream = attr.into(); - let item: TokenStream = item.into(); - impl_overseer_gen(attr, item).unwrap_or_else(|err| err.to_compile_error()).into() -} - -pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result { - let args = parse_attr(attr)?; - let message_wrapper = args.wrapper_enum_name; - - let span = proc_macro2::Span::call_site(); - let ds = parse2::(orig.clone())?; - match ds.fields { - syn::Fields::Named(named) => { - let overseer_name = ds.ident.clone(); - - // collect the indepedentent subsystem generics - // which need to be carried along, there are the non-generated ones - let mut orig_generics = ds.generics; - - // remove default types - let mut baggage_generic_idents = HashSet::with_capacity(orig_generics.params.len()); - orig_generics.params = orig_generics - .params - .into_iter() - .map(|mut generic| { - match generic { - GenericParam::Type(ref mut param) => { - baggage_generic_idents.insert(param.ident.clone()); - param.eq_token = None; - param.default = None; - } - _ => {} - } - generic - }) - .collect(); - - let (subsystems, baggage) = parse_overseer_struct_field(baggage_generic_idents, named)?; - let info = OverseerInfo { - subsystems, - baggage, - overseer_name, - message_wrapper, - }; - - let mut additive = impl_overseer_struct(&info)?; - - additive.extend(impl_message_wrapper_enum(&info)?); - additive.extend(impl_channels_out_struct(&info)?); - additive.extend(impl_replacable_subsystem(&info)?); - - additive.extend(inc::include_static_rs()?); - - Ok(additive) - } - syn::Fields::Unit => Err(Error::new(span, "Must be a struct with named fields. Not an unit struct.")), - syn::Fields::Unnamed(_) => { - Err(Error::new(span, "Must be a struct with named fields. Not an unnamed fields struct.")) - } - } -} - - - -#[cfg(test)] -mod tests { - use super::*; - use quote::quote; - - #[test] - fn basic() { - let attr = quote! { - overloard(AllMessages, x, y) - }; - - let item = quote! { - pub struct Ooooh where S: SpawnThat { - #[subsystem(Foo)] - sub0: FooSubsystem, - - #[subsystem(Bar)] - yyy: BaersBuyBilliardBalls, - - #[subsystem(Twain)] - fff: Beeeeep, - - spawner: S, - metrics: Metrics, - } - }; - - let output = impl_overseer_gen(attr, item).expect("Simple example always works. qed"); - println!("//generated:"); - println!("{}", output); - } - - #[test] - fn ui() { - let t = trybuild::TestCases::new(); - t.compile_fail("tests/ui/err-*.rs"); - t.pass("tests/ui/ok-*.rs"); - } -} +pub use tracing; +pub use overseer_gen_proc_macro::*; diff --git a/node/overseer/subsystems-gen/Cargo.toml b/node/overseer/subsystems-gen/Cargo.toml index 2720663c293d..bfb04c1ea639 100644 --- a/node/overseer/subsystems-gen/Cargo.toml +++ b/node/overseer/subsystems-gen/Cargo.toml @@ -5,6 +5,9 @@ authors = ["Parity Technologies "] edition = "2018" description = "Small proc macro to create mocking level iface type helpers" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [lib] proc-macro = true From be930df5df952bd0a526fef707da4fe0aab513fa Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 11 May 2021 09:14:31 +0200 Subject: [PATCH 017/161] fixins --- .../proc-macro/src/impl_channels_out.rs | 18 ++++++++---------- .../proc-macro/src/impl_overseer.rs | 2 +- .../tests/ui/err-01-replace-w-inadequate.rs | 2 +- .../proc-macro/tests/ui/err-03-subsys-twice.rs | 2 +- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index f5741abbda5d..1f40c80f683b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -56,11 +56,10 @@ pub(crate) fn impl_channels_out_struct( }; if res.is_err() { - // XXX FIXME - // tracing::debug!( - // target: LOG_TARGET, - // "Failed to send a message to another subsystem", - // ); + tracing::debug!( + target: LOG_TARGET, + "Failed to send a message to another subsystem", + ); } } @@ -80,11 +79,10 @@ pub(crate) fn impl_channels_out_struct( }; if res.is_err() { - // FIXME - // tracing::debug!( - // target: LOG_TARGET, - // "Failed to send a message to another subsystem", - // ); + tracing::debug!( + target: LOG_TARGET, + "Failed to send a message to another subsystem", + ); } } } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index eca3cfd4c7fc..3a2751c7460f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -181,7 +181,7 @@ pub(crate) fn impl_builder( #channel_name_unbounded: #channel_name_unbounded_tx .clone(), )* } - } + }; // #( #launch subsystem ) diff --git a/node/overseer/overseer-gen/proc-macro/tests/ui/err-01-replace-w-inadequate.rs b/node/overseer/overseer-gen/proc-macro/tests/ui/err-01-replace-w-inadequate.rs index 6ca3c10cac87..43245b43591c 100644 --- a/node/overseer/overseer-gen/proc-macro/tests/ui/err-01-replace-w-inadequate.rs +++ b/node/overseer/overseer-gen/proc-macro/tests/ui/err-01-replace-w-inadequate.rs @@ -20,7 +20,7 @@ struct Overseer { sub0: AwesomeSubSys, #[subsystem(Orange)] - shots_of: TequilaInABar + shots_of: TequilaInABar, other: Stuff, } diff --git a/node/overseer/overseer-gen/proc-macro/tests/ui/err-03-subsys-twice.rs b/node/overseer/overseer-gen/proc-macro/tests/ui/err-03-subsys-twice.rs index 1718f42b8082..b6d1bfdca3c3 100644 --- a/node/overseer/overseer-gen/proc-macro/tests/ui/err-03-subsys-twice.rs +++ b/node/overseer/overseer-gen/proc-macro/tests/ui/err-03-subsys-twice.rs @@ -22,7 +22,7 @@ struct Overseer { sub0: AwesomeSubSys, #[subsystem(Orange)] - shots_of: TequilaInABar + shots_of: TequilaInABar, other: Stuff, } From d01a318b1030254bef1ee05ef7165ca2d6f2ed76 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 11 May 2021 09:20:59 +0200 Subject: [PATCH 018/161] message --- node/overseer/overseer-gen/proc-macro/src/parse.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse.rs b/node/overseer/overseer-gen/proc-macro/src/parse.rs index 306fc294075a..440fb4f763c8 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse.rs @@ -174,7 +174,7 @@ pub(crate) fn parse_overseer_struct_field( let variant = syn::parse2::(dbg!(attr_tokens.clone()))?; if variant.consumes.len() != 1 { - return Err(Error::new(attr_tokens.span(), "Currently only exactly one consumes message is supported.")) + return Err(Error::new(attr_tokens.span(), "Currently only exactly one message can be consumed per subsystem.")) } consumes_idents.extend(variant.consumes.into_iter()); From d3f5bcbcaa037a39aa43e19472c2b65f393bed50 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 11 May 2021 09:21:24 +0200 Subject: [PATCH 019/161] drop subsystems gen --- Cargo.lock | 13 +- Cargo.toml | 1 - node/overseer/Cargo.toml | 2 +- node/overseer/overseer-gen/Cargo.toml | 2 +- node/overseer/subsystems-gen/Cargo.toml | 21 -- node/overseer/subsystems-gen/src/lib.rs | 179 ------------------ .../subsystems-gen/tests/ui/err-01-enum.rs | 13 -- .../tests/ui/err-01-enum.stderr | 5 - .../tests/ui/err-01-generic-used-twice.rs | 17 -- .../tests/ui/err-01-generic-used-twice.stderr | 14 -- .../tests/ui/err-01-no-generic.rs | 17 -- .../tests/ui/err-01-no-generics.stderr | 14 -- .../tests/ui/ok-01-w-generics.rs | 17 -- 13 files changed, 3 insertions(+), 312 deletions(-) delete mode 100644 node/overseer/subsystems-gen/Cargo.toml delete mode 100644 node/overseer/subsystems-gen/src/lib.rs delete mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-enum.rs delete mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-enum.stderr delete mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.rs delete mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.stderr delete mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-no-generic.rs delete mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-no-generics.stderr delete mode 100644 node/overseer/subsystems-gen/tests/ui/ok-01-w-generics.rs diff --git a/Cargo.lock b/Cargo.lock index 9461c3f09ae7..6e7d9a077829 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6251,8 +6251,8 @@ dependencies = [ "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-util", + "polkadot-overseer-gen", "polkadot-primitives", - "polkadot-procmacro-overseer-subsystems-gen", "sc-client-api", "sp-api", "sp-core", @@ -6323,17 +6323,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "polkadot-procmacro-overseer-subsystems-gen" -version = "0.1.0" -dependencies = [ - "assert_matches", - "proc-macro2", - "quote", - "syn", - "trybuild", -] - [[package]] name = "polkadot-procmacro-subsystem-dispatch-gen" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index becf65a89301..3aa64eb1dc10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,7 +73,6 @@ members = [ "node/overseer", "node/overseer/overseer-gen", "node/overseer/overseer-gen/proc-macro", - "node/overseer/subsystems-gen", "node/primitives", "node/service", "node/subsystem", diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml index 3ecb90f45ffa..8de92b587c82 100644 --- a/node/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -12,7 +12,7 @@ futures = "0.3.12" futures-timer = "3.0.2" polkadot-node-primitives = { package = "polkadot-node-primitives", path = "../primitives" } polkadot-node-subsystem-util = { path = "../subsystem-util" } -polkadot-procmacro-overseer-subsystems-gen = { path = "./subsystems-gen" } +polkadot-overseer-gen = { path = "./overseer-gen" } polkadot-primitives = { path = "../../primitives" } polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../subsystem" } tracing = "0.1.25" diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index aa57102ef6e8..e4f7847f3b0b 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -6,5 +6,5 @@ edition = "2018" description = "Generate an overseer including builder pattern and message wrapper from a single struct." [dependencies] -polkadot-overseer-gen-proc-macro = { path = "./proc-macro" } +overseer-gen-proc-macro = { package = "polkadot-overseer-gen-proc-macro", path = "./proc-macro" } tracing = "0.1" diff --git a/node/overseer/subsystems-gen/Cargo.toml b/node/overseer/subsystems-gen/Cargo.toml deleted file mode 100644 index bfb04c1ea639..000000000000 --- a/node/overseer/subsystems-gen/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "polkadot-procmacro-overseer-subsystems-gen" -version = "0.1.0" -authors = ["Parity Technologies "] -edition = "2018" -description = "Small proc macro to create mocking level iface type helpers" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[lib] -proc-macro = true - -[dependencies] -syn = { version = "1.0.60", features = ["full", "extra-traits"] } -quote = "1.0.9" -proc-macro2 = "1.0.24" -assert_matches = "1.5.0" - -[dev-dependencies] -trybuild = "1.0.41" diff --git a/node/overseer/subsystems-gen/src/lib.rs b/node/overseer/subsystems-gen/src/lib.rs deleted file mode 100644 index c15d08bb04f4..000000000000 --- a/node/overseer/subsystems-gen/src/lib.rs +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use std::collections::HashSet; - -use proc_macro2::TokenStream; -use quote::quote; - -use syn::{Error, GenericParam, Ident, Result, Type, parse2}; - -#[proc_macro_derive(AllSubsystemsGen)] -pub fn subsystems_gen(item: proc_macro::TokenStream) -> proc_macro::TokenStream { - let item: TokenStream = item.into(); - impl_subsystems_gen(item).unwrap_or_else(|err| err.to_compile_error()).into() -} - -fn impl_subsystems_gen(item: TokenStream) -> Result { - let span = proc_macro2::Span::call_site(); - let ds = parse2::(item.clone())?; - - match ds.fields { - syn::Fields::Named(named) => { - #[derive(Clone)] - struct NameTyTup { - field: Ident, - ty: Type, - } - let mut orig_generics = ds.generics; - // remove default types - orig_generics.params = orig_generics.params.into_iter().map(|mut generic| { - match generic { - GenericParam::Type(ref mut param) => { - param.eq_token = None; - param.default = None; - } - _ => {} - } - generic - }).collect(); - - // prepare a hashmap of generic type to member that uses it - let generic_types = orig_generics.params.iter().filter_map(|generic| { - if let GenericParam::Type(param) = generic { - Some(param.ident.clone()) - } else { - None - } - }).collect::>(); - - let strukt_ty = ds.ident; - - if generic_types.is_empty() { - return Err(Error::new(strukt_ty.span(), "struct must have at least one generic parameter.")) - } - - // collect all fields that exist, and all fields that are replaceable - let mut replacable_items = Vec::::with_capacity(64); - let mut all_fields = replacable_items.clone(); - - - let mut duplicate_generic_detection = HashSet::::with_capacity(64); - - for field in named.named { - let field_ident = field.ident.clone().ok_or_else(|| Error::new(span, "Member field must have a name."))?; - let ty = field.ty.clone(); - let ntt = NameTyTup { field: field_ident, ty }; - - replacable_items.push(ntt.clone()); - - - // assure every generic is used exactly once - let ty_ident = match field.ty { - Type::Path(path) => path.path.get_ident().cloned().ok_or_else(|| { - Error::new(proc_macro2::Span::call_site(), "Expected an identifier, but got a path.") - }), - _ => return Err(Error::new(proc_macro2::Span::call_site(), "Must be path.")) - }?; - - if generic_types.contains(&ty_ident) { - if let Some(previous) = duplicate_generic_detection.replace(ty_ident) { - return Err(Error::new(previous.span(), "Generic type parameters may only be used for exactly one field, but is used more than once.")) - } - } - - all_fields.push(ntt); - } - - - let msg = "Generated by #[derive(AllSubsystemsGen)] derive proc-macro."; - let mut additive = TokenStream::new(); - - // generate an impl of `fn replace_#name` - for NameTyTup { field: replacable_item, ty: replacable_item_ty } in replacable_items { - let keeper = all_fields.iter().filter(|ntt| ntt.field != replacable_item).map(|ntt| ntt.field.clone()); - let strukt_ty = strukt_ty.clone(); - let fname = Ident::new(&format!("replace_{}", replacable_item), span); - // adjust the generics such that the appropriate member type is replaced - let mut modified_generics = orig_generics.clone(); - modified_generics.params = modified_generics.params.into_iter().map(|mut generic| { - match generic { - GenericParam::Type(ref mut param) => { - param.eq_token = None; - param.default = None; - if match &replacable_item_ty { - Type::Path(path) => - path.path.get_ident().filter(|&ident| ident == ¶m.ident).is_some(), - _ => false - } { - param.ident = Ident::new("NEW", span); - } - } - _ => {} - } - generic - }).collect(); - - additive.extend(quote! { - impl #orig_generics #strukt_ty #orig_generics { - #[doc = #msg] - pub fn #fname < NEW > (self, replacement: NEW) -> #strukt_ty #modified_generics { - #strukt_ty :: #modified_generics { - #replacable_item: replacement, - #( - #keeper: self.#keeper, - )* - } - } - } - }); - } - - Ok(additive) - } - syn::Fields::Unit => Err(Error::new(span, "Must be a struct with named fields. Not an unit struct.")), - syn::Fields::Unnamed(_) => { - Err(Error::new(span, "Must be a struct with named fields. Not an unnamed fields struct.")) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn basic() { - let item = quote! { - pub struct AllSubsystems { - pub a: A, - pub beee: B, - pub dj: CD, - } - }; - - let output = impl_subsystems_gen(item).expect("Simple example always works. qed"); - println!("//generated:"); - println!("{}", output); - } - - #[test] - fn ui() { - let t = trybuild::TestCases::new(); - t.compile_fail("tests/ui/err-*.rs"); - t.pass("tests/ui/ok-*.rs"); - } -} diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-enum.rs b/node/overseer/subsystems-gen/tests/ui/err-01-enum.rs deleted file mode 100644 index 318636279ea5..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/err-01-enum.rs +++ /dev/null @@ -1,13 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; - -#[derive(Clone, AllSubsystemsGen)] -enum AllSubsystems { - A(A), - B(B), -} - -fn main() { - let all = AllSubsystems::::A(0u8); -} diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-enum.stderr b/node/overseer/subsystems-gen/tests/ui/err-01-enum.stderr deleted file mode 100644 index 5f61df1057cb..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/err-01-enum.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: expected `struct` - --> $DIR/err-01-enum.rs:6:1 - | -6 | enum AllSubsystems { - | ^^^^ diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.rs b/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.rs deleted file mode 100644 index f89939d5c306..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.rs +++ /dev/null @@ -1,17 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; - -#[derive(Clone, AllSubsystemsGen)] -struct AllSubsystems { - a: X, - b: X, -} - -fn main() { - let all = AllSubsystems:: { - a: 0_u16, - b: 1_u16, - }; - let _all = all.replace_a(77u8); -} diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.stderr b/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.stderr deleted file mode 100644 index 23e1404ff822..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.stderr +++ /dev/null @@ -1,14 +0,0 @@ -error: Generic type parameters may only be used for exactly one field, but is used more than once. - --> $DIR/err-01-generic-used-twice.rs:7:5 - | -7 | a: X, - | ^ - -error[E0599]: no method named `replace_a` found for struct `AllSubsystems` in the current scope - --> $DIR/err-01-generic-used-twice.rs:16:17 - | -6 | struct AllSubsystems { - | ----------------------- method `replace_a` not found for this -... -16 | let _all = all.replace_a(77u8); - | ^^^^^^^^^ method not found in `AllSubsystems` diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-no-generic.rs b/node/overseer/subsystems-gen/tests/ui/err-01-no-generic.rs deleted file mode 100644 index 0466eb444cd9..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/err-01-no-generic.rs +++ /dev/null @@ -1,17 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; - -#[derive(Clone, AllSubsystemsGen)] -struct AllSubsystems { - a: f32, - b: u16, -} - -fn main() { - let all = AllSubsystems { - a: 0_f32, - b: 1_u16, - }; - let _all = all.replace_a(77u8); -} diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-no-generics.stderr b/node/overseer/subsystems-gen/tests/ui/err-01-no-generics.stderr deleted file mode 100644 index 1de880ae433c..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/err-01-no-generics.stderr +++ /dev/null @@ -1,14 +0,0 @@ -error: Generic type parameters may only be used once have at least one generic parameter. - --> $DIR/err-01-no-generics.rs:7:5 - | -7 | a: X, - | ^ - -error[E0599]: no method named `replace_a` found for struct `AllSubsystems` in the current scope - --> $DIR/err-01-no-generics.rs:16:17 - | -6 | struct AllSubsystems { - | ----------------------- method `replace_a` not found for this -... -16 | let _all = all.replace_a(77u8); - | ^^^^^^^^^ method not found in `AllSubsystems` diff --git a/node/overseer/subsystems-gen/tests/ui/ok-01-w-generics.rs b/node/overseer/subsystems-gen/tests/ui/ok-01-w-generics.rs deleted file mode 100644 index 1519990a0a55..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/ok-01-w-generics.rs +++ /dev/null @@ -1,17 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; - -#[derive(Clone, AllSubsystemsGen)] -struct AllSubsystems { - a: A, - b: B, -} - -fn main() { - let all = AllSubsystems:: { - a: 0u8, - b: 1u16, - }; - let _all: AllSubsystems<_,_> = all.replace_a::(777_777u32); -} From bb97f08f3fc04cac7ad947b552d121a7312ffaf5 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 11 May 2021 15:43:53 +0200 Subject: [PATCH 020/161] refactor --- node/overseer/overseer-gen/Cargo.toml | 8 +- .../overseer-gen/proc-macro/Cargo.toml | 1 + .../overseer-gen/proc-macro/src/dispatch.rs | 145 ----- .../proc-macro/src/impl_channels_out.rs | 6 +- .../proc-macro/src/impl_dispatch.rs | 47 ++ .../proc-macro/src/impl_message_wrapper.rs | 42 +- .../proc-macro/src/impl_overseer.rs | 125 +++- .../proc-macro/src/impl_replace.rs | 3 +- .../overseer-gen/proc-macro/src/inc/static.rs | 83 --- .../overseer-gen/proc-macro/src/lib.rs | 58 +- .../overseer-gen/proc-macro/src/parse.rs | 196 +++++- .../overseer-gen/proc-macro/src/tests.rs | 31 + node/overseer/overseer-gen/src/lib.rs | 140 +++- node/overseer/overseer-gen/src/tests.rs | 11 + .../tests/ui/err-01-replace-w-inadequate.rs | 2 +- .../{proc-macro => }/tests/ui/err-02-enum.rs | 4 +- .../tests/ui/err-03-subsys-twice.rs | 3 +- .../{proc-macro => }/tests/ui/ok-01-boring.rs | 5 +- .../tests/ui/ok-02-w-generic.rs | 5 +- .../tests/ui/ok-03-no-dispatch.rs | 2 +- node/overseer/src/handler.rs | 53 ++ node/overseer/src/lib.rs | 607 +----------------- node/overseer/src/subsystems.rs | 0 23 files changed, 660 insertions(+), 917 deletions(-) delete mode 100644 node/overseer/overseer-gen/proc-macro/src/dispatch.rs create mode 100644 node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs delete mode 100644 node/overseer/overseer-gen/proc-macro/src/inc/static.rs create mode 100644 node/overseer/overseer-gen/proc-macro/src/tests.rs create mode 100644 node/overseer/overseer-gen/src/tests.rs rename node/overseer/overseer-gen/{proc-macro => }/tests/ui/err-01-replace-w-inadequate.rs (92%) rename node/overseer/overseer-gen/{proc-macro => }/tests/ui/err-02-enum.rs (91%) rename node/overseer/overseer-gen/{proc-macro => }/tests/ui/err-03-subsys-twice.rs (93%) rename node/overseer/overseer-gen/{proc-macro => }/tests/ui/ok-01-boring.rs (89%) rename node/overseer/overseer-gen/{proc-macro => }/tests/ui/ok-02-w-generic.rs (89%) rename node/overseer/overseer-gen/{proc-macro => }/tests/ui/ok-03-no-dispatch.rs (91%) create mode 100644 node/overseer/src/handler.rs delete mode 100644 node/overseer/src/subsystems.rs diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index e4f7847f3b0b..10890f13026e 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -6,5 +6,11 @@ edition = "2018" description = "Generate an overseer including builder pattern and message wrapper from a single struct." [dependencies] -overseer-gen-proc-macro = { package = "polkadot-overseer-gen-proc-macro", path = "./proc-macro" } tracing = "0.1" +futures = "0.3" + +overseer-gen-proc-macro = { package = "polkadot-overseer-gen-proc-macro", path = "./proc-macro" } +metered = { package = "metered-channel", path = "../../metered-channel" } + +[dev-dependencies] +trybuild = "1.0.41" diff --git a/node/overseer/overseer-gen/proc-macro/Cargo.toml b/node/overseer/overseer-gen/proc-macro/Cargo.toml index 29dd07ae65f3..3b02a5d8bc82 100644 --- a/node/overseer/overseer-gen/proc-macro/Cargo.toml +++ b/node/overseer/overseer-gen/proc-macro/Cargo.toml @@ -20,3 +20,4 @@ assert_matches = "1.5.0" [dev-dependencies] trybuild = "1.0.41" +assert_matches = "1.5.0" diff --git a/node/overseer/overseer-gen/proc-macro/src/dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/dispatch.rs deleted file mode 100644 index eb56305a7669..000000000000 --- a/node/overseer/overseer-gen/proc-macro/src/dispatch.rs +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use proc_macro2::TokenStream; -use quote::{quote, ToTokens}; -use std::fmt; -use syn::{parse2, Error, Fields, FieldsNamed, FieldsUnnamed, Ident, ItemEnum, Path, Result, Type, Variant}; - -/// An enum variant without base type. -#[derive(Clone)] -struct EnumVariantDispatchWithTy { - // enum ty name - ty: Ident, - // variant - variant: EnumVariantDispatch, -} - -impl fmt::Debug for EnumVariantDispatchWithTy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}::{:?}", self.ty, self.variant) - } -} - -impl ToTokens for EnumVariantDispatchWithTy { - fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { - if let Some(inner) = &self.variant.inner { - let enum_name = &self.ty; - let variant_name = &self.variant.name; - - let quoted = quote! { - #enum_name::#variant_name(#inner::from(event)) - }; - quoted.to_tokens(tokens); - } - } -} - -/// An enum variant without the base type, contains the relevant inner type. -#[derive(Clone)] -struct EnumVariantDispatch { - /// variant name - name: Ident, - /// The inner type for which a `From::from` impl is anticipated from the input type. - /// No code will be generated for this enum variant if `inner` is `None`. - inner: Option, -} - -impl fmt::Debug for EnumVariantDispatch { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}(..)", self.name) - } -} - -fn prepare_enum_variant(variant: &mut Variant) -> Result { - let skip = variant.attrs.iter().find(|attr| attr.path.is_ident("skip")).is_some(); - variant.attrs = variant.attrs.iter().filter(|attr| !attr.path.is_ident("skip")).cloned().collect::>(); - - let variant = variant.clone(); - let span = variant.ident.span(); - let inner = match variant.fields.clone() { - // look for one called inner - Fields::Named(FieldsNamed { brace_token: _, named }) if !skip => named - .iter() - .find_map(|field| { - if let Some(ident) = &field.ident { - if ident == "inner" { - return Some(Some(field.ty.clone())); - } - } - None - }) - .ok_or_else(|| Error::new(span, "To dispatch with struct enum variant, one element must named `inner`"))?, - - // technically, if it has no inner types we cound not require the #[skip] annotation, but better make it consistent - Fields::Unnamed(FieldsUnnamed { paren_token: _, unnamed }) if !skip => unnamed - .first() - .map(|field| Some(field.ty.clone())) - .ok_or_else(|| Error::new(span, "Must be annotated with skip, even if no inner types exist."))?, - _ if skip => None, - Fields::Unit => return Err(Error::new(span, "Must be annotated with #[skip].")), - Fields::Unnamed(_) => { - return Err(Error::new( - span, - "Must be annotated with #[skip] or have in `inner` element which impls `From<_>`.", - )) - } - Fields::Named(_) => { - return Err(Error::new( - span, - "Must be annotated with #[skip] or the first wrapped type must impl `From<_>`.", - )) - } - }; - - Ok(EnumVariantDispatch { name: variant.ident, inner }) -} - -fn impl_wrapper_dispatch_fn(name: Ident, fields: &[SubSysField]) -> Result { - let event_ty = parse2::(attr)?; - - let mut ie = parse2::(item)?; - - let message_enum = ie.ident.clone(); - let variants = ie.variants.iter_mut().try_fold(Vec::::new(), |mut acc, variant| { - let variant = prepare_enum_variant(variant)?; - if variant.inner.is_some() { - acc.push(EnumVariantDispatchWithTy { ty: message_enum.clone(), variant }) - } - Ok::<_, syn::Error>(acc) - })?; - - - let msg = "Generated by #[subsystem_dispatch_gen] proc-macro."; - - let mut x = quote! { - impl #message_enum { - #[doc = #msg] - pub fn dispatch_iter(event: #event_ty) -> impl Iterator + Send { - let mut iter = None.into_iter(); - - #( - let mut iter = iter.chain(std::iter::once(event.focus().ok().map(|event| { - #variants - }))); - )* - iter.filter_map(|x| x) - } - } - }; - - Ok(x) -} diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 1f40c80f683b..0321e77b2ba4 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -1,6 +1,6 @@ use quote::quote; -use syn::{Ident, Result}; +use syn::Result; use super::*; @@ -18,12 +18,12 @@ pub(crate) fn impl_channels_out_struct( let ts = quote! { #[derive(Debug)] - struct MessagePacket { + pub struct MessagePacket { signals_received: usize, message: T, } - fn make_packet(signals_received: usize, message: T) -> MessagePacket { + pub fn make_packet(signals_received: usize, message: T) -> MessagePacket { MessagePacket { signals_received, message, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs new file mode 100644 index 000000000000..4a71cdd6c7b3 --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -0,0 +1,47 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use proc_macro2::TokenStream; +use quote::quote; +use syn::{Ident, Result}; +use super::*; + + +pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { + let message_wrapper = &info.message_wrapper; + + let dispatchable = info.subsystems().into_iter().filter(|ssf| !ssf.no_dispatch).map(|ssf| ssf.consumes.clone()).collect::>(); + + let msg = "Generated by #[subsystem_dispatch_gen] proc-macro."; + let incoming_event_ty= &info.incoming_event_ty.clone(); + let ts = quote! { + impl #message_wrapper { + #[doc = #msg] + pub fn dispatch_iter(event: #incoming_event_ty) -> impl Iterator + Send { + let mut iter = None.into_iter(); + + #( + let mut iter = iter.chain(std::iter::once(event.focus().ok().map(|event| { + #message_wrapper :: #dispatchable ( <#dispatchable as ::std::from::From < #incoming_event_ty > > :: from( event ) ) + }))); + )* + iter.filter_map(|x: Option<_>| x) + } + } + }; + + Ok(ts) +} diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 6d9ffb086dd8..6bc6b8cde3e2 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -1,5 +1,5 @@ use quote::quote; -use syn::{Ident, Result}; +use syn::Result; use super::*; @@ -12,7 +12,7 @@ pub(crate) fn impl_message_wrapper_enum( let message_wrapper = &info.message_wrapper; let msg = "Generated message type wrapper"; - let x = quote! { + let ts = quote! { #[doc = #msg] #[derive(Debug, Clone)] enum #message_wrapper { @@ -20,6 +20,42 @@ pub(crate) fn impl_message_wrapper_enum( #consumes ( #consumes ), )* } + + #( + impl ::std::from::From<#consumes> for #message_wrapper { + fn from(src: #consumes) -> Self { + #message_wrapper :: #consumes ( #consumes ) + } + } + )* + + #[derive(Debug, Clone)] + pub struct OverseerSubsystemSender { + channels: ChannelsOut, + signals_received: SignalsReceived, + } + + #[async_trait::async_trait] + impl SubsystemSender for OverseerSubsystemSender { + async fn send_message(&mut self, msg: #message_wrapper) { + self.channels.send_and_log_error(self.signals_received.load(), msg).await; + } + + async fn send_messages(&mut self, msgs: T) + where T: IntoIterator + Send, T::IntoIter: Send + { + // This can definitely be optimized if necessary. + for msg in msgs { + self.send_message(msg).await; + } + } + + fn send_unbounded_message(&mut self, msg: #message_wrapper) { + self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); + } + } + }; - Ok(x) + + Ok(ts) } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 3a2751c7460f..d5bdd9957cb2 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -1,6 +1,4 @@ -use proc_macro2::{Span}; use quote::quote; -use syn::Generics; use syn::{Ident, Result}; use super::*; @@ -21,16 +19,26 @@ pub(crate) fn impl_overseer_struct( let baggage_generic_ty = &info.baggage_generic_types(); let generics = quote! { - < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > + < Ctx, S, #( #baggage_generic_ty, )* #( #field_ty, )* > }; - let _where_clause = quote! { + let where_clause = quote! { where Ctx: SubsystemContext, + S: SpawnNamed, #( #field_ty : Subsystem )* }; + let message_channel_capacity = info.message_channel_capacity; + let signal_channel_capacity = info.signal_channel_capacity; + + let spawner_doc = "Responsible for driving the subsystem futures."; + let running_subsystems_doc = "The set of running subsystems."; + let mut ts = quote! { + const CHANNEL_CAPACITY: usize = #message_channel_capacity; + const SIGNAL_CHANNEL_CAPACITY: usize = #signal_channel_capacity; + pub struct #overseer_name #generics { #( #field_name: #field_ty, @@ -39,9 +47,16 @@ pub(crate) fn impl_overseer_struct( #( #baggage_name: #baggage_ty, )* + + #[doc = #spawner_doc] + spawner: S, + + #[doc = #running_subsystems_doc] + running_subsystems: FuturesUnordered>>, + } - impl #generics #overseer_name #generics { + impl #generics #overseer_name #generics #where_clause { pub async fn stop(mut self) { #( let _ = self. #field_name .send_signal(OverseerSignal::Conclude).await; @@ -94,6 +109,7 @@ pub(crate) fn impl_builder( let field_name = &info.subsystem_names(); let field_ty = &info.subsystem_generic_types(); + let channel_name = &info.channel_names(""); let channel_name_unbounded = &info.channel_names("_unbounded"); let channel_name_tx = &info.channel_names("_tx"); @@ -106,14 +122,18 @@ pub(crate) fn impl_builder( let baggage_name = &info.baggage_names(); let generics = quote! { - < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > + < Ctx, S, #( #baggage_generic_ty, )* #( #field_ty, )* > }; let where_clause = quote! { where + Ctx: SubsystemContext, + S: SpawnNamed, #( #field_ty : Subsystem, )* }; + let message_wrapper = &info.message_wrapper; + let ts = quote! { impl #generics #overseer_name #generics #where_clause { @@ -130,6 +150,7 @@ pub(crate) fn impl_builder( #( #baggage_name : ::std::option::Option< #baggage_name >, )* + spawner: ::std::option::Option< >, } impl #generics #builder #generics #where_clause { @@ -140,7 +161,14 @@ pub(crate) fn impl_builder( } )* - fn build(mut self, ctx: Ctx) -> (#overseer_name #generics, #handler) { + fn build(mut self, create_ctx: F) -> (#overseer_name #generics, #handler) + where F: FnMut( + metered::MeteredReceiver, + meteted::SubsystemIncomingMessages< #message_wrapper >, + ChannelsOut, + metered::UnboundedMeteredSender, + ) -> Ctx, + { let overseer = #overseer_name :: #generics { #( #field_name : self. #field_name .unwrap(), @@ -150,8 +178,6 @@ pub(crate) fn impl_builder( )* }; - const CHANNEL_CAPACITY: usize = 1024; - const SIGNAL_CHANNEL_CAPACITY: usize = 64; let (events_tx, events_rx) = ::metered::channel(SIGNAL_CHANNEL_CAPACITY); let handler = #handler { @@ -175,7 +201,7 @@ pub(crate) fn impl_builder( ChannelsOut { #( - channel_name: #channel_name_tx .clone(), + #channel_name: #channel_name_tx .clone(), )* #( #channel_name_unbounded: #channel_name_unbounded_tx .clone(), @@ -185,6 +211,85 @@ pub(crate) fn impl_builder( // #( #launch subsystem ) + enum TaskKind { + Regular, + Blocking, + } + + let spawner = &mut overseer.spawner; + + let mut spawn = | + message_tx: metered::MeteredSender>, + message_rx: SubsystemIncomingMessages, + unbounded_meter: metered::Meter, + to_subsystems: ChannelsOut, + to_overseer_tx: metered::UnboundedMeteredSender, + s: impl Subsystem>, + metrics: &Metrics, + futures: &mut FuturesUnordered>>, + task_kind: TaskKind, + | -> SubsystemResult> + { + let (signal_tx, signal_rx) = metered::channel(SIGNAL_CHANNEL_CAPACITY); + let ctx = create_ctx( + signal_rx, + message_rx, + to_subsystems, + to_overseer_tx, + ); + let SpawnedSubsystem { future, name } = s.start(ctx); + + let (terminated_tx, terminated_rx) = oneshot::channel(); + + let fut = Box::pin(async move { + if let Err(e) = future.await { + tracing::error!(subsystem=name, err = ?e, "subsystem exited with error"); + } else { + tracing::debug!(subsystem=name, "subsystem exited without an error"); + } + let _ = terminated_tx.send(()); + }); + + match task_kind { + TaskKind::Regular => spawner.spawn(name, fut), + TaskKind::Blocking => spawner.spawn_blocking(name, fut), + } + + futures.push(Box::pin(terminated_rx.map(|e| { tracing::warn!(err = ?e, "dropping error"); Ok(()) }))); + + let instance = Some(SubsystemInstance::< #message_wrapper > { + meters: SubsystemMeters { + unbounded: unbounded_meter, + bounded: message_tx.meter().clone(), + signals: signal_tx.meter().clone(), + }, + tx_signal: signal_tx, + tx_bounded: message_tx, + signals_received: 0, + name, + }); + + Ok(OverseenSubsystem::< #message_wrapper > { + instance, + }) + }; + + + #( + let #field_name = spawn( + #channel_name_tx, + stream::select( + #channel_name_tx, + #channel_name_unbounded_tx, + ), + channels_out.clone(), + to_overseer_tx.clone(), + #field_name, + &mut running_subsystems, + TaskKind::Blocking, + )?; + )* + (overseer, handler) } } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs index f93edba440b9..139a82f586db 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs @@ -19,12 +19,13 @@ pub(crate) fn impl_replacable_subsystem( let baggage_name = &info.baggage_names(); let generics = quote! { - < Ctx, #( #baggage_generic_ty, )* #( #field_ty, )* > + < Ctx, S, #( #baggage_generic_ty, )* #( #field_ty, )* > }; let where_clause = quote! { where Ctx: SubsystemContext, + S: SpawnNamed, #( #field_ty : Subsystem, )* }; diff --git a/node/overseer/overseer-gen/proc-macro/src/inc/static.rs b/node/overseer/overseer-gen/proc-macro/src/inc/static.rs deleted file mode 100644 index 421ab3e2d911..000000000000 --- a/node/overseer/overseer-gen/proc-macro/src/inc/static.rs +++ /dev/null @@ -1,83 +0,0 @@ -trait MapSubsystem { - type Output; - - fn map_subsystem(&self, sub: T) -> Self::Output; -} - -impl MapSubsystem for F where F: Fn(T) -> U { - type Output = U; - - fn map_subsystem(&self, sub: T) -> U { - (self)(sub) - } -} - -type SubsystemIncomingMessages = ::futures::stream::Select< - ::metered::MeteredReceiver>, - ::metered::UnboundedMeteredReceiver>, ->; - - - -#[derive(Debug, Default, Clone)] -struct SignalsReceived(Arc); - -impl SignalsReceived { - fn load(&self) -> usize { - self.0.load(atomic::Ordering::SeqCst) - } - - fn inc(&self) { - self.0.fetch_add(1, atomic::Ordering::SeqCst); - } -} - -/// A sender from subsystems to other subsystems. -#[derive(Debug, Clone)] -pub struct OverseerSubsystemSender { - channels: ChannelsOut, - signals_received: SignalsReceived, -} - -#[async_trait::async_trait] -impl SubsystemSender for OverseerSubsystemSender { - async fn send_message(&mut self, msg: AllMessages) { - self.channels.send_and_log_error(self.signals_received.load(), msg).await; - } - - async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send - { - // This can definitely be optimized if necessary. - for msg in msgs { - self.send_message(msg).await; - } - } - - fn send_unbounded_message(&mut self, msg: AllMessages) { - self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); - } -} - -#[derive(Clone)] -struct SubsystemMeters { - bounded: metered::Meter, - unbounded: metered::Meter, - signals: metered::Meter, -} - -impl SubsystemMeters { - fn read(&self) -> SubsystemMeterReadouts { - SubsystemMeterReadouts { - bounded: self.bounded.read(), - unbounded: self.unbounded.read(), - signals: self.signals.read(), - } - } -} - -struct SubsystemMeterReadouts { - bounded: metered::Readout, - unbounded: metered::Readout, - signals: metered::Readout, -} diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index ca2ea0fbbc7b..7f665581e7ca 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -15,7 +15,8 @@ // along with Polkadot. If not, see . use proc_macro2::TokenStream; -use syn::{parse2, Error, GenericParam, Result}; +use proc_macro2::Span; +use syn::{parse2, Error, GenericParam, Result, Ident}; use std::collections::HashSet; mod parse; @@ -23,13 +24,15 @@ mod impl_overseer; mod impl_replace; mod impl_channels_out; mod impl_message_wrapper; -mod inc; +mod impl_dispatch; +// mod inc; use parse::*; use impl_overseer::*; use impl_replace::*; use impl_channels_out::*; use impl_message_wrapper::*; +use impl_dispatch::*; #[proc_macro_attribute] pub fn overlord(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -39,8 +42,8 @@ pub fn overlord(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> } pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result { - let args = parse_attr(attr)?; - let message_wrapper = args.wrapper_enum_name; + let args: AttrArgs = syn::parse2(attr)?; + let message_wrapper = args.message_wrapper; let span = proc_macro2::Span::call_site(); let ds = parse2::(orig.clone())?; @@ -76,6 +79,9 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< baggage, overseer_name, message_wrapper, + message_channel_capacity: args.message_channel_capacity, + signal_channel_capacity: args.signal_channel_capacity, + incoming_event_ty: Ident::new("Network", Span::call_site()), // FIXME }; let mut additive = impl_overseer_struct(&info)?; @@ -83,8 +89,9 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< additive.extend(impl_message_wrapper_enum(&info)?); additive.extend(impl_channels_out_struct(&info)?); additive.extend(impl_replacable_subsystem(&info)?); + additive.extend(impl_dispatch(&info)?); - additive.extend(inc::include_static_rs()?); + // additive.extend(inc::include_static_rs()?); Ok(additive) } @@ -95,44 +102,5 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< } } - - #[cfg(test)] -mod tests { - use super::*; - use quote::quote; - - #[test] - fn basic() { - let attr = quote! { - overloard(AllMessages, x, y) - }; - - let item = quote! { - pub struct Ooooh where S: SpawnThat { - #[subsystem(Foo)] - sub0: FooSubsystem, - - #[subsystem(Bar)] - yyy: BaersBuyBilliardBalls, - - #[subsystem(Twain)] - fff: Beeeeep, - - spawner: S, - metrics: Metrics, - } - }; - - let output = impl_overseer_gen(attr, item).expect("Simple example always works. qed"); - println!("//generated:"); - println!("{}", output); - } - - #[test] - fn ui() { - let t = trybuild::TestCases::new(); - t.compile_fail("tests/ui/err-*.rs"); - t.pass("tests/ui/ok-*.rs"); - } -} +mod tests; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse.rs b/node/overseer/overseer-gen/proc-macro/src/parse.rs index 440fb4f763c8..cc3052d77bdb 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse.rs @@ -1,6 +1,6 @@ -use proc_macro2::{Span, TokenStream}; -use std::collections::HashSet; -use syn::AttrStyle; +use proc_macro2::Span; +use std::collections::{HashMap, HashSet}; +use syn::{AttrStyle, Path}; use syn::punctuated::Punctuated; use syn::parse::Parse; use syn::token::Paren; @@ -14,8 +14,9 @@ use syn::Error; use syn::Attribute; use syn::Type; +use syn::LitInt; /// A field of the struct annotated with -/// `#[subsystem(no_dispatch, A | B | C)]` +/// `#[subsystem(no_dispatch, , A | B | C)]` #[derive(Clone, Debug)] pub(crate) struct SubSysField { /// Name of the field. @@ -26,8 +27,12 @@ pub(crate) struct SubSysField { pub(crate) ty: Ident, /// Type to be consumed by the subsystem. pub(crate) consumes: Ident, - /// Consumes is a set of messages, are these to be dispatched? + /// If `no_dispatch` is present, if the message is incomming via + /// an extern `Event`, it will not be dispatched to all subsystems. pub(crate) no_dispatch: bool, + /// If the subsystem imlementation is blocking execution and hence + /// has to be spawned on a separate thread or thread pool. + pub(crate) blocking: bool, } fn try_type_to_ident(ty: Type, span: Span) -> Result { @@ -39,23 +44,134 @@ fn try_type_to_ident(ty: Type, span: Span) -> Result { } } + +use syn::parse::ParseBuffer; + +#[derive(Clone, Debug)] +enum AttrItem { + ExternEventType(Path), + MessageWrapperName(Ident), + SignalChannelCapacity(LitInt), + MessageChannelCapacity(LitInt), +} + +impl Spanned for AttrItem { + fn span(&self) -> Span { + match self { + AttrItem::ExternEventType(x) => x.span(), + AttrItem::MessageWrapperName(x) => x.span(), + AttrItem::SignalChannelCapacity(x) => x.span(), + AttrItem::MessageChannelCapacity(x) => x.span(), + } + } +} + +impl AttrItem { + fn key(&self) -> &'static str { + match self { + AttrItem::ExternEventType(_) => "event", + AttrItem::MessageWrapperName(_) => "gen", + AttrItem::SignalChannelCapacity(_) => "signal_capacity", + AttrItem::MessageChannelCapacity(_) => "message_capacity", + } + } +} + +impl Parse for AttrItem { + fn parse(input: &ParseBuffer) -> Result { + let key = input.parse::()?; + Ok(if key == "event" { + let _ = input.parse::()?; + let path = input.parse::()?; + AttrItem::ExternEventType(path) + } else if key == "gen" { + let _ = input.parse::()?; + let wrapper_message = input.parse::()?; + AttrItem::MessageWrapperName(wrapper_message) + } else if key == "signal_capacity" { + let _ = input.parse::()?; + let value = input.parse::()?; + AttrItem::SignalChannelCapacity(value) + } else if key == "message_capacity" { + let _ = input.parse::()?; + let value = input.parse::()?; + AttrItem::MessageChannelCapacity(value) + } else { + return Err(Error::new(Span::call_site(), "Expected one of `gen`, `signal_capacity`, or `message_capacity`.")) + }) + } +} + /// Attribute arguments +#[derive(Clone, Debug)] pub(crate) struct AttrArgs { - pub(crate) wrapper_enum_name: Ident, - pub(crate) signal_capacity: usize, - pub(crate) message_capacity: usize, + pub(crate) message_wrapper: Ident, + pub(crate) extern_event_ty: Path, + pub(crate) signal_channel_capacity: u64, + pub(crate) message_channel_capacity: u64, } -pub(crate) fn parse_attr(_attr: TokenStream) -> Result { - Ok(AttrArgs { - wrapper_enum_name: Ident::new("AllMessages", Span::call_site()), - signal_capacity: 64usize, - message_capacity: 1024usize, - }) +impl Parse for AttrArgs { + fn parse(input: &ParseBuffer) -> Result { + let span = Span::call_site(); + + let content; + let _paren = syn::parenthesized!(content in input); + let items: Punctuated<_, Token![,]> = content.parse_terminated(AttrItem::parse)?; + + let mut unique = HashMap::<&str, AttrItem, std::collections::hash_map::RandomState>::default(); + for item in items { + if let Some(first) = unique.insert(item.key(), item.clone()) { + let mut e = Error::new(item.span(), "Duplicate definition found"); + e.combine(Error::new(first.span(), "previously defined here.")); + return Err(e) + } + } + + let signal_channel_capacity = if let Some(item) = unique.get("signal_capacity") { + if let AttrItem::SignalChannelCapacity(lit) = item { + lit.base10_parse::()? + } else { + unreachable!() + } + } else { + 64 + }; + + let message_channel_capacity = if let Some(item) = unique.get("message_capacity") { + if let AttrItem::MessageChannelCapacity(lit) = item { + lit.base10_parse::()? + } else { + unreachable!() + } + } else { + 1024 + }; + let extern_event_ty = unique.get("event") + .map(|x| if let AttrItem::ExternEventType(x) = x { x.clone() } else { unreachable!() } ) + .ok_or_else(|| { + Error::new(span, "Must declare the external event type via `event=..`.") + })?; + + let message_wrapper = unique.get("gen") + .map(|x| if let AttrItem::MessageWrapperName(x) = x { x.clone() } else { unreachable!() } ) + .ok_or_else(|| { + Error::new(span, "Must declare the external event type via `event=..`.") + })?; + + Ok(AttrArgs { + signal_channel_capacity, + message_channel_capacity, + extern_event_ty, + message_wrapper, + }) + } } pub(crate) struct SubSystemTag { + #[allow(dead_code)] pub(crate) attrs: Vec, + #[allow(dead_code)] pub(crate) paren_token: Paren, pub(crate) no_dispatch: bool, pub(crate) consumes: Punctuated, @@ -67,7 +183,18 @@ impl Parse for SubSystemTag { Ok(Self { attrs: Attribute::parse_outer(input)?, paren_token: syn::parenthesized!(content in input), - no_dispatch: false, // FIXME + no_dispatch: { + if content.peek(Ident) && content.peek2(Token![,]) { + let ident = content.parse::()?; + if ident != "no_dispatch" { + return Err(Error::new(ident.span(), "Allowed tags is only `no_dispatch` at this time")) + } + let _ = content.parse::()?; + true + } else { + false + } + }, consumes: content.parse_terminated(Ident::parse)?, }) } @@ -95,6 +222,14 @@ pub(crate) struct OverseerInfo { /// Name of the overseer struct, used as a prefix for /// almost all generated types. pub(crate) overseer_name: Ident, + + /// Size of the bounded channel. + pub(crate) message_channel_capacity: u64, + /// Size of the bounded signal channel. + pub(crate) signal_channel_capacity: u64, + + /// Incoming event type, commonly from the network bridge. + pub(crate) incoming_event_ty: Ident, } impl OverseerInfo { @@ -105,6 +240,9 @@ impl OverseerInfo { pub(crate) fn subsystem_names(&self) -> Vec { self.subsystems.iter().map(|ssf| ssf.name.clone()).collect::>() } + + #[allow(dead_code)] + // FIXME use as the defaults pub(crate) fn subsystem_types(&self) -> Vec { self.subsystems.iter().map(|ssf| ssf.ty.clone()).collect::>() } @@ -192,6 +330,7 @@ pub(crate) fn parse_overseer_struct_field( ty: try_type_to_ident(ty, span)?, consumes: consumes_idents[0].clone(), no_dispatch, + blocking: false, // FIXME XXX }); } else { let field_ty = try_type_to_ident(ty, Span::call_site())?; @@ -215,7 +354,7 @@ pub(crate) fn parse_overseer_struct_field( // let mut combined_generics = orig_generics.clone(); // let mut ctx_generic = GenericParam::Type( -// syn::parse2::(quote! { +// syn::parse::(quote! { // Ctx: SubsystemContext, // }) // .unwrap(), @@ -224,7 +363,7 @@ pub(crate) fn parse_overseer_struct_field( // let mut subsys_generics = subsystems.iter().map(|ssf| { // let subsys_generic_name = ssf.generic.clone(); // GenericParam::Type( -// syn::parse2::(quote! { +// syn::parse::(quote! { // #subsys_generic_name: Subsystem, // }) // .unwrap(), @@ -234,3 +373,26 @@ pub(crate) fn parse_overseer_struct_field( // combined_generics.params.extend(subsys_generics); // Ok(combined_generics) // } + + +#[cfg(test)] +mod tests { + use syn::parse_quote; + use assert_matches::assert_matches; + use super::*; + + #[test] + fn attr() { + let attr: AttrArgs = parse_quote! { + (gen=AllMessage, event=::some::where::ExternEvent, signal_capacity=111, message_capacity=222,) + }; + assert_matches!(attr, AttrArgs { + message_channel_capacity, + signal_channel_capacity, + .. + } => { + assert_eq!(message_channel_capacity, 222); + assert_eq!(signal_channel_capacity, 111); + }); + } +} diff --git a/node/overseer/overseer-gen/proc-macro/src/tests.rs b/node/overseer/overseer-gen/proc-macro/src/tests.rs new file mode 100644 index 000000000000..6046d9248fbd --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/src/tests.rs @@ -0,0 +1,31 @@ +use super::*; +use quote::quote; + +#[test] +fn basic() { + let attr = quote! { + overloard(event=OverseerSignal,gen=AllMessages) + }; + + let item = quote! { + pub struct Ooooh where X: Secrit { + #[subsystem(no_dispatch, Foo)] + sub0: FooSubsystem, + + #[subsystem(blocking, Bar)] + yyy: BaersBuyBilliardBalls, + + #[subsystem(no_dispatch, blocking, Twain)] + fff: Beeeeep, + + #[subsystem(Rope)] + mc: MountainCave, + + metrics: Metrics, + } + }; + + let output = impl_overseer_gen(attr, item).expect("Simple example always works. qed"); + println!("//generated:"); + println!("{}", output); +} diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index f304d246fda9..9e45cc45157b 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -1,2 +1,140 @@ -pub use tracing; +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! # Overseer +//! +//! `overseer` implements the Overseer architecture described in the +//! [implementers-guide](https://w3f.github.io/parachain-implementers-guide/node/index.html). +//! For the motivations behind implementing the overseer itself you should +//! check out that guide, documentation in this crate will be mostly discussing +//! technical stuff. +//! +//! An `Overseer` is something that allows spawning/stopping and overseing +//! asynchronous tasks as well as establishing a well-defined and easy to use +//! protocol that the tasks can use to communicate with each other. It is desired +//! that this protocol is the only way tasks communicate with each other, however +//! at this moment there are no foolproof guards against other ways of communication. +//! +//! The `Overseer` is instantiated with a pre-defined set of `Subsystems` that +//! share the same behavior from `Overseer`'s point of view. +//! +//! ```text +//! +-----------------------------+ +//! | Overseer | +//! +-----------------------------+ +//! +//! ................| Overseer "holds" these and uses |.............. +//! . them to (re)start things . +//! . . +//! . +-------------------+ +---------------------+ . +//! . | Subsystem1 | | Subsystem2 | . +//! . +-------------------+ +---------------------+ . +//! . | | . +//! .................................................................. +//! | | +//! start() start() +//! V V +//! ..................| Overseer "runs" these |....................... +//! . +--------------------+ +---------------------+ . +//! . | SubsystemInstance1 | | SubsystemInstance2 | . +//! . +--------------------+ +---------------------+ . +//! .................................................................. +//! ``` + +// #![deny(unused_results)] +// unused dependencies can not work for test and examples at the same time +// yielding false positives +#![warn(missing_docs)] + pub use overseer_gen_proc_macro::*; +pub use tracing; +pub use metered; +use std::sync::atomic::{self, AtomicUsize}; +use std::sync::Arc; + +/// A running instance of some [`Subsystem`]. +/// +/// [`Subsystem`]: trait.Subsystem.html +struct SubsystemInstance { + tx_signal: metered::MeteredSender, + tx_bounded: metered::MeteredSender>, + meters: SubsystemMeters, + signals_received: usize, + name: &'static str, +} + + +trait MapSubsystem { + type Output; + + fn map_subsystem(&self, sub: T) -> Self::Output; +} + +impl MapSubsystem for F where F: Fn(T) -> U { + type Output = U; + + fn map_subsystem(&self, sub: T) -> U { + (self)(sub) + } +} + +type SubsystemIncomingMessages = ::futures::stream::Select< + ::metered::MeteredReceiver>, + ::metered::UnboundedMeteredReceiver>, +>; + + + +#[derive(Debug, Default, Clone)] +struct SignalsReceived(Arc); + +impl SignalsReceived { + fn load(&self) -> usize { + self.0.load(atomic::Ordering::SeqCst) + } + + fn inc(&self) { + self.0.fetch_add(1, atomic::Ordering::SeqCst); + } +} + + +#[derive(Clone)] +struct SubsystemMeters { + bounded: metered::Meter, + unbounded: metered::Meter, + signals: metered::Meter, +} + +impl SubsystemMeters { + fn read(&self) -> SubsystemMeterReadouts { + SubsystemMeterReadouts { + bounded: self.bounded.read(), + unbounded: self.unbounded.read(), + signals: self.signals.read(), + } + } +} + +struct SubsystemMeterReadouts { + bounded: metered::Readout, + unbounded: metered::Readout, + signals: metered::Readout, +} + + +#[cfg(test)] +mod tests; diff --git a/node/overseer/overseer-gen/src/tests.rs b/node/overseer/overseer-gen/src/tests.rs new file mode 100644 index 000000000000..36d0089c1b7c --- /dev/null +++ b/node/overseer/overseer-gen/src/tests.rs @@ -0,0 +1,11 @@ + +#[test] +fn ui_pass() { + let t = trybuild::TestCases::new(); + t.pass("tests/ui/ok-*.rs"); +} +#[test] +fn ui_compile_fail() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/err-*.rs"); +} diff --git a/node/overseer/overseer-gen/proc-macro/tests/ui/err-01-replace-w-inadequate.rs b/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs similarity index 92% rename from node/overseer/overseer-gen/proc-macro/tests/ui/err-01-replace-w-inadequate.rs rename to node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs index 43245b43591c..e23d5ba16b0b 100644 --- a/node/overseer/overseer-gen/proc-macro/tests/ui/err-01-replace-w-inadequate.rs +++ b/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use polkadot_procmacro_overseer_gen::overlord; +use polkadot_overseer_gen_proc_macro::overlord; struct X; diff --git a/node/overseer/overseer-gen/proc-macro/tests/ui/err-02-enum.rs b/node/overseer/overseer-gen/tests/ui/err-02-enum.rs similarity index 91% rename from node/overseer/overseer-gen/proc-macro/tests/ui/err-02-enum.rs rename to node/overseer/overseer-gen/tests/ui/err-02-enum.rs index 42d5947c0149..0db83e644875 100644 --- a/node/overseer/overseer-gen/proc-macro/tests/ui/err-02-enum.rs +++ b/node/overseer/overseer-gen/tests/ui/err-02-enum.rs @@ -1,14 +1,14 @@ #![allow(dead_code)] -use polkadot_procmacro_overseer_gen::overlord; +use polkadot_overseer_gen_proc_macro::overlord; struct Msg(u8); #[derive(Default, Clone, Copy)] struct AwesomeSub; -#[derive(Clone, AllSubsystemsGen)] #[overlord(Wrapper)] +#[derive(Clone, AllSubsystemsGen)] enum Overseer { #[subsystem(Msg)] Sub0(AwesomeSub), diff --git a/node/overseer/overseer-gen/proc-macro/tests/ui/err-03-subsys-twice.rs b/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs similarity index 93% rename from node/overseer/overseer-gen/proc-macro/tests/ui/err-03-subsys-twice.rs rename to node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs index b6d1bfdca3c3..4b91e667309c 100644 --- a/node/overseer/overseer-gen/proc-macro/tests/ui/err-03-subsys-twice.rs +++ b/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs @@ -1,7 +1,6 @@ #![allow(dead_code)] -use polkadot_procmacro_overseer_gen::overlord; - +use polkadot_overseer_gen_proc_macro::overlord; struct X; struct Z; diff --git a/node/overseer/overseer-gen/proc-macro/tests/ui/ok-01-boring.rs b/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs similarity index 89% rename from node/overseer/overseer-gen/proc-macro/tests/ui/ok-01-boring.rs rename to node/overseer/overseer-gen/tests/ui/ok-01-boring.rs index ccf999c254b3..da6b2d131193 100644 --- a/node/overseer/overseer-gen/proc-macro/tests/ui/ok-01-boring.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs @@ -1,15 +1,14 @@ #![allow(dead_code)] -use polkadot_procmacro_overseer_gen::overlord; - +use polkadot_overseer_gen_proc_macro::overlord; struct X; #[derive(Default, Clone, Copy)] struct AwesomeSubSys; -#[derive(Clone, AllSubsystemsGen)] #[overlord(Wrapper)] +#[derive(Clone, AllSubsystemsGen)] struct Overseer { #[subsystem(X)] sub0: AwesomeSubSys, diff --git a/node/overseer/overseer-gen/proc-macro/tests/ui/ok-02-w-generic.rs b/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs similarity index 89% rename from node/overseer/overseer-gen/proc-macro/tests/ui/ok-02-w-generic.rs rename to node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs index c8136343faaf..8731ff46400a 100644 --- a/node/overseer/overseer-gen/proc-macro/tests/ui/ok-02-w-generic.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs @@ -1,7 +1,6 @@ #![allow(dead_code)] -use polkadot_procmacro_overseer_gen::overlord; - +use polkadot_overseer_gen_proc_macro::overlord; trait MetaMeta {} #[derive(Debug)] @@ -10,8 +9,8 @@ struct MsgStrukt(u8); #[derive(Default, Clone, Copy)] struct AwesomeSubSys; -#[derive(Clone)] #[overlord(Wrapper)] +#[derive(Clone)] struct Overseer { #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, diff --git a/node/overseer/overseer-gen/proc-macro/tests/ui/ok-03-no-dispatch.rs b/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs similarity index 91% rename from node/overseer/overseer-gen/proc-macro/tests/ui/ok-03-no-dispatch.rs rename to node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs index af99446b9de6..a92562733b30 100644 --- a/node/overseer/overseer-gen/proc-macro/tests/ui/ok-03-no-dispatch.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use polkadot_procmacro_overseer_gen::overlord; +use polkadot_overseer_gen_proc_macro::overlord; #[derive(Debug)] struct MsgStrukt(u8); diff --git a/node/overseer/src/handler.rs b/node/overseer/src/handler.rs new file mode 100644 index 000000000000..275035624325 --- /dev/null +++ b/node/overseer/src/handler.rs @@ -0,0 +1,53 @@ +/// A handler used to communicate with the [`Overseer`]. +/// +/// [`Overseer`]: struct.Overseer.html +#[derive(Clone)] +pub struct OverseerHandler { + events_tx: metered::MeteredSender, +} + +impl OverseerHandler { + /// Inform the `Overseer` that that some block was imported. + #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] + pub async fn block_imported(&mut self, block: BlockInfo) { + self.send_and_log_error(Event::BlockImported(block)).await + } + + /// Send some message to one of the `Subsystem`s. + #[tracing::instrument(level = "trace", skip(self, msg), fields(subsystem = LOG_TARGET))] + pub async fn send_msg(&mut self, msg: impl Into) { + self.send_and_log_error(Event::MsgToSubsystem(msg.into())).await + } + + /// Inform the `Overseer` that some block was finalized. + #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] + pub async fn block_finalized(&mut self, block: BlockInfo) { + self.send_and_log_error(Event::BlockFinalized(block)).await + } + + /// Wait for a block with the given hash to be in the active-leaves set. + /// + /// The response channel responds if the hash was activated and is closed if the hash was deactivated. + /// Note that due the fact the overseer doesn't store the whole active-leaves set, only deltas, + /// the response channel may never return if the hash was deactivated before this call. + /// In this case, it's the caller's responsibility to ensure a timeout is set. + #[tracing::instrument(level = "trace", skip(self, response_channel), fields(subsystem = LOG_TARGET))] + pub async fn wait_for_activation(&mut self, hash: Hash, response_channel: oneshot::Sender>) { + self.send_and_log_error(Event::ExternalRequest(ExternalRequest::WaitForActivation { + hash, + response_channel + })).await + } + + /// Tell `Overseer` to shutdown. + #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] + pub async fn stop(&mut self) { + self.send_and_log_error(Event::Stop).await + } + + async fn send_and_log_error(&mut self, event: Event) { + if self.events_tx.send(event).await.is_err() { + tracing::info!(target: LOG_TARGET, "Failed to send an event to Overseer"); + } + } +} diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index af21a83f3455..bd392977bfc7 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -136,209 +136,6 @@ impl HeadSupportsParachains for Arc where } } -/// This struct is passed as an argument to create a new instance of an [`Overseer`]. -/// -/// As any entity that satisfies the interface may act as a [`Subsystem`] this allows -/// mocking in the test code: -/// -/// Each [`Subsystem`] is supposed to implement some interface that is generic over -/// message type that is specific to this [`Subsystem`]. At the moment not all -/// subsystems are implemented and the rest can be mocked with the [`DummySubsystem`]. -#[derive(Debug, Clone, AllSubsystemsGen)] -pub struct AllSubsystems< - CV = (), CB = (), CS = (), SD = (), AD = (), AR = (), BS = (), BD = (), P = (), - RA = (), AS = (), NB = (), CA = (), CG = (), CP = (), ApD = (), ApV = (), - GS = (), -> { - /// A candidate validation subsystem. - pub candidate_validation: CV, - /// A candidate backing subsystem. - pub candidate_backing: CB, - /// A candidate selection subsystem. - pub candidate_selection: CS, - /// A statement distribution subsystem. - pub statement_distribution: SD, - /// An availability distribution subsystem. - pub availability_distribution: AD, - /// An availability recovery subsystem. - pub availability_recovery: AR, - /// A bitfield signing subsystem. - pub bitfield_signing: BS, - /// A bitfield distribution subsystem. - pub bitfield_distribution: BD, - /// A provisioner subsystem. - pub provisioner: P, - /// A runtime API subsystem. - pub runtime_api: RA, - /// An availability store subsystem. - pub availability_store: AS, - /// A network bridge subsystem. - pub network_bridge: NB, - /// A Chain API subsystem. - pub chain_api: CA, - /// A Collation Generation subsystem. - pub collation_generation: CG, - /// A Collator Protocol subsystem. - pub collator_protocol: CP, - /// An Approval Distribution subsystem. - pub approval_distribution: ApD, - /// An Approval Voting subsystem. - pub approval_voting: ApV, - /// A Connection Request Issuer subsystem. - pub gossip_support: GS, -} - -impl - AllSubsystems -{ - /// Create a new instance of [`AllSubsystems`]. - /// - /// Each subsystem is set to [`DummySystem`]. - /// - ///# Note - /// - /// Because of a bug in rustc it is required that when calling this function, - /// you provide a "random" type for the first generic parameter: - /// - /// ``` - /// polkadot_overseer::AllSubsystems::<()>::dummy(); - /// ``` - pub fn dummy() -> AllSubsystems< - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - > { - AllSubsystems { - candidate_validation: DummySubsystem, - candidate_backing: DummySubsystem, - candidate_selection: DummySubsystem, - statement_distribution: DummySubsystem, - availability_distribution: DummySubsystem, - availability_recovery: DummySubsystem, - bitfield_signing: DummySubsystem, - bitfield_distribution: DummySubsystem, - provisioner: DummySubsystem, - runtime_api: DummySubsystem, - availability_store: DummySubsystem, - network_bridge: DummySubsystem, - chain_api: DummySubsystem, - collation_generation: DummySubsystem, - collator_protocol: DummySubsystem, - approval_distribution: DummySubsystem, - approval_voting: DummySubsystem, - gossip_support: DummySubsystem, - } - } - - fn as_ref(&self) -> AllSubsystems<&'_ CV, &'_ CB, &'_ CS, &'_ SD, &'_ AD, &'_ AR, &'_ BS, &'_ BD, &'_ P, &'_ RA, &'_ AS, &'_ NB, &'_ CA, &'_ CG, &'_ CP, &'_ ApD, &'_ ApV, &'_ GS> { - AllSubsystems { - candidate_validation: &self.candidate_validation, - candidate_backing: &self.candidate_backing, - candidate_selection: &self.candidate_selection, - statement_distribution: &self.statement_distribution, - availability_distribution: &self.availability_distribution, - availability_recovery: &self.availability_recovery, - bitfield_signing: &self.bitfield_signing, - bitfield_distribution: &self.bitfield_distribution, - provisioner: &self.provisioner, - runtime_api: &self.runtime_api, - availability_store: &self.availability_store, - network_bridge: &self.network_bridge, - chain_api: &self.chain_api, - collation_generation: &self.collation_generation, - collator_protocol: &self.collator_protocol, - approval_distribution: &self.approval_distribution, - approval_voting: &self.approval_voting, - gossip_support: &self.gossip_support, - } - } - - fn map_subsystems(self, m: M) - -> AllSubsystems< - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - > - where - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem

, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - { - AllSubsystems { - candidate_validation: m.map_subsystem(self.candidate_validation), - candidate_backing: m.map_subsystem(self.candidate_backing), - candidate_selection: m.map_subsystem(self.candidate_selection), - statement_distribution: m.map_subsystem(self.statement_distribution), - availability_distribution: m.map_subsystem(self.availability_distribution), - availability_recovery: m.map_subsystem(self.availability_recovery), - bitfield_signing: m.map_subsystem(self.bitfield_signing), - bitfield_distribution: m.map_subsystem(self.bitfield_distribution), - provisioner: m.map_subsystem(self.provisioner), - runtime_api: m.map_subsystem(self.runtime_api), - availability_store: m.map_subsystem(self.availability_store), - network_bridge: m.map_subsystem(self.network_bridge), - chain_api: m.map_subsystem(self.chain_api), - collation_generation: m.map_subsystem(self.collation_generation), - collator_protocol: m.map_subsystem(self.collator_protocol), - approval_distribution: m.map_subsystem(self.approval_distribution), - approval_voting: m.map_subsystem(self.approval_voting), - gossip_support: m.map_subsystem(self.gossip_support), - } - } -} - -type AllSubsystemsSame = AllSubsystems< - T, T, T, T, T, - T, T, T, T, T, - T, T, T, T, T, - T, T, T, ->; - /// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. /// /// It wraps a system-wide [`AllMessages`] type that represents all possible @@ -364,6 +161,17 @@ enum ToOverseer { }, } + + +impl Debug for ToOverseer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ToOverseer::SpawnJob { .. } => write!(f, "OverseerMessage::Spawn(..)"), + ToOverseer::SpawnBlockingJob { .. } => write!(f, "OverseerMessage::SpawnBlocking(..)") + } + } +} + /// An event telling the `Overseer` on the particular block /// that has been imported or finalized. /// @@ -417,59 +225,7 @@ enum ExternalRequest { }, } -/// A handler used to communicate with the [`Overseer`]. -/// -/// [`Overseer`]: struct.Overseer.html -#[derive(Clone)] -pub struct OverseerHandler { - events_tx: metered::MeteredSender, -} - -impl OverseerHandler { - /// Inform the `Overseer` that that some block was imported. - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - pub async fn block_imported(&mut self, block: BlockInfo) { - self.send_and_log_error(Event::BlockImported(block)).await - } - - /// Send some message to one of the `Subsystem`s. - #[tracing::instrument(level = "trace", skip(self, msg), fields(subsystem = LOG_TARGET))] - pub async fn send_msg(&mut self, msg: impl Into) { - self.send_and_log_error(Event::MsgToSubsystem(msg.into())).await - } - - /// Inform the `Overseer` that some block was finalized. - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - pub async fn block_finalized(&mut self, block: BlockInfo) { - self.send_and_log_error(Event::BlockFinalized(block)).await - } - - /// Wait for a block with the given hash to be in the active-leaves set. - /// - /// The response channel responds if the hash was activated and is closed if the hash was deactivated. - /// Note that due the fact the overseer doesn't store the whole active-leaves set, only deltas, - /// the response channel may never return if the hash was deactivated before this call. - /// In this case, it's the caller's responsibility to ensure a timeout is set. - #[tracing::instrument(level = "trace", skip(self, response_channel), fields(subsystem = LOG_TARGET))] - pub async fn wait_for_activation(&mut self, hash: Hash, response_channel: oneshot::Sender>) { - self.send_and_log_error(Event::ExternalRequest(ExternalRequest::WaitForActivation { - hash, - response_channel - })).await - } - - /// Tell `Overseer` to shutdown. - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - pub async fn stop(&mut self) { - self.send_and_log_error(Event::Stop).await - } - async fn send_and_log_error(&mut self, event: Event) { - if self.events_tx.send(event).await.is_err() { - tracing::info!(target: LOG_TARGET, "Failed to send an event to Overseer"); - } - } -} /// Glues together the [`Overseer`] and `BlockchainEvents` by forwarding /// import and finality notifications into the [`OverseerHandler`]. @@ -506,259 +262,6 @@ pub async fn forward_events>( } } -impl Debug for ToOverseer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ToOverseer::SpawnJob { .. } => write!(f, "OverseerMessage::Spawn(..)"), - ToOverseer::SpawnBlockingJob { .. } => write!(f, "OverseerMessage::SpawnBlocking(..)") - } - } -} - -/// A running instance of some [`Subsystem`]. -/// -/// [`Subsystem`]: trait.Subsystem.html -struct SubsystemInstance { - tx_signal: metered::MeteredSender, - tx_bounded: metered::MeteredSender>, - meters: SubsystemMeters, - signals_received: usize, - name: &'static str, -} - -#[derive(Debug)] -struct MessagePacket { - signals_received: usize, - message: T, -} - -fn make_packet(signals_received: usize, message: T) -> MessagePacket { - MessagePacket { - signals_received, - message, - } -} - -// The channels held by every subsystem to communicate with every other subsystem. -#[derive(Debug, Clone)] -struct ChannelsOut { - candidate_validation: metered::MeteredSender>, - candidate_backing: metered::MeteredSender>, - candidate_selection: metered::MeteredSender>, - statement_distribution: metered::MeteredSender>, - availability_distribution: metered::MeteredSender>, - availability_recovery: metered::MeteredSender>, - bitfield_signing: metered::MeteredSender>, - bitfield_distribution: metered::MeteredSender>, - provisioner: metered::MeteredSender>, - runtime_api: metered::MeteredSender>, - availability_store: metered::MeteredSender>, - network_bridge: metered::MeteredSender>, - chain_api: metered::MeteredSender>, - collation_generation: metered::MeteredSender>, - collator_protocol: metered::MeteredSender>, - approval_distribution: metered::MeteredSender>, - approval_voting: metered::MeteredSender>, - gossip_support: metered::MeteredSender>, - - candidate_validation_unbounded: metered::UnboundedMeteredSender>, - candidate_backing_unbounded: metered::UnboundedMeteredSender>, - candidate_selection_unbounded: metered::UnboundedMeteredSender>, - statement_distribution_unbounded: metered::UnboundedMeteredSender>, - availability_distribution_unbounded: metered::UnboundedMeteredSender>, - availability_recovery_unbounded: metered::UnboundedMeteredSender>, - bitfield_signing_unbounded: metered::UnboundedMeteredSender>, - bitfield_distribution_unbounded: metered::UnboundedMeteredSender>, - provisioner_unbounded: metered::UnboundedMeteredSender>, - runtime_api_unbounded: metered::UnboundedMeteredSender>, - availability_store_unbounded: metered::UnboundedMeteredSender>, - network_bridge_unbounded: metered::UnboundedMeteredSender>, - chain_api_unbounded: metered::UnboundedMeteredSender>, - collation_generation_unbounded: metered::UnboundedMeteredSender>, - collator_protocol_unbounded: metered::UnboundedMeteredSender>, - approval_distribution_unbounded: metered::UnboundedMeteredSender>, - approval_voting_unbounded: metered::UnboundedMeteredSender>, - gossip_support_unbounded: metered::UnboundedMeteredSender>, -} - -impl ChannelsOut { - async fn send_and_log_error( - &mut self, - signals_received: usize, - message: AllMessages, - ) { - let res = match message { - AllMessages::CandidateValidation(msg) => { - self.candidate_validation.send(make_packet(signals_received, msg)).await - }, - AllMessages::CandidateBacking(msg) => { - self.candidate_backing.send(make_packet(signals_received, msg)).await - }, - AllMessages::CandidateSelection(msg) => { - self.candidate_selection.send(make_packet(signals_received, msg)).await - }, - AllMessages::StatementDistribution(msg) => { - self.statement_distribution.send(make_packet(signals_received, msg)).await - }, - AllMessages::AvailabilityDistribution(msg) => { - self.availability_distribution.send(make_packet(signals_received, msg)).await - }, - AllMessages::AvailabilityRecovery(msg) => { - self.availability_recovery.send(make_packet(signals_received, msg)).await - }, - AllMessages::BitfieldDistribution(msg) => { - self.bitfield_distribution.send(make_packet(signals_received, msg)).await - }, - AllMessages::BitfieldSigning(msg) => { - self.bitfield_signing.send(make_packet(signals_received, msg)).await - }, - AllMessages::Provisioner(msg) => { - self.provisioner.send(make_packet(signals_received, msg)).await - }, - AllMessages::RuntimeApi(msg) => { - self.runtime_api.send(make_packet(signals_received, msg)).await - }, - AllMessages::AvailabilityStore(msg) => { - self.availability_store.send(make_packet(signals_received, msg)).await - }, - AllMessages::NetworkBridge(msg) => { - self.network_bridge.send(make_packet(signals_received, msg)).await - }, - AllMessages::ChainApi(msg) => { - self.chain_api.send(make_packet(signals_received, msg)).await - }, - AllMessages::CollationGeneration(msg) => { - self.collation_generation.send(make_packet(signals_received, msg)).await - }, - AllMessages::CollatorProtocol(msg) => { - self.collator_protocol.send(make_packet(signals_received, msg)).await - }, - AllMessages::ApprovalDistribution(msg) => { - self.approval_distribution.send(make_packet(signals_received, msg)).await - }, - AllMessages::ApprovalVoting(msg) => { - self.approval_voting.send(make_packet(signals_received, msg)).await - }, - AllMessages::GossipSupport(msg) => { - self.gossip_support.send(make_packet(signals_received, msg)).await - }, - }; - - if res.is_err() { - tracing::debug!( - target: LOG_TARGET, - "Failed to send a message to another subsystem", - ); - } - } - - - fn send_unbounded_and_log_error( - &self, - signals_received: usize, - message: AllMessages, - ) { - let res = match message { - AllMessages::CandidateValidation(msg) => { - self.candidate_validation_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::CandidateBacking(msg) => { - self.candidate_backing_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::CandidateSelection(msg) => { - self.candidate_selection_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::StatementDistribution(msg) => { - self.statement_distribution_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::AvailabilityDistribution(msg) => { - self.availability_distribution_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::AvailabilityRecovery(msg) => { - self.availability_recovery_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::BitfieldDistribution(msg) => { - self.bitfield_distribution_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::BitfieldSigning(msg) => { - self.bitfield_signing_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::Provisioner(msg) => { - self.provisioner_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::RuntimeApi(msg) => { - self.runtime_api_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::AvailabilityStore(msg) => { - self.availability_store_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::NetworkBridge(msg) => { - self.network_bridge_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::ChainApi(msg) => { - self.chain_api_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::CollationGeneration(msg) => { - self.collation_generation_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::CollatorProtocol(msg) => { - self.collator_protocol_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::ApprovalDistribution(msg) => { - self.approval_distribution_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::ApprovalVoting(msg) => { - self.approval_voting_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::GossipSupport(msg) => { - self.gossip_support_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - }; - - if res.is_err() { - tracing::debug!( - target: LOG_TARGET, - "Failed to send a message to another subsystem", - ); - } - } -} type SubsystemIncomingMessages = stream::Select< metered::MeteredReceiver>, @@ -778,33 +281,6 @@ impl SignalsReceived { } } -/// A sender from subsystems to other subsystems. -#[derive(Debug, Clone)] -pub struct OverseerSubsystemSender { - channels: ChannelsOut, - signals_received: SignalsReceived, -} - -#[async_trait::async_trait] -impl SubsystemSender for OverseerSubsystemSender { - async fn send_message(&mut self, msg: AllMessages) { - self.channels.send_and_log_error(self.signals_received.load(), msg).await; - } - - async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send - { - // This can definitely be optimized if necessary. - for msg in msgs { - self.send_message(msg).await; - } - } - - fn send_unbounded_message(&mut self, msg: AllMessages) { - self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); - } -} - /// A context type that is given to the [`Subsystem`] upon spawning. /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s /// or to spawn it's [`SubsystemJob`]s. @@ -2169,67 +1645,6 @@ where } } -enum TaskKind { - Regular, - Blocking, -} - -fn spawn( - spawner: &mut S, - message_tx: metered::MeteredSender>, - message_rx: SubsystemIncomingMessages, - unbounded_meter: metered::Meter, - to_subsystems: ChannelsOut, - to_overseer_tx: metered::UnboundedMeteredSender, - s: impl Subsystem>, - metrics: &Metrics, - futures: &mut FuturesUnordered>>, - task_kind: TaskKind, -) -> SubsystemResult> { - let (signal_tx, signal_rx) = metered::channel(SIGNAL_CHANNEL_CAPACITY); - let ctx = OverseerSubsystemContext::new( - signal_rx, - message_rx, - to_subsystems, - to_overseer_tx, - metrics.clone(), - ); - let SpawnedSubsystem { future, name } = s.start(ctx); - - let (tx, rx) = oneshot::channel(); - - let fut = Box::pin(async move { - if let Err(e) = future.await { - tracing::error!(subsystem=name, err = ?e, "subsystem exited with error"); - } else { - tracing::debug!(subsystem=name, "subsystem exited without an error"); - } - let _ = tx.send(()); - }); - - match task_kind { - TaskKind::Regular => spawner.spawn(name, fut), - TaskKind::Blocking => spawner.spawn_blocking(name, fut), - } - - futures.push(Box::pin(rx.map(|e| { tracing::warn!(err = ?e, "dropping error"); Ok(()) }))); - - let instance = Some(SubsystemInstance { - meters: SubsystemMeters { - unbounded: unbounded_meter, - bounded: message_tx.meter().clone(), - signals: signal_tx.meter().clone(), - }, - tx_signal: signal_tx, - tx_bounded: message_tx, - signals_received: 0, - name, - }); - - Ok(OverseenSubsystem { - instance, - }) -} #[cfg(test)] mod tests; diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs deleted file mode 100644 index e69de29bb2d1..000000000000 From 6d05f4e603f1316245585b99b5630bdd2a233a11 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 11 May 2021 15:44:36 +0200 Subject: [PATCH 021/161] refactor --- Cargo.lock | 3 ++ node/subsystem/Cargo.toml | 1 - node/subsystem/src/errors.rs | 54 ++++++++++++++++++++++++++++++++ node/subsystem/src/lib.rs | 52 +------------------------------ node/subsystem/src/messages.rs | 57 ---------------------------------- 5 files changed, 58 insertions(+), 109 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e7d9a077829..20f3ec2d4f78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6263,8 +6263,11 @@ dependencies = [ name = "polkadot-overseer-gen" version = "0.1.0" dependencies = [ + "futures 0.3.14", + "metered-channel", "polkadot-overseer-gen-proc-macro", "tracing", + "trybuild", ] [[package]] diff --git a/node/subsystem/Cargo.toml b/node/subsystem/Cargo.toml index 590af20343e8..7b2e8e6055ce 100644 --- a/node/subsystem/Cargo.toml +++ b/node/subsystem/Cargo.toml @@ -22,7 +22,6 @@ polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-primitives = { path = "../../primitives" } polkadot-statement-table = { path = "../../statement-table" } polkadot-node-jaeger = { path = "../jaeger" } -polkadot-procmacro-subsystem-dispatch-gen = { path = "dispatch-gen" } sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } smallvec = "1.6.1" sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/subsystem/src/errors.rs b/node/subsystem/src/errors.rs index acd33cff1dfb..6b7832bf1e7a 100644 --- a/node/subsystem/src/errors.rs +++ b/node/subsystem/src/errors.rs @@ -77,3 +77,57 @@ impl std::fmt::Display for RecoveryError { } impl std::error::Error for RecoveryError {} + + + + +/// An error type that describes faults that may happen +/// +/// These are: +/// * Channels being closed +/// * Subsystems dying when they are not expected to +/// * Subsystems not dying when they are told to die +/// * etc. +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum SubsystemError { + #[error(transparent)] + NotifyCancellation(#[from] oneshot::Canceled), + + #[error(transparent)] + QueueError(#[from] mpsc::SendError), + + #[error(transparent)] + TaskSpawn(#[from] futures::task::SpawnError), + + #[error(transparent)] + Infallible(#[from] std::convert::Infallible), + + #[error(transparent)] + Prometheus(#[from] substrate_prometheus_endpoint::PrometheusError), + + #[error(transparent)] + Jaeger(#[from] JaegerError), + + #[error("Failed to {0}")] + Context(String), + + #[error("Subsystem stalled: {0}")] + SubsystemStalled(&'static str), + + /// Per origin (or subsystem) annotations to wrap an error. + #[error("Error originated in {origin}")] + FromOrigin { + /// An additional anotation tag for the origin of `source`. + origin: &'static str, + /// The wrapped error. Marked as source for tracking the error chain. + #[source] source: Box + }, +} + +impl SubsystemError { + /// Adds a `str` as `origin` to the given error `err`. + pub fn with_origin(origin: &'static str, err: E) -> Self { + Self::FromOrigin { origin, source: Box::new(err) } + } +} diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 3cc4894a493e..fd4c80cb9392 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -39,6 +39,7 @@ pub use polkadot_node_jaeger as jaeger; pub use jaeger::*; use self::messages::AllMessages; +pub use self::errors::SubsystemError; /// How many slots are stack-reserved for active leaves updates /// @@ -144,57 +145,6 @@ pub enum FromOverseer { } -/// An error type that describes faults that may happen -/// -/// These are: -/// * Channels being closed -/// * Subsystems dying when they are not expected to -/// * Subsystems not dying when they are told to die -/// * etc. -#[derive(thiserror::Error, Debug)] -#[allow(missing_docs)] -pub enum SubsystemError { - #[error(transparent)] - NotifyCancellation(#[from] oneshot::Canceled), - - #[error(transparent)] - QueueError(#[from] mpsc::SendError), - - #[error(transparent)] - TaskSpawn(#[from] futures::task::SpawnError), - - #[error(transparent)] - Infallible(#[from] std::convert::Infallible), - - #[error(transparent)] - Prometheus(#[from] substrate_prometheus_endpoint::PrometheusError), - - #[error(transparent)] - Jaeger(#[from] JaegerError), - - #[error("Failed to {0}")] - Context(String), - - #[error("Subsystem stalled: {0}")] - SubsystemStalled(&'static str), - - /// Per origin (or subsystem) annotations to wrap an error. - #[error("Error originated in {origin}")] - FromOrigin { - /// An additional anotation tag for the origin of `source`. - origin: &'static str, - /// The wrapped error. Marked as source for tracking the error chain. - #[source] source: Box - }, -} - -impl SubsystemError { - /// Adds a `str` as `origin` to the given error `err`. - pub fn with_origin(origin: &'static str, err: E) -> Self { - Self::FromOrigin { origin, source: Box::new(err) } - } -} - /// An asynchronous subsystem task.. /// /// In essence it's just a newtype wrapping a `BoxFuture`. diff --git a/node/subsystem/src/messages.rs b/node/subsystem/src/messages.rs index a0d274ec6fd4..6728f2fd7db3 100644 --- a/node/subsystem/src/messages.rs +++ b/node/subsystem/src/messages.rs @@ -674,63 +674,6 @@ pub enum ApprovalDistributionMessage { pub enum GossipSupportMessage { } -/// A message type tying together all message types that are used across Subsystems. -#[subsystem_dispatch_gen(NetworkBridgeEvent)] -#[derive(Debug, derive_more::From)] -pub enum AllMessages { - /// Message for the validation subsystem. - #[skip] - CandidateValidation(CandidateValidationMessage), - /// Message for the candidate backing subsystem. - #[skip] - CandidateBacking(CandidateBackingMessage), - /// Message for the candidate selection subsystem. - #[skip] - CandidateSelection(CandidateSelectionMessage), - /// Message for the Chain API subsystem. - #[skip] - ChainApi(ChainApiMessage), - /// Message for the Collator Protocol subsystem. - #[skip] - CollatorProtocol(CollatorProtocolMessage), - /// Message for the statement distribution subsystem. - StatementDistribution(StatementDistributionMessage), - /// Message for the availability distribution subsystem. - #[skip] - AvailabilityDistribution(AvailabilityDistributionMessage), - /// Message for the availability recovery subsystem. - #[skip] - AvailabilityRecovery(AvailabilityRecoveryMessage), - /// Message for the bitfield distribution subsystem. - BitfieldDistribution(BitfieldDistributionMessage), - /// Message for the bitfield signing subsystem. - #[skip] - BitfieldSigning(BitfieldSigningMessage), - /// Message for the Provisioner subsystem. - #[skip] - Provisioner(ProvisionerMessage), - /// Message for the Runtime API subsystem. - #[skip] - RuntimeApi(RuntimeApiMessage), - /// Message for the availability store subsystem. - #[skip] - AvailabilityStore(AvailabilityStoreMessage), - /// Message for the network bridge subsystem. - #[skip] - NetworkBridge(NetworkBridgeMessage), - /// Message for the Collation Generation subsystem. - #[skip] - CollationGeneration(CollationGenerationMessage), - /// Message for the Approval Voting subsystem. - #[skip] - ApprovalVoting(ApprovalVotingMessage), - /// Message for the Approval Distribution subsystem. - ApprovalDistribution(ApprovalDistributionMessage), - /// Message for the Gossip Support subsystem. - #[skip] - GossipSupport(GossipSupportMessage), -} - impl From> for AvailabilityDistributionMessage { fn from(req: IncomingRequest) -> Self { Self::PoVFetchingRequest(req) From 7bdd3885819ab9b33a9233456f0f2a2937b9f553 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 11 May 2021 15:45:05 +0200 Subject: [PATCH 022/161] drop dispatch-gen crate, obsolete --- Cargo.lock | 12 - Cargo.toml | 1 - node/subsystem/dispatch-gen/Cargo.toml | 18 -- node/subsystem/dispatch-gen/src/lib.rs | 208 ------------------ .../tests/ui/err-01-missing-skip.rs | 37 ---- .../tests/ui/err-01-missing-skip.stderr | 14 -- .../tests/ui/err-02-missing-from.rs | 41 ---- .../tests/ui/err-02-missing-from.stderr | 10 - .../tests/ui/ok-01-with-intermediate.rs | 48 ---- 9 files changed, 389 deletions(-) delete mode 100644 node/subsystem/dispatch-gen/Cargo.toml delete mode 100644 node/subsystem/dispatch-gen/src/lib.rs delete mode 100644 node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.rs delete mode 100644 node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.stderr delete mode 100644 node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.rs delete mode 100644 node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.stderr delete mode 100644 node/subsystem/dispatch-gen/tests/ui/ok-01-with-intermediate.rs diff --git a/Cargo.lock b/Cargo.lock index 20f3ec2d4f78..90f048f049c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6173,7 +6173,6 @@ dependencies = [ "polkadot-node-primitives", "polkadot-node-subsystem-test-helpers", "polkadot-primitives", - "polkadot-procmacro-subsystem-dispatch-gen", "polkadot-statement-table", "sc-network", "smallvec 1.6.1", @@ -6326,17 +6325,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "polkadot-procmacro-subsystem-dispatch-gen" -version = "0.1.0" -dependencies = [ - "assert_matches", - "proc-macro2", - "quote", - "syn", - "trybuild", -] - [[package]] name = "polkadot-rpc" version = "0.9.1" diff --git a/Cargo.toml b/Cargo.toml index 3aa64eb1dc10..9f298bd38fea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,6 @@ members = [ "node/primitives", "node/service", "node/subsystem", - "node/subsystem/dispatch-gen", "node/subsystem-test-helpers", "node/subsystem-util", "node/jaeger", diff --git a/node/subsystem/dispatch-gen/Cargo.toml b/node/subsystem/dispatch-gen/Cargo.toml deleted file mode 100644 index 09de1362c92c..000000000000 --- a/node/subsystem/dispatch-gen/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "polkadot-procmacro-subsystem-dispatch-gen" -version = "0.1.0" -authors = ["Parity Technologies "] -edition = "2018" -description = "Small proc macro to create the distribution code for network events" - -[lib] -proc-macro = true - -[dependencies] -syn = { version = "1.0.60", features = ["full"] } -quote = "1.0.9" -proc-macro2 = "1.0.24" -assert_matches = "1.5.0" - -[dev-dependencies] -trybuild = "1.0.41" diff --git a/node/subsystem/dispatch-gen/src/lib.rs b/node/subsystem/dispatch-gen/src/lib.rs deleted file mode 100644 index 737712639cff..000000000000 --- a/node/subsystem/dispatch-gen/src/lib.rs +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use proc_macro2::TokenStream; -use quote::{quote, ToTokens}; -use std::fmt; -use syn::{parse2, Error, Fields, FieldsNamed, FieldsUnnamed, Ident, ItemEnum, Path, Result, Type, Variant}; - -#[proc_macro_attribute] -pub fn subsystem_dispatch_gen(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream { - let attr: TokenStream = attr.into(); - let item: TokenStream = item.into(); - let mut backup = item.clone(); - impl_subsystem_dispatch_gen(attr.into(), item).unwrap_or_else(|err| { - backup.extend(err.to_compile_error()); - backup - }).into() -} - -/// An enum variant without base type. -#[derive(Clone)] -struct EnumVariantDispatchWithTy { - // enum ty name - ty: Ident, - // variant - variant: EnumVariantDispatch, -} - -impl fmt::Debug for EnumVariantDispatchWithTy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}::{:?}", self.ty, self.variant) - } -} - -impl ToTokens for EnumVariantDispatchWithTy { - fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { - if let Some(inner) = &self.variant.inner { - let enum_name = &self.ty; - let variant_name = &self.variant.name; - - let quoted = quote! { - #enum_name::#variant_name(#inner::from(event)) - }; - quoted.to_tokens(tokens); - } - } -} - -/// An enum variant without the base type, contains the relevant inner type. -#[derive(Clone)] -struct EnumVariantDispatch { - /// variant name - name: Ident, - /// The inner type for which a `From::from` impl is anticipated from the input type. - /// No code will be generated for this enum variant if `inner` is `None`. - inner: Option, -} - -impl fmt::Debug for EnumVariantDispatch { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}(..)", self.name) - } -} - -fn prepare_enum_variant(variant: &mut Variant) -> Result { - let skip = variant.attrs.iter().find(|attr| attr.path.is_ident("skip")).is_some(); - variant.attrs = variant.attrs.iter().filter(|attr| !attr.path.is_ident("skip")).cloned().collect::>(); - - let variant = variant.clone(); - let span = variant.ident.span(); - let inner = match variant.fields.clone() { - // look for one called inner - Fields::Named(FieldsNamed { brace_token: _, named }) if !skip => named - .iter() - .find_map( - |field| { - if let Some(ident) = &field.ident { - if ident == "inner" { - return Some(Some(field.ty.clone())) - } - } - None - }, - ) - .ok_or_else(|| { - Error::new(span, "To dispatch with struct enum variant, one element must named `inner`") - })?, - - // technically, if it has no inner types we cound not require the #[skip] annotation, but better make it consistent - Fields::Unnamed(FieldsUnnamed { paren_token: _, unnamed }) if !skip => unnamed - .first() - .map(|field| Some(field.ty.clone())) - .ok_or_else(|| Error::new(span, "Must be annotated with skip, even if no inner types exist."))?, - _ if skip => None, - Fields::Unit => { - return Err(Error::new( - span, - "Must be annotated with #[skip].", - )) - } - Fields::Unnamed(_) => { - return Err(Error::new( - span, - "Must be annotated with #[skip] or have in `inner` element which impls `From<_>`.", - )) - } - Fields::Named(_) => { - return Err(Error::new( - span, - "Must be annotated with #[skip] or the first wrapped type must impl `From<_>`.", - )) - } - }; - - Ok(EnumVariantDispatch { name: variant.ident, inner }) -} - -fn impl_subsystem_dispatch_gen(attr: TokenStream, item: TokenStream) -> Result { - let event_ty = parse2::(attr)?; - - let mut ie = parse2::(item)?; - - let message_enum = ie.ident.clone(); - let variants = ie.variants.iter_mut().try_fold(Vec::::new(), |mut acc, variant| { - let variant = prepare_enum_variant(variant)?; - if variant.inner.is_some() { - acc.push(EnumVariantDispatchWithTy { ty: message_enum.clone(), variant }) - } - Ok::<_, syn::Error>(acc) - })?; - - let mut orig = ie.to_token_stream(); - - let msg = "Generated by #[subsystem_dispatch_gen] proc-macro."; - - orig.extend(quote! { - impl #message_enum { - #[doc = #msg] - pub fn dispatch_iter(event: #event_ty) -> impl Iterator + Send { - let mut iter = None.into_iter(); - - #( - let mut iter = iter.chain(std::iter::once(event.focus().ok().map(|event| { - #variants - }))); - )* - iter.filter_map(|x| x) - } - } - }); - Ok(orig) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn basic() { - let attr = quote! { - NetEvent - }; - - let item = quote! { - /// Documentation. - #[derive(Clone)] - enum AllMessages { - - Sub1(Inner1), - - #[skip] - /// D3 - Sub3, - - /// D4 - #[skip] - Sub4(Inner2), - - /// D2 - Sub2(Inner2), - } - }; - - let output = impl_subsystem_dispatch_gen(attr, item).expect("Simple example always works. qed"); - println!("//generated:"); - println!("{}", output); - } - - #[test] - fn ui() { - let t = trybuild::TestCases::new(); - t.compile_fail("tests/ui/err-*.rs"); - t.pass("tests/ui/ok-*.rs"); - } -} diff --git a/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.rs b/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.rs deleted file mode 100644 index 7248a7181e49..000000000000 --- a/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.rs +++ /dev/null @@ -1,37 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_subsystem_dispatch_gen::subsystem_dispatch_gen; - -/// The event type in question. -#[derive(Clone, Copy)] -enum Event { - Smth, - Else, -} - -impl Event { - fn focus(&self) -> std::result::Result { - unimplemented!("foo") - } -} - -/// This should have a `From` impl but does not. -#[derive(Clone)] -enum Inner { - Foo, - Bar(Event), -} - -#[subsystem_dispatch_gen(Event)] -#[derive(Clone)] -enum AllMessages { - /// Foo - Vvvvvv(Inner), - - /// Missing a `#[skip]` annotation - Uuuuu, -} - -fn main() { - let _x = AllMessages::dispatch_iter(Event::Else); -} diff --git a/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.stderr b/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.stderr deleted file mode 100644 index 855521d2c4ef..000000000000 --- a/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.stderr +++ /dev/null @@ -1,14 +0,0 @@ -error: Must be annotated with #[skip]. - --> $DIR/err-01-missing-skip.rs:32:5 - | -32 | Uuuuu, - | ^^^^^ - -error[E0599]: no variant or associated item named `dispatch_iter` found for enum `AllMessages` in the current scope - --> $DIR/err-01-missing-skip.rs:36:27 - | -27 | enum AllMessages { - | ---------------- variant or associated item `dispatch_iter` not found here -... -36 | let _x = AllMessages::dispatch_iter(Event::Else); - | ^^^^^^^^^^^^^ variant or associated item not found in `AllMessages` diff --git a/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.rs b/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.rs deleted file mode 100644 index a7abef2c8709..000000000000 --- a/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.rs +++ /dev/null @@ -1,41 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_subsystem_dispatch_gen::subsystem_dispatch_gen; - -/// The event type in question. -#[derive(Clone, Copy, Debug)] -enum Event { - Smth, - Else, -} - -impl Event { - fn focus(&self) -> std::result::Result { - Ok(Intermediate(self.clone())) - } -} - -#[derive(Debug, Clone)] -struct Intermediate(Event); - - -/// This should have a `From` impl but does not. -#[derive(Debug, Clone)] -enum Inner { - Foo, - Bar(Intermediate), -} - -#[subsystem_dispatch_gen(Event)] -#[derive(Clone)] -enum AllMessages { - /// Foo - Vvvvvv(Inner), - - #[skip] - Uuuuu, -} - -fn main() { - let _x = AllMessages::dispatch_iter(Event::Else); -} diff --git a/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.stderr b/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.stderr deleted file mode 100644 index bf82201a7e40..000000000000 --- a/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.stderr +++ /dev/null @@ -1,10 +0,0 @@ -error[E0308]: mismatched types - --> $DIR/err-02-missing-from.rs:29:1 - | -29 | #[subsystem_dispatch_gen(Event)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | | - | expected enum `Inner`, found struct `Intermediate` - | help: try using a variant of the expected enum: `Inner::Bar(#[subsystem_dispatch_gen(Event)])` - | - = note: this error originates in an attribute macro (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/node/subsystem/dispatch-gen/tests/ui/ok-01-with-intermediate.rs b/node/subsystem/dispatch-gen/tests/ui/ok-01-with-intermediate.rs deleted file mode 100644 index b160bf9ce1c1..000000000000 --- a/node/subsystem/dispatch-gen/tests/ui/ok-01-with-intermediate.rs +++ /dev/null @@ -1,48 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_subsystem_dispatch_gen::subsystem_dispatch_gen; - -/// The event type in question. -#[derive(Clone, Copy, Debug)] -enum Event { - Smth, - Else, -} - -impl Event { - fn focus(&self) -> std::result::Result { - Ok(Intermediate(self.clone())) - } -} - - -#[derive(Debug, Clone)] -struct Intermediate(Event); - - -/// This should have a `From` impl but does not. -#[derive(Clone, Debug)] -enum Inner { - Foo, - Bar(Intermediate), -} - -impl From for Inner { - fn from(src: Intermediate) -> Self { - Inner::Bar(src) - } -} - -#[subsystem_dispatch_gen(Event)] -#[derive(Clone)] -enum AllMessages { - /// Foo - Vvvvvv(Inner), - - #[skip] - Uuuuu, -} - -fn main() { - let _x = AllMessages::dispatch_iter(Event::Else); -} From 581df1f27f4627846ed03a8c5dd321e33592f989 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 11 May 2021 16:27:06 +0200 Subject: [PATCH 023/161] more impl details --- .../proc-macro/src/impl_overseer.rs | 153 ++++++++---------- .../overseer-gen/proc-macro/src/parse.rs | 111 +++++-------- .../overseer-gen/proc-macro/src/tests.rs | 20 ++- node/overseer/overseer-gen/src/lib.rs | 6 +- 4 files changed, 132 insertions(+), 158 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index d5bdd9957cb2..5421206b3fce 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -26,7 +26,7 @@ pub(crate) fn impl_overseer_struct( where Ctx: SubsystemContext, S: SpawnNamed, - #( #field_ty : Subsystem )* + #( #field_ty : Subsystem, )* }; let message_channel_capacity = info.message_channel_capacity; @@ -121,6 +121,8 @@ pub(crate) fn impl_builder( let baggage_generic_ty = &info.baggage_generic_types(); let baggage_name = &info.baggage_names(); + let blocking = &info.subsystems().iter().map(|x| x.blocking).collect::>(); + let generics = quote! { < Ctx, S, #( #baggage_generic_ty, )* #( #field_ty, )* > }; @@ -169,14 +171,6 @@ pub(crate) fn impl_builder( metered::UnboundedMeteredSender, ) -> Ctx, { - let overseer = #overseer_name :: #generics { - #( - #field_name : self. #field_name .unwrap(), - )* - #( - #baggage_name : self. #baggage_name .unwrap(), - )* - }; let (events_tx, events_rx) = ::metered::channel(SIGNAL_CHANNEL_CAPACITY); @@ -187,9 +181,6 @@ pub(crate) fn impl_builder( let (to_overseer_tx, to_overseer_rx) = metered::unbounded(); - let mut running_subsystems = FuturesUnordered::new(); - - let channels_out = { #( let (#channel_name_tx, #channel_name_rx) = ::metered::channel::>(CHANNEL_CAPACITY); @@ -209,87 +200,83 @@ pub(crate) fn impl_builder( } }; - // #( #launch subsystem ) - - enum TaskKind { - Regular, - Blocking, - } - let spawner = &mut overseer.spawner; - let mut spawn = | - message_tx: metered::MeteredSender>, - message_rx: SubsystemIncomingMessages, - unbounded_meter: metered::Meter, - to_subsystems: ChannelsOut, - to_overseer_tx: metered::UnboundedMeteredSender, - s: impl Subsystem>, - metrics: &Metrics, - futures: &mut FuturesUnordered>>, - task_kind: TaskKind, - | -> SubsystemResult> - { - let (signal_tx, signal_rx) = metered::channel(SIGNAL_CHANNEL_CAPACITY); - let ctx = create_ctx( - signal_rx, - message_rx, - to_subsystems, - to_overseer_tx, - ); - let SpawnedSubsystem { future, name } = s.start(ctx); - - let (terminated_tx, terminated_rx) = oneshot::channel(); - - let fut = Box::pin(async move { - if let Err(e) = future.await { - tracing::error!(subsystem=name, err = ?e, "subsystem exited with error"); - } else { - tracing::debug!(subsystem=name, "subsystem exited without an error"); - } - let _ = terminated_tx.send(()); - }); + let mut running_subsystems = FuturesUnordered::>>::new(); - match task_kind { - TaskKind::Regular => spawner.spawn(name, fut), - TaskKind::Blocking => spawner.spawn_blocking(name, fut), - } + #( + let #field_name: OverseenSubsystem< #message_wrapper > = { - futures.push(Box::pin(terminated_rx.map(|e| { tracing::warn!(err = ?e, "dropping error"); Ok(()) }))); + let unbounded_meter = #channel_name_unbounded_tx .meter().clone(); - let instance = Some(SubsystemInstance::< #message_wrapper > { - meters: SubsystemMeters { - unbounded: unbounded_meter, - bounded: message_tx.meter().clone(), - signals: signal_tx.meter().clone(), - }, - tx_signal: signal_tx, - tx_bounded: message_tx, - signals_received: 0, - name, - }); - - Ok(OverseenSubsystem::< #message_wrapper > { - instance, - }) - }; + let message_tx: metered::MeteredSender> = #channel_name_tx; - - #( - let #field_name = spawn( - #channel_name_tx, - stream::select( + let message_rx: SubsystemIncomingMessages< #message_wrapper > = stream::select( #channel_name_tx, #channel_name_unbounded_tx, - ), - channels_out.clone(), - to_overseer_tx.clone(), - #field_name, - &mut running_subsystems, - TaskKind::Blocking, - )?; + ); + + let (signal_tx, signal_rx) = metered::channel(SIGNAL_CHANNEL_CAPACITY); + + let ctx = create_ctx( + signal_rx, + message_rx, + channels_out.clone(), + to_overseer_tx.clone(), + ); + + let SpawnedSubsystem { future, name } = #field_name .start(ctx); + + let (terminated_tx, terminated_rx) = oneshot::channel(); + + let fut = Box::pin(async move { + if let Err(e) = future.await { + tracing::error!(subsystem=name, err = ?e, "subsystem exited with error"); + } else { + tracing::debug!(subsystem=name, "subsystem exited without an error"); + } + let _ = terminated_tx.send(()); + }); + + if #blocking { + spawner.spawn_blocking(name, fut); + } else { + spawner.spawn(name, fut); + } + + futures.push(Box::pin(terminated_rx.map(|e| { tracing::warn!(err = ?e, "dropping error"); Ok(()) }))); + + let instance = Some(SubsystemInstance::< #message_wrapper > { + meters: SubsystemMeters { + unbounded: unbounded_meter, + bounded: message_tx.meter().clone(), + signals: signal_tx.meter().clone(), + }, + tx_signal: signal_tx, + tx_bounded: message_tx, + signals_received: 0, + name, + }); + + OverseenSubsystem::< #message_wrapper > { + instance, + } + }; )* + let overseer = #overseer_name :: #generics { + #( + #field_name : self. #field_name .unwrap(), + )* + + #( + #baggage_name : self. #baggage_name .unwrap(), + )* + + running_instance, + spawner, + }; + (overseer, handler) } } diff --git a/node/overseer/overseer-gen/proc-macro/src/parse.rs b/node/overseer/overseer-gen/proc-macro/src/parse.rs index cc3052d77bdb..28509e9426d3 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse.rs @@ -1,5 +1,5 @@ use proc_macro2::Span; -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, hash_map::RandomState}; use syn::{AttrStyle, Path}; use syn::punctuated::Punctuated; use syn::parse::Parse; @@ -119,7 +119,7 @@ impl Parse for AttrArgs { let _paren = syn::parenthesized!(content in input); let items: Punctuated<_, Token![,]> = content.parse_terminated(AttrItem::parse)?; - let mut unique = HashMap::<&str, AttrItem, std::collections::hash_map::RandomState>::default(); + let mut unique = HashMap::<&str, AttrItem, RandomState>::default(); for item in items { if let Some(first) = unique.insert(item.key(), item.clone()) { let mut e = Error::new(item.span(), "Duplicate definition found"); @@ -174,28 +174,48 @@ pub(crate) struct SubSystemTag { #[allow(dead_code)] pub(crate) paren_token: Paren, pub(crate) no_dispatch: bool, + pub(crate) blocking: bool, pub(crate) consumes: Punctuated, } impl Parse for SubSystemTag { fn parse(input: syn::parse::ParseStream) -> Result { + let attrs = Attribute::parse_outer(input)?; + let content; + let paren_token = syn::parenthesized!(content in input); + + let parse_tags = || -> Result> { + if content.peek(Ident) && content.peek2(Token![,]) { + let ident = content.parse::()?; + let _ = content.parse::()?; + Ok(Some(ident)) + } else { + Ok(None) + } + }; + + let mut unique = HashSet::<_, RandomState>::default(); + while let Some(ident) = parse_tags()? { + if ident != "no_dispatch" && ident != "blocking" { + return Err(Error::new(ident.span(), "Allowed tags are only `no_dispatch` or `blocking`.")) + } + if !unique.insert(ident.to_string()) { + return Err(Error::new(ident.span(), "Allowed tags are only `no_dispatch` or `blocking`.")) + } + } + let no_dispatch = unique.get("no_dispatch").is_some(); + let blocking = unique.get("blocking").is_some(); + + + let consumes = content.parse_terminated(Ident::parse)?; + Ok(Self { - attrs: Attribute::parse_outer(input)?, - paren_token: syn::parenthesized!(content in input), - no_dispatch: { - if content.peek(Ident) && content.peek2(Token![,]) { - let ident = content.parse::()?; - if ident != "no_dispatch" { - return Err(Error::new(ident.span(), "Allowed tags is only `no_dispatch` at this time")) - } - let _ = content.parse::()?; - true - } else { - false - } - }, - consumes: content.parse_terminated(Ident::parse)?, + attrs, + paren_token, + no_dispatch, + blocking, + consumes, }) } } @@ -310,9 +330,9 @@ pub(crate) fn parse_overseer_struct_field( } let mut consumes_idents = Vec::with_capacity(attrs.len()); - let variant = syn::parse2::(dbg!(attr_tokens.clone()))?; + let variant = syn::parse2::(attr_tokens.clone())?; if variant.consumes.len() != 1 { - return Err(Error::new(attr_tokens.span(), "Currently only exactly one message can be consumed per subsystem.")) + return Err(Error::new(attr_tokens.span(), "Exactly one message can be consumed per subsystem.")) } consumes_idents.extend(variant.consumes.into_iter()); @@ -343,56 +363,3 @@ pub(crate) fn parse_overseer_struct_field( } Ok((subsystems, baggage)) } - - -// /// Extend the originally provided `Generics` with those generated by the subsystems, -// /// namely `Sub#N`, plus one more `Ctx` which is generic over the subsystem context to use. -// pub(crate) fn extra_generics_combine_generics( -// orig_generics: &Generics, -// subsystems: &[SubSysField], -// ) -> Result { -// let mut combined_generics = orig_generics.clone(); - -// let mut ctx_generic = GenericParam::Type( -// syn::parse::(quote! { -// Ctx: SubsystemContext, -// }) -// .unwrap(), -// ); - -// let mut subsys_generics = subsystems.iter().map(|ssf| { -// let subsys_generic_name = ssf.generic.clone(); -// GenericParam::Type( -// syn::parse::(quote! { -// #subsys_generic_name: Subsystem, -// }) -// .unwrap(), -// ) -// }); -// combined_generics.params.extend(Some(ctx_generic)); -// combined_generics.params.extend(subsys_generics); -// Ok(combined_generics) -// } - - -#[cfg(test)] -mod tests { - use syn::parse_quote; - use assert_matches::assert_matches; - use super::*; - - #[test] - fn attr() { - let attr: AttrArgs = parse_quote! { - (gen=AllMessage, event=::some::where::ExternEvent, signal_capacity=111, message_capacity=222,) - }; - assert_matches!(attr, AttrArgs { - message_channel_capacity, - signal_channel_capacity, - .. - } => { - assert_eq!(message_channel_capacity, 222); - assert_eq!(signal_channel_capacity, 111); - }); - } -} diff --git a/node/overseer/overseer-gen/proc-macro/src/tests.rs b/node/overseer/overseer-gen/proc-macro/src/tests.rs index 6046d9248fbd..18d80f07a2ca 100644 --- a/node/overseer/overseer-gen/proc-macro/src/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/tests.rs @@ -1,10 +1,12 @@ use super::*; use quote::quote; +use assert_matches::assert_matches; +use syn::parse_quote; #[test] fn basic() { let attr = quote! { - overloard(event=OverseerSignal,gen=AllMessages) + (event=OverseerSignal,gen=AllMessages) }; let item = quote! { @@ -29,3 +31,19 @@ fn basic() { println!("//generated:"); println!("{}", output); } + + +#[test] +fn attr_parsing_works() { + let attr: AttrArgs = parse_quote! { + (gen=AllMessage, event=::some::why::ExternEvent, signal_capacity=111, message_capacity=222,) + }; + assert_matches!(attr, AttrArgs { + message_channel_capacity, + signal_channel_capacity, + .. + } => { + assert_eq!(message_channel_capacity, 222); + assert_eq!(signal_channel_capacity, 111); + }); +} diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 9e45cc45157b..744881bf0df6 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -77,7 +77,8 @@ struct SubsystemInstance { } -trait MapSubsystem { +/// A helper trait to map a subsystem to smth. else. +pub(crate) trait MapSubsystem { type Output; fn map_subsystem(&self, sub: T) -> Self::Output; @@ -91,7 +92,8 @@ impl MapSubsystem for F where F: Fn(T) -> U { } } -type SubsystemIncomingMessages = ::futures::stream::Select< +/// Incoming messages from both the bounded and unbounded channel. +pub type SubsystemIncomingMessages = ::futures::stream::Select< ::metered::MeteredReceiver>, ::metered::UnboundedMeteredReceiver>, >; From c985c421586478cb0a45c59fbbc48ceef5ea96a5 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 11 May 2021 17:47:34 +0200 Subject: [PATCH 024/161] fixins --- .../proc-macro/src/impl_overseer.rs | 18 ++- .../overseer-gen/proc-macro/src/parse.rs | 29 ++-- node/overseer/src/lib.rs | 136 ++++++++---------- node/subsystem/src/lib.rs | 2 +- 4 files changed, 88 insertions(+), 97 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 5421206b3fce..2843a4cc2ca0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -32,9 +32,6 @@ pub(crate) fn impl_overseer_struct( let message_channel_capacity = info.message_channel_capacity; let signal_channel_capacity = info.signal_channel_capacity; - let spawner_doc = "Responsible for driving the subsystem futures."; - let running_subsystems_doc = "The set of running subsystems."; - let mut ts = quote! { const CHANNEL_CAPACITY: usize = #message_channel_capacity; const SIGNAL_CHANNEL_CAPACITY: usize = #signal_channel_capacity; @@ -48,12 +45,17 @@ pub(crate) fn impl_overseer_struct( #baggage_name: #baggage_ty, )* - #[doc = #spawner_doc] + /// Responsible for driving the subsystem futures. spawner: S, - #[doc = #running_subsystems_doc] + /// The set of running subsystems. running_subsystems: FuturesUnordered>>, + /// Gather running subsystems' outbound streams into one. + to_overseer_rx: Fuse>, + + /// Events that are sent to the overseer from the outside world. + events_rx: metered::MeteredReceiver, } impl #generics #overseer_name #generics #where_clause { @@ -266,15 +268,17 @@ pub(crate) fn impl_builder( let overseer = #overseer_name :: #generics { #( - #field_name : self. #field_name .unwrap(), + #field_name, )* #( #baggage_name : self. #baggage_name .unwrap(), )* - running_instance, spawner, + running_instance, + events_rx, + to_overseer_rx, }; (overseer, handler) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse.rs b/node/overseer/overseer-gen/proc-macro/src/parse.rs index 28509e9426d3..9c17684bb3cc 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse.rs @@ -50,6 +50,7 @@ use syn::parse::ParseBuffer; #[derive(Clone, Debug)] enum AttrItem { ExternEventType(Path), + ExternOverseerSignalType(Path), MessageWrapperName(Ident), SignalChannelCapacity(LitInt), MessageChannelCapacity(LitInt), @@ -62,6 +63,7 @@ impl Spanned for AttrItem { AttrItem::MessageWrapperName(x) => x.span(), AttrItem::SignalChannelCapacity(x) => x.span(), AttrItem::MessageChannelCapacity(x) => x.span(), + AttrItem::ExternOverseerSignalType(x) => x.span(), } } } @@ -70,6 +72,7 @@ impl AttrItem { fn key(&self) -> &'static str { match self { AttrItem::ExternEventType(_) => "event", + AttrItem::ExternOverseerSignalType(_) => "signal", AttrItem::MessageWrapperName(_) => "gen", AttrItem::SignalChannelCapacity(_) => "signal_capacity", AttrItem::MessageChannelCapacity(_) => "message_capacity", @@ -80,24 +83,25 @@ impl AttrItem { impl Parse for AttrItem { fn parse(input: &ParseBuffer) -> Result { let key = input.parse::()?; - Ok(if key == "event" { - let _ = input.parse::()?; + let span = Span::call_site(); + let _ = input.parse::()?; + Ok(if key == "signal" { + let path = input.parse::()?; + AttrItem::ExternOverseerSignalType(path) + } else if key == "event" { let path = input.parse::()?; AttrItem::ExternEventType(path) } else if key == "gen" { - let _ = input.parse::()?; let wrapper_message = input.parse::()?; AttrItem::MessageWrapperName(wrapper_message) } else if key == "signal_capacity" { - let _ = input.parse::()?; let value = input.parse::()?; AttrItem::SignalChannelCapacity(value) } else if key == "message_capacity" { - let _ = input.parse::()?; let value = input.parse::()?; AttrItem::MessageChannelCapacity(value) } else { - return Err(Error::new(Span::call_site(), "Expected one of `gen`, `signal_capacity`, or `message_capacity`.")) + return Err(Error::new(span, "Expected one of `gen`, `signal_capacity`, or `message_capacity`.")) }) } } @@ -147,6 +151,12 @@ impl Parse for AttrArgs { } else { 1024 }; + let extern_signal_ty = unique.get("signal") + .map(|x| if let AttrItem::ExternOverseerSignalType(x) = x { x.clone() } else { unreachable!() } ) + .ok_or_else(|| { + Error::new(span, "Must declare the overseer signals type via `signal=..`.") + })?; + let extern_event_ty = unique.get("event") .map(|x| if let AttrItem::ExternEventType(x) = x { x.clone() } else { unreachable!() } ) .ok_or_else(|| { @@ -156,7 +166,7 @@ impl Parse for AttrArgs { let message_wrapper = unique.get("gen") .map(|x| if let AttrItem::MessageWrapperName(x) = x { x.clone() } else { unreachable!() } ) .ok_or_else(|| { - Error::new(span, "Must declare the external event type via `event=..`.") + Error::new(span, "Must declare the generated type via `gen=..`.") })?; Ok(AttrArgs { @@ -342,15 +352,14 @@ pub(crate) fn parse_overseer_struct_field( Error::new(span, "Subsystem must consume at least one message") ) } - let no_dispatch = variant.no_dispatch; subsystems.push(SubSysField { name: ident, generic: Ident::new(format!("Sub{}", idx).as_str(), Span::call_site()), ty: try_type_to_ident(ty, span)?, consumes: consumes_idents[0].clone(), - no_dispatch, - blocking: false, // FIXME XXX + no_dispatch: variant.no_dispatch, + blocking: variant.blocking, }); } else { let field_ty = try_type_to_ident(ty, Span::call_site())?; diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index bd392977bfc7..eabc70bd0626 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -138,12 +138,7 @@ impl HeadSupportsParachains for Arc where /// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. /// -/// It wraps a system-wide [`AllMessages`] type that represents all possible -/// messages in the system. -/// -/// [`AllMessages`]: enum.AllMessages.html -/// [`Subsystem`]: trait.Subsystem.html -/// [`Overseer`]: struct.Overseer.html +/// Used to launch jobs. enum ToOverseer { /// A message that wraps something the `Subsystem` is desiring to /// spawn on the overseer and a `oneshot::Sender` to signal the result @@ -263,10 +258,7 @@ pub async fn forward_events>( } -type SubsystemIncomingMessages = stream::Select< - metered::MeteredReceiver>, - metered::UnboundedMeteredReceiver>, ->; +type SubsystemIncomingMessages = SubsystemIncomingMessages; #[derive(Debug, Default, Clone)] struct SignalsReceived(Arc); @@ -522,40 +514,62 @@ struct SubsystemMeterReadouts { } /// The `Overseer` itself. +#[overlord(event=Event, signal=OverseerSignal, gen=AllMessages)] pub struct Overseer { - /// Handles to all subsystems. - subsystems: AllSubsystems< - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - >, - - /// Spawner to spawn tasks to. - s: S, - - /// Here we keep handles to spawned subsystems to be notified when they terminate. - running_subsystems: FuturesUnordered>>, - - /// Gather running subsystems' outbound streams into one. - to_overseer_rx: Fuse>, - - /// Events that are sent to the overseer from the outside world - events_rx: metered::MeteredReceiver, + #[subsystem(no_dispatch, CandidateValidationMessage)] + candidate_validation, + + #[subsystem(no_dispatch, CandidateBackingMessage)] + candidate_backing, + + #[subsystem(no_dispatch, CandidateSelectionMessage)] + candidate_selection, + + #[subsystem(StatementDistributionMessage)] + statement_distribution, + + #[subsystem(no_dispatch, AvailabilityDistributionMessage)] + availability_distribution, + + #[subsystem(no_dispatch, AvailabilityRecoveryMessage)] + availability_recovery, + + #[subsystem(no_dispatch, BitfieldSigningMessage)] + bitfield_signing, + + #[subsystem(BitfieldDistributionMessage)] + bitfield_distribution, + + #[subsystem(no_dispatch, ProvisionerMessage)] + provisioner, + + #[subsystem(no_dispatch, RuntimeApiMessage)] + runtime_api, + + #[subsystem(no_dispatch, blocking, AvailabilityStoreMessage)] + availability_store, + + #[subsystem(no_dispatch, NetworkBridgeMessage)] + network_bridge, + + #[subsystem(no_dispatch, blocking, ChainApiMessage)] + chain_api, + + #[subsystem(no_dispatch, CollationGenerationMessage)] + collation_generation, + + #[subsystem(no_dispatch, CollatorProtocolMessage)] + collator_protocol, + + #[subsystem(ApprovalDistributionMessage)] + approval_distribution, + + #[subsystem(no_dispatch, blocking, ApprovalVotingMessage)] + approval_voting, + + #[subsystem(no_dispatch, GossipSupportMessage)] + gossip_support, + /// External listeners waiting for a hash to be in the active-leave set. activation_external_listeners: HashMap>>>, @@ -1309,42 +1323,6 @@ where Ok((this, handler)) } - // Stop the overseer. - async fn stop(mut self) { - let _ = self.subsystems.candidate_validation.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.candidate_backing.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.candidate_selection.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.statement_distribution.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.availability_distribution.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.availability_recovery.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.bitfield_signing.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.bitfield_distribution.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.provisioner.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.runtime_api.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.availability_store.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.network_bridge.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.chain_api.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.collator_protocol.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.collation_generation.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.approval_distribution.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.approval_voting.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.gossip_support.send_signal(OverseerSignal::Conclude).await; - - let mut stop_delay = Delay::new(Duration::from_secs(STOP_DELAY)).fuse(); - - loop { - select! { - _ = self.running_subsystems.next() => { - if self.running_subsystems.is_empty() { - break; - } - }, - _ = stop_delay => break, - complete => break, - } - } - } - /// Run the `Overseer`. #[tracing::instrument(skip(self), fields(subsystem = LOG_TARGET))] pub async fn run(mut self) -> SubsystemResult<()> { diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index fd4c80cb9392..4dd30c2aef75 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -133,7 +133,7 @@ pub enum OverseerSignal { /// /// It is generic over over the message type `M` that a particular `Subsystem` may use. #[derive(Debug)] -pub enum FromOverseer { +pub enum FromOverseer { /// Signal from the `Overseer`. Signal(OverseerSignal), From aabd88cbfd3e9b1a91ddef61c751c25010448db3 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 11 May 2021 18:05:49 +0200 Subject: [PATCH 025/161] =?UTF-8?q?=F0=9F=8D=8C-split?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- node/overseer/src/lib.rs | 593 +---------------------------------- node/overseer/src/metrics.rs | 192 ++++++++++++ 2 files changed, 206 insertions(+), 579 deletions(-) create mode 100644 node/overseer/src/metrics.rs diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index eabc70bd0626..040ee2be7250 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -96,10 +96,10 @@ use polkadot_node_subsystem_util::{TimeoutExt, metrics::{self, prometheus}, mete use polkadot_node_primitives::SpawnNamed; use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; -// A capacity of bounded channels inside the overseer. -const CHANNEL_CAPACITY: usize = 1024; -// The capacity of signal channels to subsystems. -const SIGNAL_CHANNEL_CAPACITY: usize = 64; + +mod metrics; + +use crate::metrics::*; // A graceful `Overseer` teardown time delay. const STOP_DELAY: u64 = 1; @@ -592,177 +592,6 @@ pub struct Overseer { metrics: Metrics, } -/// Overseer Prometheus metrics. -#[derive(Clone)] -struct MetricsInner { - activated_heads_total: prometheus::Counter, - deactivated_heads_total: prometheus::Counter, - messages_relayed_total: prometheus::Counter, - to_subsystem_bounded_sent: prometheus::GaugeVec, - to_subsystem_bounded_received: prometheus::GaugeVec, - to_subsystem_unbounded_sent: prometheus::GaugeVec, - to_subsystem_unbounded_received: prometheus::GaugeVec, - signals_sent: prometheus::GaugeVec, - signals_received: prometheus::GaugeVec, -} - -#[derive(Default, Clone)] -struct Metrics(Option); - -impl Metrics { - fn on_head_activated(&self) { - if let Some(metrics) = &self.0 { - metrics.activated_heads_total.inc(); - } - } - - fn on_head_deactivated(&self) { - if let Some(metrics) = &self.0 { - metrics.deactivated_heads_total.inc(); - } - } - - fn on_message_relayed(&self) { - if let Some(metrics) = &self.0 { - metrics.messages_relayed_total.inc(); - } - } - - fn channel_fill_level_snapshot( - &self, - to_subsystem: AllSubsystemsSame<(&'static str, SubsystemMeterReadouts)>, - ) { - self.0.as_ref().map(|metrics| { - to_subsystem.map_subsystems( - |(name, readouts): (_, SubsystemMeterReadouts)| { - metrics.to_subsystem_bounded_sent.with_label_values(&[name]) - .set(readouts.bounded.sent as u64); - - metrics.to_subsystem_bounded_received.with_label_values(&[name]) - .set(readouts.bounded.received as u64); - - metrics.to_subsystem_unbounded_sent.with_label_values(&[name]) - .set(readouts.unbounded.sent as u64); - - metrics.to_subsystem_unbounded_received.with_label_values(&[name]) - .set(readouts.unbounded.received as u64); - - metrics.signals_sent.with_label_values(&[name]) - .set(readouts.signals.sent as u64); - - metrics.signals_received.with_label_values(&[name]) - .set(readouts.signals.received as u64); - }); - }); - } -} - -impl metrics::Metrics for Metrics { - fn try_register(registry: &prometheus::Registry) -> Result { - let metrics = MetricsInner { - activated_heads_total: prometheus::register( - prometheus::Counter::new( - "parachain_activated_heads_total", - "Number of activated heads." - )?, - registry, - )?, - deactivated_heads_total: prometheus::register( - prometheus::Counter::new( - "parachain_deactivated_heads_total", - "Number of deactivated heads." - )?, - registry, - )?, - messages_relayed_total: prometheus::register( - prometheus::Counter::new( - "parachain_messages_relayed_total", - "Number of messages relayed by Overseer." - )?, - registry, - )?, - to_subsystem_bounded_sent: prometheus::register( - prometheus::GaugeVec::::new( - prometheus::Opts::new( - "parachain_subsystem_bounded_sent", - "Number of elements sent to subsystems' bounded queues", - ), - &[ - "subsystem_name", - ], - )?, - registry, - )?, - to_subsystem_bounded_received: prometheus::register( - prometheus::GaugeVec::::new( - prometheus::Opts::new( - "parachain_subsystem_bounded_received", - "Number of elements received by subsystems' bounded queues", - ), - &[ - "subsystem_name", - ], - )?, - registry, - )?, - to_subsystem_unbounded_sent: prometheus::register( - prometheus::GaugeVec::::new( - prometheus::Opts::new( - "parachain_subsystem_unbounded_sent", - "Number of elements sent to subsystems' unbounded queues", - ), - &[ - "subsystem_name", - ], - )?, - registry, - )?, - to_subsystem_unbounded_received: prometheus::register( - prometheus::GaugeVec::::new( - prometheus::Opts::new( - "parachain_subsystem_unbounded_received", - "Number of elements received by subsystems' unbounded queues", - ), - &[ - "subsystem_name", - ], - )?, - registry, - )?, - signals_sent: prometheus::register( - prometheus::GaugeVec::::new( - prometheus::Opts::new( - "parachain_overseer_signals_sent", - "Number of signals sent by overseer to subsystems", - ), - &[ - "subsystem_name", - ], - )?, - registry, - )?, - signals_received: prometheus::register( - prometheus::GaugeVec::::new( - prometheus::Opts::new( - "parachain_overseer_signals_received", - "Number of signals received by subsystems from overseer", - ), - &[ - "subsystem_name", - ], - )?, - registry, - )?, - }; - Ok(Metrics(Some(metrics))) - } -} - -impl fmt::Debug for Metrics { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("Metrics {{...}}") - } -} impl Overseer where @@ -782,8 +611,7 @@ where /// . +-----------+ +-----------+ +----------+ +---------+ . /// ............................................................... /// | - /// probably `spawn` - /// a `job` + /// `spawn` a `job` /// | /// V /// +-----------+ @@ -881,395 +709,15 @@ where ApV: Subsystem> + Send, GS: Subsystem> + Send, { - let (events_tx, events_rx) = metered::channel(CHANNEL_CAPACITY); - - let handler = OverseerHandler { - events_tx: events_tx.clone(), - }; - - let metrics = ::register(prometheus_registry)?; - - let (to_overseer_tx, to_overseer_rx) = metered::unbounded(); - - let mut running_subsystems = FuturesUnordered::new(); - - let (candidate_validation_bounded_tx, candidate_validation_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (candidate_backing_bounded_tx, candidate_backing_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (candidate_selection_bounded_tx, candidate_selection_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (statement_distribution_bounded_tx, statement_distribution_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (availability_distribution_bounded_tx, availability_distribution_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (availability_recovery_bounded_tx, availability_recovery_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (bitfield_signing_bounded_tx, bitfield_signing_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (bitfield_distribution_bounded_tx, bitfield_distribution_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (provisioner_bounded_tx, provisioner_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (runtime_api_bounded_tx, runtime_api_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (availability_store_bounded_tx, availability_store_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (network_bridge_bounded_tx, network_bridge_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (chain_api_bounded_tx, chain_api_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (collator_protocol_bounded_tx, collator_protocol_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (collation_generation_bounded_tx, collation_generation_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (approval_distribution_bounded_tx, approval_distribution_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (approval_voting_bounded_tx, approval_voting_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (gossip_support_bounded_tx, gossip_support_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - - let (candidate_validation_unbounded_tx, candidate_validation_unbounded_rx) - = metered::unbounded(); - let (candidate_backing_unbounded_tx, candidate_backing_unbounded_rx) - = metered::unbounded(); - let (candidate_selection_unbounded_tx, candidate_selection_unbounded_rx) - = metered::unbounded(); - let (statement_distribution_unbounded_tx, statement_distribution_unbounded_rx) - = metered::unbounded(); - let (availability_distribution_unbounded_tx, availability_distribution_unbounded_rx) - = metered::unbounded(); - let (availability_recovery_unbounded_tx, availability_recovery_unbounded_rx) - = metered::unbounded(); - let (bitfield_signing_unbounded_tx, bitfield_signing_unbounded_rx) - = metered::unbounded(); - let (bitfield_distribution_unbounded_tx, bitfield_distribution_unbounded_rx) - = metered::unbounded(); - let (provisioner_unbounded_tx, provisioner_unbounded_rx) - = metered::unbounded(); - let (runtime_api_unbounded_tx, runtime_api_unbounded_rx) - = metered::unbounded(); - let (availability_store_unbounded_tx, availability_store_unbounded_rx) - = metered::unbounded(); - let (network_bridge_unbounded_tx, network_bridge_unbounded_rx) - = metered::unbounded(); - let (chain_api_unbounded_tx, chain_api_unbounded_rx) - = metered::unbounded(); - let (collator_protocol_unbounded_tx, collator_protocol_unbounded_rx) - = metered::unbounded(); - let (collation_generation_unbounded_tx, collation_generation_unbounded_rx) - = metered::unbounded(); - let (approval_distribution_unbounded_tx, approval_distribution_unbounded_rx) - = metered::unbounded(); - let (approval_voting_unbounded_tx, approval_voting_unbounded_rx) - = metered::unbounded(); - let (gossip_support_unbounded_tx, gossip_support_unbounded_rx) - = metered::unbounded(); - - let channels_out = ChannelsOut { - candidate_validation: candidate_validation_bounded_tx.clone(), - candidate_backing: candidate_backing_bounded_tx.clone(), - candidate_selection: candidate_selection_bounded_tx.clone(), - statement_distribution: statement_distribution_bounded_tx.clone(), - availability_distribution: availability_distribution_bounded_tx.clone(), - availability_recovery: availability_recovery_bounded_tx.clone(), - bitfield_signing: bitfield_signing_bounded_tx.clone(), - bitfield_distribution: bitfield_distribution_bounded_tx.clone(), - provisioner: provisioner_bounded_tx.clone(), - runtime_api: runtime_api_bounded_tx.clone(), - availability_store: availability_store_bounded_tx.clone(), - network_bridge: network_bridge_bounded_tx.clone(), - chain_api: chain_api_bounded_tx.clone(), - collator_protocol: collator_protocol_bounded_tx.clone(), - collation_generation: collation_generation_bounded_tx.clone(), - approval_distribution: approval_distribution_bounded_tx.clone(), - approval_voting: approval_voting_bounded_tx.clone(), - gossip_support: gossip_support_bounded_tx.clone(), - - candidate_validation_unbounded: candidate_validation_unbounded_tx.clone(), - candidate_backing_unbounded: candidate_backing_unbounded_tx.clone(), - candidate_selection_unbounded: candidate_selection_unbounded_tx.clone(), - statement_distribution_unbounded: statement_distribution_unbounded_tx.clone(), - availability_distribution_unbounded: availability_distribution_unbounded_tx.clone(), - availability_recovery_unbounded: availability_recovery_unbounded_tx.clone(), - bitfield_signing_unbounded: bitfield_signing_unbounded_tx.clone(), - bitfield_distribution_unbounded: bitfield_distribution_unbounded_tx.clone(), - provisioner_unbounded: provisioner_unbounded_tx.clone(), - runtime_api_unbounded: runtime_api_unbounded_tx.clone(), - availability_store_unbounded: availability_store_unbounded_tx.clone(), - network_bridge_unbounded: network_bridge_unbounded_tx.clone(), - chain_api_unbounded: chain_api_unbounded_tx.clone(), - collator_protocol_unbounded: collator_protocol_unbounded_tx.clone(), - collation_generation_unbounded: collation_generation_unbounded_tx.clone(), - approval_distribution_unbounded: approval_distribution_unbounded_tx.clone(), - approval_voting_unbounded: approval_voting_unbounded_tx.clone(), - gossip_support_unbounded: gossip_support_unbounded_tx.clone(), - }; - - let candidate_validation_subsystem = spawn( - &mut s, - candidate_validation_bounded_tx, - stream::select(candidate_validation_bounded_rx, candidate_validation_unbounded_rx), - candidate_validation_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.candidate_validation, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let candidate_backing_subsystem = spawn( - &mut s, - candidate_backing_bounded_tx, - stream::select(candidate_backing_bounded_rx, candidate_backing_unbounded_rx), - candidate_backing_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.candidate_backing, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let candidate_selection_subsystem = spawn( - &mut s, - candidate_selection_bounded_tx, - stream::select(candidate_selection_bounded_rx, candidate_selection_unbounded_rx), - candidate_selection_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.candidate_selection, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let statement_distribution_subsystem = spawn( - &mut s, - statement_distribution_bounded_tx, - stream::select(statement_distribution_bounded_rx, statement_distribution_unbounded_rx), - candidate_validation_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.statement_distribution, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let availability_distribution_subsystem = spawn( - &mut s, - availability_distribution_bounded_tx, - stream::select(availability_distribution_bounded_rx, availability_distribution_unbounded_rx), - availability_distribution_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.availability_distribution, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let availability_recovery_subsystem = spawn( - &mut s, - availability_recovery_bounded_tx, - stream::select(availability_recovery_bounded_rx, availability_recovery_unbounded_rx), - availability_recovery_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.availability_recovery, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let bitfield_signing_subsystem = spawn( - &mut s, - bitfield_signing_bounded_tx, - stream::select(bitfield_signing_bounded_rx, bitfield_signing_unbounded_rx), - bitfield_signing_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.bitfield_signing, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let bitfield_distribution_subsystem = spawn( - &mut s, - bitfield_distribution_bounded_tx, - stream::select(bitfield_distribution_bounded_rx, bitfield_distribution_unbounded_rx), - bitfield_distribution_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.bitfield_distribution, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let provisioner_subsystem = spawn( - &mut s, - provisioner_bounded_tx, - stream::select(provisioner_bounded_rx, provisioner_unbounded_rx), - provisioner_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.provisioner, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let runtime_api_subsystem = spawn( - &mut s, - runtime_api_bounded_tx, - stream::select(runtime_api_bounded_rx, runtime_api_unbounded_rx), - runtime_api_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.runtime_api, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let availability_store_subsystem = spawn( - &mut s, - availability_store_bounded_tx, - stream::select(availability_store_bounded_rx, availability_store_unbounded_rx), - availability_store_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.availability_store, - &metrics, - &mut running_subsystems, - TaskKind::Blocking, - )?; - - let network_bridge_subsystem = spawn( - &mut s, - network_bridge_bounded_tx, - stream::select(network_bridge_bounded_rx, network_bridge_unbounded_rx), - network_bridge_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.network_bridge, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let chain_api_subsystem = spawn( - &mut s, - chain_api_bounded_tx, - stream::select(chain_api_bounded_rx, chain_api_unbounded_rx), - chain_api_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.chain_api, - &metrics, - &mut running_subsystems, - TaskKind::Blocking, - )?; - - let collation_generation_subsystem = spawn( - &mut s, - collation_generation_bounded_tx, - stream::select(collation_generation_bounded_rx, collation_generation_unbounded_rx), - collation_generation_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.collation_generation, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let collator_protocol_subsystem = spawn( - &mut s, - collator_protocol_bounded_tx, - stream::select(collator_protocol_bounded_rx, collator_protocol_unbounded_rx), - collator_protocol_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.collator_protocol, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let approval_distribution_subsystem = spawn( - &mut s, - approval_distribution_bounded_tx, - stream::select(approval_distribution_bounded_rx, approval_distribution_unbounded_rx), - approval_distribution_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.approval_distribution, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let approval_voting_subsystem = spawn( - &mut s, - approval_voting_bounded_tx, - stream::select(approval_voting_bounded_rx, approval_voting_unbounded_rx), - approval_voting_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.approval_voting, - &metrics, - &mut running_subsystems, - TaskKind::Blocking, - )?; - - let gossip_support_subsystem = spawn( - &mut s, - gossip_support_bounded_tx, - stream::select(gossip_support_bounded_rx, gossip_support_unbounded_rx), - gossip_support_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.gossip_support, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let leaves = leaves - .into_iter() - .map(|BlockInfo { hash, parent_hash: _, number }| (hash, number)) - .collect(); - - let active_leaves = HashMap::new(); - let activation_external_listeners = HashMap::new(); - - let subsystems = AllSubsystems { - candidate_validation: candidate_validation_subsystem, - candidate_backing: candidate_backing_subsystem, - candidate_selection: candidate_selection_subsystem, - statement_distribution: statement_distribution_subsystem, - availability_distribution: availability_distribution_subsystem, - availability_recovery: availability_recovery_subsystem, - bitfield_signing: bitfield_signing_subsystem, - bitfield_distribution: bitfield_distribution_subsystem, - provisioner: provisioner_subsystem, - runtime_api: runtime_api_subsystem, - availability_store: availability_store_subsystem, - network_bridge: network_bridge_subsystem, - chain_api: chain_api_subsystem, - collation_generation: collation_generation_subsystem, - collator_protocol: collator_protocol_subsystem, - approval_distribution: approval_distribution_subsystem, - approval_voting: approval_voting_subsystem, - gossip_support: gossip_support_subsystem, - }; + let (overseer, handler) = Overseer::builder() + .metrics(::register(prometheus_registry)?) + .active_leaves(leaves + .into_iter() + .map(|BlockInfo { hash, parent_hash: _, number }| (hash, number)) + .collect()) + .supports_parachains(supports_parachains), + .build(s); { struct ExtractNameAndMeters; impl<'a, T: 'a> MapSubsystem<&'a OverseenSubsystem> for ExtractNameAndMeters { @@ -1287,7 +735,7 @@ where } } - let subsystem_meters = subsystems.as_ref().map_subsystems(ExtractNameAndMeters); + let subsystem_meters = overseer.subsystems().map_subsystems(ExtractNameAndMeters); let metronome_metrics = metrics.clone(); let metronome = Metronome::new(std::time::Duration::from_millis(950)) .for_each(move |_| { @@ -1306,19 +754,6 @@ where s.spawn("metrics_metronome", Box::pin(metronome)); } - let this = Self { - subsystems, - s, - running_subsystems, - to_overseer_rx: to_overseer_rx.fuse(), - events_rx, - activation_external_listeners, - leaves, - active_leaves, - metrics, - span_per_active_leaf: Default::default(), - supports_parachains, - }; Ok((this, handler)) } diff --git a/node/overseer/src/metrics.rs b/node/overseer/src/metrics.rs new file mode 100644 index 000000000000..96fccd789aaf --- /dev/null +++ b/node/overseer/src/metrics.rs @@ -0,0 +1,192 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Prometheus metrics related to the overseer and its channels. + +use super::*; +use polkadot_node_subsystem_util::{TimeoutExt, metrics::{self, prometheus}, metered, Metronome}; + +/// Overseer Prometheus metrics. +#[derive(Clone)] +struct MetricsInner { + activated_heads_total: prometheus::Counter, + deactivated_heads_total: prometheus::Counter, + messages_relayed_total: prometheus::Counter, + to_subsystem_bounded_sent: prometheus::GaugeVec, + to_subsystem_bounded_received: prometheus::GaugeVec, + to_subsystem_unbounded_sent: prometheus::GaugeVec, + to_subsystem_unbounded_received: prometheus::GaugeVec, + signals_sent: prometheus::GaugeVec, + signals_received: prometheus::GaugeVec, +} + +#[derive(Default, Clone)] +pub(crate) struct Metrics(Option); + +impl Metrics { + pub(crate) fn on_head_activated(&self) { + if let Some(metrics) = &self.0 { + metrics.activated_heads_total.inc(); + } + } + + pub(crate) fn on_head_deactivated(&self) { + if let Some(metrics) = &self.0 { + metrics.deactivated_heads_total.inc(); + } + } + + pub(crate) fn on_message_relayed(&self) { + if let Some(metrics) = &self.0 { + metrics.messages_relayed_total.inc(); + } + } + + pub(crate) fn channel_fill_level_snapshot( + &self, + to_subsystem: AllSubsystemsSame<(&'static str, SubsystemMeterReadouts)>, + ) { + self.0.as_ref().map(|metrics| { + to_subsystem.map_subsystems( + |(name, readouts): (_, SubsystemMeterReadouts)| { + metrics.to_subsystem_bounded_sent.with_label_values(&[name]) + .set(readouts.bounded.sent as u64); + + metrics.to_subsystem_bounded_received.with_label_values(&[name]) + .set(readouts.bounded.received as u64); + + metrics.to_subsystem_unbounded_sent.with_label_values(&[name]) + .set(readouts.unbounded.sent as u64); + + metrics.to_subsystem_unbounded_received.with_label_values(&[name]) + .set(readouts.unbounded.received as u64); + + metrics.signals_sent.with_label_values(&[name]) + .set(readouts.signals.sent as u64); + + metrics.signals_received.with_label_values(&[name]) + .set(readouts.signals.received as u64); + }); + }); + } +} + +impl metrics::Metrics for Metrics { + fn try_register(registry: &prometheus::Registry) -> Result { + let metrics = MetricsInner { + activated_heads_total: prometheus::register( + prometheus::Counter::new( + "parachain_activated_heads_total", + "Number of activated heads." + )?, + registry, + )?, + deactivated_heads_total: prometheus::register( + prometheus::Counter::new( + "parachain_deactivated_heads_total", + "Number of deactivated heads." + )?, + registry, + )?, + messages_relayed_total: prometheus::register( + prometheus::Counter::new( + "parachain_messages_relayed_total", + "Number of messages relayed by Overseer." + )?, + registry, + )?, + to_subsystem_bounded_sent: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "parachain_subsystem_bounded_sent", + "Number of elements sent to subsystems' bounded queues", + ), + &[ + "subsystem_name", + ], + )?, + registry, + )?, + to_subsystem_bounded_received: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "parachain_subsystem_bounded_received", + "Number of elements received by subsystems' bounded queues", + ), + &[ + "subsystem_name", + ], + )?, + registry, + )?, + to_subsystem_unbounded_sent: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "parachain_subsystem_unbounded_sent", + "Number of elements sent to subsystems' unbounded queues", + ), + &[ + "subsystem_name", + ], + )?, + registry, + )?, + to_subsystem_unbounded_received: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "parachain_subsystem_unbounded_received", + "Number of elements received by subsystems' unbounded queues", + ), + &[ + "subsystem_name", + ], + )?, + registry, + )?, + signals_sent: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "parachain_overseer_signals_sent", + "Number of signals sent by overseer to subsystems", + ), + &[ + "subsystem_name", + ], + )?, + registry, + )?, + signals_received: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "parachain_overseer_signals_received", + "Number of signals received by subsystems from overseer", + ), + &[ + "subsystem_name", + ], + )?, + registry, + )?, + }; + Ok(Metrics(Some(metrics))) + } +} + +impl fmt::Debug for Metrics { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Metrics {{...}}") + } +} From 7644dc8f286ea6bb17d650bdda02629c9281b25c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 12 May 2021 11:25:02 +0200 Subject: [PATCH 026/161] more fixins --- .../proc-macro/src/impl_dispatch.rs | 17 ++-- .../proc-macro/src/impl_overseer.rs | 68 +++++++++------- .../proc-macro/src/impl_replace.rs | 4 +- .../overseer-gen/proc-macro/src/lib.rs | 3 +- .../overseer-gen/proc-macro/src/parse.rs | 9 ++- .../overseer-gen/proc-macro/src/tests.rs | 4 +- node/overseer/overseer-gen/src/lib.rs | 32 +++++++- .../overseer-gen/tests/ui/ok-01-boring.rs | 9 ++- .../tests/ui/ok-03-no-dispatch.rs | 13 +-- node/overseer/src/handler.rs | 4 +- node/overseer/src/lib.rs | 81 +++++-------------- 11 files changed, 131 insertions(+), 113 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index 4a71cdd6c7b3..33ba9aa4132a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -26,17 +26,24 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { let dispatchable = info.subsystems().into_iter().filter(|ssf| !ssf.no_dispatch).map(|ssf| ssf.consumes.clone()).collect::>(); let msg = "Generated by #[subsystem_dispatch_gen] proc-macro."; - let incoming_event_ty= &info.incoming_event_ty.clone(); + let extern_event_ty= &info.extern_event_ty.clone(); + let ts = quote! { impl #message_wrapper { #[doc = #msg] - pub fn dispatch_iter(event: #incoming_event_ty) -> impl Iterator + Send { + pub fn dispatch_iter(event: #extern_event_ty) -> impl Iterator + Send { let mut iter = None.into_iter(); #( - let mut iter = iter.chain(std::iter::once(event.focus().ok().map(|event| { - #message_wrapper :: #dispatchable ( <#dispatchable as ::std::from::From < #incoming_event_ty > > :: from( event ) ) - }))); + let mut iter = iter.chain( + std::iter::once( + event.focus().ok().map(|event| { + #message_wrapper :: #dispatchable ( + #dispatchable :: from( event ) + ) + }) + ) + ); )* iter.filter_map(|x: Option<_>| x) } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 2843a4cc2ca0..32768f44e4c4 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -3,15 +3,14 @@ use syn::{Ident, Result}; use super::*; - pub(crate) fn impl_overseer_struct( info: &OverseerInfo, ) -> Result { let message_wrapper = &info.message_wrapper.clone(); let overseer_name = info.overseer_name.clone(); - let field_name = &info.subsystem_names(); + let subsystem_name = &info.subsystem_names(); - let field_ty = &info.subsystem_generic_types(); + let subsystem_generic_ty = &info.subsystem_generic_types(); let baggage_name = &info.baggage_names(); let baggage_ty = &info.baggage_types(); @@ -19,16 +18,18 @@ pub(crate) fn impl_overseer_struct( let baggage_generic_ty = &info.baggage_generic_types(); let generics = quote! { - < Ctx, S, #( #baggage_generic_ty, )* #( #field_ty, )* > + < Ctx, S, #( #baggage_generic_ty, )* > }; let where_clause = quote! { where Ctx: SubsystemContext, S: SpawnNamed, - #( #field_ty : Subsystem, )* + #( #subsystem_generic_ty : Subsystem, )* }; + let consumes = &info.consumes(); + let message_channel_capacity = info.message_channel_capacity; let signal_channel_capacity = info.signal_channel_capacity; @@ -37,10 +38,12 @@ pub(crate) fn impl_overseer_struct( const SIGNAL_CHANNEL_CAPACITY: usize = #signal_channel_capacity; pub struct #overseer_name #generics { + // Subsystem instances. #( - #field_name: #field_ty, + #subsystem_name: OverseenSubsystem< SubsystemInstance < #consumes > >, )* + // Non-subsystem members. #( #baggage_name: #baggage_ty, )* @@ -52,7 +55,7 @@ pub(crate) fn impl_overseer_struct( running_subsystems: FuturesUnordered>>, /// Gather running subsystems' outbound streams into one. - to_overseer_rx: Fuse>, + to_overseer_rx: Fuse>, /// Events that are sent to the overseer from the outside world. events_rx: metered::MeteredReceiver, @@ -61,7 +64,7 @@ pub(crate) fn impl_overseer_struct( impl #generics #overseer_name #generics #where_clause { pub async fn stop(mut self) { #( - let _ = self. #field_name .send_signal(OverseerSignal::Conclude).await; + let _ = self. #subsystem_name .send_signal(OverseerSignal::Conclude).await; )* loop { select! { @@ -79,7 +82,7 @@ pub(crate) fn impl_overseer_struct( pub async fn broadcast_signal(&mut self, signal: OverseerSignal) -> SubsystemResult<()> { #( - self. #field_name .send_signal(signal.clone()).await; + self. #subsystem_name .send_signal(signal.clone()).await; )* let _ = signal; @@ -89,7 +92,7 @@ pub(crate) fn impl_overseer_struct( pub async fn route_message(&mut self, msg: #message_wrapper) -> SubsystemResult<()> { match msg { #( - #field_ty (msg) => self. #field_name .send_message(msg).await?, + #message_wrapper :: #consumes (msg) => self. #subsystem_name .send_message(msg).await?, )* } Ok(()) @@ -108,8 +111,8 @@ pub(crate) fn impl_builder( let builder = Ident::new(&(overseer_name.to_string() + "Builder"), overseer_name.span()); let handler = Ident::new(&(overseer_name.to_string() + "Handler"), overseer_name.span()); - let field_name = &info.subsystem_names(); - let field_ty = &info.subsystem_generic_types(); + let subsystem_name = &info.subsystem_names(); + let subsystem_generic_ty = &info.subsystem_generic_types(); let channel_name = &info.channel_names(""); let channel_name_unbounded = &info.channel_names("_unbounded"); @@ -126,46 +129,55 @@ pub(crate) fn impl_builder( let blocking = &info.subsystems().iter().map(|x| x.blocking).collect::>(); let generics = quote! { - < Ctx, S, #( #baggage_generic_ty, )* #( #field_ty, )* > + < Ctx, S, #( #baggage_generic_ty, )* > + }; + + let builder_generics = quote! { + < Ctx, S, #( #baggage_generic_ty, )* #( #subsystem_generic_ty, )* > + }; + + let builder_additional_generics = quote! { + < #( #subsystem_generic_ty, )* > }; let where_clause = quote! { where Ctx: SubsystemContext, S: SpawnNamed, - #( #field_ty : Subsystem, )* + #( #subsystem_generic_ty : Subsystem, )* }; + let consumes = &info.consumes(); let message_wrapper = &info.message_wrapper; let ts = quote! { impl #generics #overseer_name #generics #where_clause { - fn builder() -> #builder { + fn builder::< #builder_additional_generics >() -> #builder #builder_generics { #builder :: default() } } #[derive(Debug, Clone, Default)] - struct #builder #generics { + struct #builder #builder_generics { #( - #field_name : ::std::option::Option< #field_ty >, + #subsystem_name : ::std::option::Option< #subsystem_generic_ty >, )* #( #baggage_name : ::std::option::Option< #baggage_name >, )* - spawner: ::std::option::Option< >, + spawner: ::std::option::Option< S >, } - impl #generics #builder #generics #where_clause { + impl #builder_generics #builder #builder_generics #where_clause { #( - fn #field_name (mut self, new: #field_ty ) -> #builder { - self.#field_name = Some( new ); + fn #subsystem_name (mut self, subsystem: #subsystem_generic_ty ) -> #builder { + self.#subsystem_name = Some( new ); self } )* - fn build(mut self, create_ctx: F) -> (#overseer_name #generics, #handler) + fn build(mut self, create_subsystem_ctx: F) -> (#overseer_name #generics, #handler) where F: FnMut( metered::MeteredReceiver, meteted::SubsystemIncomingMessages< #message_wrapper >, @@ -185,11 +197,11 @@ pub(crate) fn impl_builder( let channels_out = { #( - let (#channel_name_tx, #channel_name_rx) = ::metered::channel::>(CHANNEL_CAPACITY); + let (#channel_name_tx, #channel_name_rx) = ::metered::channel::>(CHANNEL_CAPACITY); )* #( - let (#channel_name_unbounded_tx, #channel_name_unbounded_rx) = ::metered::unbounded::>(); + let (#channel_name_unbounded_tx, #channel_name_unbounded_rx) = ::metered::unbounded::>(); )* ChannelsOut { @@ -207,7 +219,7 @@ pub(crate) fn impl_builder( let mut running_subsystems = FuturesUnordered::>>::new(); #( - let #field_name: OverseenSubsystem< #message_wrapper > = { + let #subsystem_name: OverseenSubsystem< #message_wrapper > = { let unbounded_meter = #channel_name_unbounded_tx .meter().clone(); @@ -220,14 +232,14 @@ pub(crate) fn impl_builder( let (signal_tx, signal_rx) = metered::channel(SIGNAL_CHANNEL_CAPACITY); - let ctx = create_ctx( + let ctx = create_subsystem_ctx( signal_rx, message_rx, channels_out.clone(), to_overseer_tx.clone(), ); - let SpawnedSubsystem { future, name } = #field_name .start(ctx); + let SpawnedSubsystem { future, name } = #subsystem_name .start(ctx); let (terminated_tx, terminated_rx) = oneshot::channel(); @@ -268,7 +280,7 @@ pub(crate) fn impl_builder( let overseer = #overseer_name :: #generics { #( - #field_name, + #subsystem_name, )* #( diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs index 139a82f586db..fc43792418d3 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs @@ -52,8 +52,8 @@ pub(crate) fn impl_replacable_subsystem( impl #generics #overseer_name #generics #where_clause { #[doc = #msg] pub fn #fn_name < #new > (self, replacement: #new) -> #overseer_name #modified_generics - where - #new: Subsystem, + where + #new: Subsystem, { #overseer_name :: #modified_generics { #replacable_item: replacement, diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 7f665581e7ca..8ccdebec6c09 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -81,7 +81,8 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< message_wrapper, message_channel_capacity: args.message_channel_capacity, signal_channel_capacity: args.signal_channel_capacity, - incoming_event_ty: Ident::new("Network", Span::call_site()), // FIXME + extern_event_ty: args.extern_event_ty, + extern_signal_ty: args.extern_signal_ty, }; let mut additive = impl_overseer_struct(&info)?; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse.rs b/node/overseer/overseer-gen/proc-macro/src/parse.rs index 9c17684bb3cc..bb7f6fa7b6c6 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse.rs @@ -111,6 +111,7 @@ impl Parse for AttrItem { pub(crate) struct AttrArgs { pub(crate) message_wrapper: Ident, pub(crate) extern_event_ty: Path, + pub(crate) extern_signal_ty: Path, pub(crate) signal_channel_capacity: u64, pub(crate) message_channel_capacity: u64, } @@ -173,6 +174,7 @@ impl Parse for AttrArgs { signal_channel_capacity, message_channel_capacity, extern_event_ty, + extern_signal_ty, message_wrapper, }) } @@ -258,8 +260,11 @@ pub(crate) struct OverseerInfo { /// Size of the bounded signal channel. pub(crate) signal_channel_capacity: u64, - /// Incoming event type, commonly from the network bridge. - pub(crate) incoming_event_ty: Ident, + /// Signals to be sent, sparse information that is used intermittendly. + pub(crate) extern_signal_ty: Path, + + /// Incoming event type from the outer world, commonly from the network. + pub(crate) extern_event_ty: Path, } impl OverseerInfo { diff --git a/node/overseer/overseer-gen/proc-macro/src/tests.rs b/node/overseer/overseer-gen/proc-macro/src/tests.rs index 18d80f07a2ca..dd5695601c76 100644 --- a/node/overseer/overseer-gen/proc-macro/src/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/tests.rs @@ -6,7 +6,7 @@ use syn::parse_quote; #[test] fn basic() { let attr = quote! { - (event=OverseerSignal,gen=AllMessages) + (event=OverseerSignal,signal=SigSigSig, gen=AllMessages) }; let item = quote! { @@ -36,7 +36,7 @@ fn basic() { #[test] fn attr_parsing_works() { let attr: AttrArgs = parse_quote! { - (gen=AllMessage, event=::some::why::ExternEvent, signal_capacity=111, message_capacity=222,) + (gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222,) }; assert_matches!(attr, AttrArgs { message_channel_capacity, diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 744881bf0df6..6085ed2bd595 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -68,7 +68,9 @@ use std::sync::Arc; /// A running instance of some [`Subsystem`]. /// /// [`Subsystem`]: trait.Subsystem.html -struct SubsystemInstance { +/// +/// `M` here is the inner message type, and not the generated `enum AllMessages`. +pub struct SubsystemInstance { tx_signal: metered::MeteredSender, tx_bounded: metered::MeteredSender>, meters: SubsystemMeters, @@ -77,6 +79,29 @@ struct SubsystemInstance { } + +/// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. +/// +/// Used to launch jobs. +pub enum ToOverseer { + /// A message that wraps something the `Subsystem` is desiring to + /// spawn on the overseer and a `oneshot::Sender` to signal the result + /// of the spawn. + SpawnJob { + name: &'static str, + s: BoxFuture<'static, ()>, + }, + + /// Same as `SpawnJob` but for blocking tasks to be executed on a + /// dedicated thread pool. + SpawnBlockingJob { + name: &'static str, + s: BoxFuture<'static, ()>, + }, +} + + + /// A helper trait to map a subsystem to smth. else. pub(crate) trait MapSubsystem { type Output; @@ -98,8 +123,6 @@ pub type SubsystemIncomingMessages = ::futures::stream::Select< ::metered::UnboundedMeteredReceiver>, >; - - #[derive(Debug, Default, Clone)] struct SignalsReceived(Arc); @@ -138,5 +161,8 @@ struct SubsystemMeterReadouts { } + + + #[cfg(test)] mod tests; diff --git a/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs b/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs index da6b2d131193..9299c54b5897 100644 --- a/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs @@ -7,8 +7,13 @@ struct X; #[derive(Default, Clone, Copy)] struct AwesomeSubSys; -#[overlord(Wrapper)] -#[derive(Clone, AllSubsystemsGen)] +#[derive(Default, Clone, Copy)] +struct SigSigSig; + +#[derive(Default, Clone, Copy)] +struct EventX; + +#[overlord(signal=self::SigSigSig, event=self::EventX, gen=AllMessages)] struct Overseer { #[subsystem(X)] sub0: AwesomeSubSys, diff --git a/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs b/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs index a92562733b30..dc630a6fb4b6 100644 --- a/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs @@ -8,8 +8,8 @@ struct MsgStrukt(u8); #[derive(Default, Clone, Copy)] struct AwesomeSubSys; -#[overlord(Wrapper)] -struct Overseer { +#[overlord(signal=, gen=AllMessages)] +struct Overseer { #[subsystem(no_dispatch, MsgStrukt)] sub0: AwesomeSubSys, @@ -19,11 +19,12 @@ struct Overseer { #[derive(Debug, Clone, Copy)] struct DummySpawner; +struct Ctx; + fn main() { - let overseer = Overseer::::builder() + let overseer = Overseer::builder() .sub0(AwesomeSubSys::default()) + .something_else(7777u32) .spawner(DummySpawner) - .build(Ctx); - - let overseer = overseer.replace_sub0(TequilaInABar::default()); + .build(|| -> Ctx { Ctx } ); } diff --git a/node/overseer/src/handler.rs b/node/overseer/src/handler.rs index 275035624325..1b56eca6be25 100644 --- a/node/overseer/src/handler.rs +++ b/node/overseer/src/handler.rs @@ -2,7 +2,7 @@ /// /// [`Overseer`]: struct.Overseer.html #[derive(Clone)] -pub struct OverseerHandler { +pub struct OverseerHandler { events_tx: metered::MeteredSender, } @@ -15,7 +15,7 @@ impl OverseerHandler { /// Send some message to one of the `Subsystem`s. #[tracing::instrument(level = "trace", skip(self, msg), fields(subsystem = LOG_TARGET))] - pub async fn send_msg(&mut self, msg: impl Into) { + pub async fn send_msg(&mut self, msg: impl Into) { self.send_and_log_error(Event::MsgToSubsystem(msg.into())).await } diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 040ee2be7250..54a99d2d300f 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -92,7 +92,7 @@ pub use polkadot_subsystem::{ Subsystem, SubsystemContext, SubsystemSender, OverseerSignal, FromOverseer, SubsystemError, SubsystemResult, SpawnedSubsystem, ActiveLeavesUpdate, ActivatedLeaf, DummySubsystem, jaeger, }; -use polkadot_node_subsystem_util::{TimeoutExt, metrics::{self, prometheus}, metered, Metronome}; +use polkadot_node_subsystem_util::{TimeoutExt, metered, Metronome}; use polkadot_node_primitives::SpawnNamed; use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; @@ -136,26 +136,6 @@ impl HeadSupportsParachains for Arc where } } -/// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. -/// -/// Used to launch jobs. -enum ToOverseer { - /// A message that wraps something the `Subsystem` is desiring to - /// spawn on the overseer and a `oneshot::Sender` to signal the result - /// of the spawn. - SpawnJob { - name: &'static str, - s: BoxFuture<'static, ()>, - }, - - /// Same as `SpawnJob` but for blocking tasks to be executed on a - /// dedicated thread pool. - SpawnBlockingJob { - name: &'static str, - s: BoxFuture<'static, ()>, - }, -} - impl Debug for ToOverseer { @@ -517,58 +497,58 @@ struct SubsystemMeterReadouts { #[overlord(event=Event, signal=OverseerSignal, gen=AllMessages)] pub struct Overseer { #[subsystem(no_dispatch, CandidateValidationMessage)] - candidate_validation, + candidate_validation: CandidateValidation, #[subsystem(no_dispatch, CandidateBackingMessage)] - candidate_backing, + candidate_backing: CandidateBacking, #[subsystem(no_dispatch, CandidateSelectionMessage)] - candidate_selection, + candidate_selection: CandidateSelection, #[subsystem(StatementDistributionMessage)] - statement_distribution, + statement_distribution: StatementDistribution, #[subsystem(no_dispatch, AvailabilityDistributionMessage)] - availability_distribution, + availability_distribution: AvailabilityDistribution, #[subsystem(no_dispatch, AvailabilityRecoveryMessage)] - availability_recovery, + availability_recovery: AvailabilityRecovery, #[subsystem(no_dispatch, BitfieldSigningMessage)] - bitfield_signing, + bitfield_signing: BitfieldSigning, #[subsystem(BitfieldDistributionMessage)] - bitfield_distribution, + bitfield_distribution: BitfieldDistribution, #[subsystem(no_dispatch, ProvisionerMessage)] - provisioner, + provisioner: Provisioner, #[subsystem(no_dispatch, RuntimeApiMessage)] - runtime_api, + runtime_api: RuntimeApi, #[subsystem(no_dispatch, blocking, AvailabilityStoreMessage)] - availability_store, + availability_store: AvailabilityStore, #[subsystem(no_dispatch, NetworkBridgeMessage)] - network_bridge, + network_bridge: NetworkBridge, #[subsystem(no_dispatch, blocking, ChainApiMessage)] - chain_api, + chain_api: ChainApi, #[subsystem(no_dispatch, CollationGenerationMessage)] - collation_generation, + collation_generation: CollationGeneration, #[subsystem(no_dispatch, CollatorProtocolMessage)] - collator_protocol, + collator_protocol: CollatorProtocol, #[subsystem(ApprovalDistributionMessage)] - approval_distribution, + approval_distribution: ApprovalDistribution, #[subsystem(no_dispatch, blocking, ApprovalVotingMessage)] - approval_voting, + approval_voting: ApprovalVoting, #[subsystem(no_dispatch, GossipSupportMessage)] - gossip_support, + gossip_support: GossipSupport, /// External listeners waiting for a hash to be in the active-leave set. @@ -593,6 +573,7 @@ pub struct Overseer { } +#[overlord] impl Overseer where S: SpawnNamed, @@ -682,32 +663,12 @@ where /// # /// # }); } /// ``` - pub fn new( + pub fn new( leaves: impl IntoIterator, - all_subsystems: AllSubsystems, prometheus_registry: Option<&prometheus::Registry>, supports_parachains: SupportsParachains, mut s: S, ) -> SubsystemResult<(Self, OverseerHandler)> - where - CV: Subsystem> + Send, - CB: Subsystem> + Send, - CS: Subsystem> + Send, - SD: Subsystem> + Send, - AD: Subsystem> + Send, - AR: Subsystem> + Send, - BS: Subsystem> + Send, - BD: Subsystem> + Send, - P: Subsystem> + Send, - RA: Subsystem> + Send, - AS: Subsystem> + Send, - NB: Subsystem> + Send, - CA: Subsystem> + Send, - CG: Subsystem> + Send, - CP: Subsystem> + Send, - ApD: Subsystem> + Send, - ApV: Subsystem> + Send, - GS: Subsystem> + Send, { let (overseer, handler) = Overseer::builder() From b0dc50207ab51e6c2cf00e284990776b170f2f48 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 12 May 2021 13:01:57 +0200 Subject: [PATCH 027/161] =?UTF-8?q?level=20up=20=F0=9F=8D=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 1 + node/overseer/overseer-gen/Cargo.toml | 2 + .../proc-macro/src/impl_channels_out.rs | 13 --- .../proc-macro/src/impl_overseer.rs | 24 ++++++ .../overseer-gen/proc-macro/src/lib.rs | 3 +- node/overseer/overseer-gen/src/lib.rs | 85 ++++++++++++------- .../tests/ui/err-01-replace-w-inadequate.rs | 2 +- .../overseer-gen/tests/ui/err-02-enum.rs | 2 +- .../tests/ui/err-03-subsys-twice.rs | 2 +- .../overseer-gen/tests/ui/ok-01-boring.rs | 2 +- .../overseer-gen/tests/ui/ok-02-w-generic.rs | 2 +- .../tests/ui/ok-03-no-dispatch.rs | 8 +- 12 files changed, 92 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 90f048f049c3..fb014bedec01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6265,6 +6265,7 @@ dependencies = [ "futures 0.3.14", "metered-channel", "polkadot-overseer-gen-proc-macro", + "sp-core", "tracing", "trybuild", ] diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index 10890f13026e..317ef0d1ce84 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -11,6 +11,8 @@ futures = "0.3" overseer-gen-proc-macro = { package = "polkadot-overseer-gen-proc-macro", path = "./proc-macro" } metered = { package = "metered-channel", path = "../../metered-channel" } +# trait SpawnNamed +sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } [dev-dependencies] trybuild = "1.0.41" diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 0321e77b2ba4..e319e649e6b6 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -17,19 +17,6 @@ pub(crate) fn impl_channels_out_struct( let consumes = &info.consumes(); let ts = quote! { - #[derive(Debug)] - pub struct MessagePacket { - signals_received: usize, - message: T, - } - - pub fn make_packet(signals_received: usize, message: T) -> MessagePacket { - MessagePacket { - signals_received, - message, - } - } - pub struct ChannelsOut { #( pub #channel_name: ::metered::MeteredSender>, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 32768f44e4c4..f5745d5ab571 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -100,6 +100,30 @@ pub(crate) fn impl_overseer_struct( }; ts.extend(impl_builder(info)?); + ts.extend(impl_subsystem_instance(info)?); + + Ok(ts) +} + + +pub(crate) fn impl_subsystem_instance(info: &OverseerInfo) -> Result { + let extern_signal_ty = &info.extern_signal_ty; + + let ts = quote::quote! { + /// A running instance of some [`Subsystem`]. + /// + /// [`Subsystem`]: trait.Subsystem.html + /// + /// `M` here is the inner message type, and _not_ the generated `enum AllMessages`. + pub struct SubsystemInstance { + tx_signal: metered::MeteredSender< #extern_signal_ty >, + tx_bounded: metered::MeteredSender>, + meters: SubsystemMeters, + signals_received: usize, + name: &'static str, + } + }; + Ok(ts) } diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 8ccdebec6c09..b5305e7feea2 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -15,8 +15,7 @@ // along with Polkadot. If not, see . use proc_macro2::TokenStream; -use proc_macro2::Span; -use syn::{parse2, Error, GenericParam, Result, Ident}; +use syn::{parse2, Error, GenericParam, Result}; use std::collections::HashSet; mod parse; diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 6085ed2bd595..003460774793 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -62,23 +62,12 @@ pub use overseer_gen_proc_macro::*; pub use tracing; pub use metered; -use std::sync::atomic::{self, AtomicUsize}; -use std::sync::Arc; - -/// A running instance of some [`Subsystem`]. -/// -/// [`Subsystem`]: trait.Subsystem.html -/// -/// `M` here is the inner message type, and not the generated `enum AllMessages`. -pub struct SubsystemInstance { - tx_signal: metered::MeteredSender, - tx_bounded: metered::MeteredSender>, - meters: SubsystemMeters, - signals_received: usize, - name: &'static str, -} +pub use sp_core::traits::SpawnNamed; +pub use futures::future::BoxFuture; +use std::sync::atomic::{self, AtomicUsize}; +use std::sync::Arc; /// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. /// @@ -88,14 +77,18 @@ pub enum ToOverseer { /// spawn on the overseer and a `oneshot::Sender` to signal the result /// of the spawn. SpawnJob { + /// Name of the task to spawn which be shown in jaeger and tracing logs. name: &'static str, + /// The future to execute. s: BoxFuture<'static, ()>, }, /// Same as `SpawnJob` but for blocking tasks to be executed on a /// dedicated thread pool. SpawnBlockingJob { + /// Name of the task to spawn which be shown in jaeger and tracing logs. name: &'static str, + /// The future to execute. s: BoxFuture<'static, ()>, }, } @@ -117,35 +110,62 @@ impl MapSubsystem for F where F: Fn(T) -> U { } } +/// A wrapping type for messages. +// FIXME XXX elaborate the purpose of this. +#[derive(Debug)] +pub struct MessagePacket { + signals_received: usize, + message: T, +} + +/// Create a packet from its parts. +pub fn make_packet(signals_received: usize, message: T) -> MessagePacket { + MessagePacket { + signals_received, + message, + } +} + /// Incoming messages from both the bounded and unbounded channel. pub type SubsystemIncomingMessages = ::futures::stream::Select< ::metered::MeteredReceiver>, ::metered::UnboundedMeteredReceiver>, >; + +/// Meter to count the received signals in total. +// XXX FIXME is there a necessity for this? Seems redundant to `ReadOuts` #[derive(Debug, Default, Clone)] -struct SignalsReceived(Arc); +pub struct SignalsReceived(Arc); impl SignalsReceived { - fn load(&self) -> usize { - self.0.load(atomic::Ordering::SeqCst) + /// Load the current value of received signals. + pub fn load(&self) -> usize { + // off by a few is ok + self.0.load(atomic::Ordering::Relaxed) } - fn inc(&self) { - self.0.fetch_add(1, atomic::Ordering::SeqCst); + /// Increase the number of signals by one. + pub fn inc(&self) { + self.0.fetch_add(1, atomic::Ordering::Acquire); } } +/// Collection of meters related to a subsystem. #[derive(Clone)] -struct SubsystemMeters { - bounded: metered::Meter, - unbounded: metered::Meter, - signals: metered::Meter, +pub struct SubsystemMeters { + #[allow(missing_docs)] + pub bounded: metered::Meter, + #[allow(missing_docs)] + pub unbounded: metered::Meter, + #[allow(missing_docs)] + pub signals: metered::Meter, } impl SubsystemMeters { - fn read(&self) -> SubsystemMeterReadouts { + /// Read the values of all subsystem `Meter`s. + pub fn read(&self) -> SubsystemMeterReadouts { SubsystemMeterReadouts { bounded: self.bounded.read(), unbounded: self.unbounded.read(), @@ -154,13 +174,16 @@ impl SubsystemMeters { } } -struct SubsystemMeterReadouts { - bounded: metered::Readout, - unbounded: metered::Readout, - signals: metered::Readout, -} - +/// Set of readouts of the `Meter`s of a subsystem. +pub struct SubsystemMeterReadouts { + #[allow(missing_docs)] + pub bounded: metered::Readout, + #[allow(missing_docs)] + pub unbounded: metered::Readout, + #[allow(missing_docs)] + pub signals: metered::Readout, +} diff --git a/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs b/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs index e23d5ba16b0b..d51728f41fe6 100644 --- a/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs +++ b/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use polkadot_overseer_gen_proc_macro::overlord; +use polkadot_overseer_gen::overlord; struct X; diff --git a/node/overseer/overseer-gen/tests/ui/err-02-enum.rs b/node/overseer/overseer-gen/tests/ui/err-02-enum.rs index 0db83e644875..04cf2f519a55 100644 --- a/node/overseer/overseer-gen/tests/ui/err-02-enum.rs +++ b/node/overseer/overseer-gen/tests/ui/err-02-enum.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use polkadot_overseer_gen_proc_macro::overlord; +use polkadot_overseer_gen::overlord; struct Msg(u8); diff --git a/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs b/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs index 4b91e667309c..4fbbdb247626 100644 --- a/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs +++ b/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use polkadot_overseer_gen_proc_macro::overlord; +use polkadot_overseer_gen::overlord; struct X; struct Z; diff --git a/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs b/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs index 9299c54b5897..4b73ad27e667 100644 --- a/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use polkadot_overseer_gen_proc_macro::overlord; +use polkadot_overseer_gen::overlord; struct X; diff --git a/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs b/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs index 8731ff46400a..4b3b7b46e725 100644 --- a/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use polkadot_overseer_gen_proc_macro::overlord; +use polkadot_overseer_gen::overlord; trait MetaMeta {} #[derive(Debug)] diff --git a/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs b/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs index dc630a6fb4b6..2e61bf7fbd46 100644 --- a/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use polkadot_overseer_gen_proc_macro::overlord; +use polkadot_overseer_gen::overlord; #[derive(Debug)] struct MsgStrukt(u8); @@ -8,7 +8,9 @@ struct MsgStrukt(u8); #[derive(Default, Clone, Copy)] struct AwesomeSubSys; -#[overlord(signal=, gen=AllMessages)] +#[derive(Default, Clone, Copy)] +struct SigSigSig; +#[overlord(signal=SigSigSig, gen=AllMessages)] struct Overseer { #[subsystem(no_dispatch, MsgStrukt)] sub0: AwesomeSubSys, @@ -22,7 +24,7 @@ struct DummySpawner; struct Ctx; fn main() { - let overseer = Overseer::builder() + let overseer = Overseer::<_>::builder() .sub0(AwesomeSubSys::default()) .something_else(7777u32) .spawner(DummySpawner) From 1170675313a7bd2612f414879b6b5432c7e3d9b6 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 12 May 2021 17:25:03 +0200 Subject: [PATCH 028/161] intermediate --- node/metered-channel/src/lib.rs | 4 +- .../src/requester/fetch_task/mod.rs | 8 +- .../src/requester/fetch_task/tests.rs | 3 +- node/network/availability-recovery/src/lib.rs | 10 +- node/overseer/Cargo.toml | 2 +- .../overseer-gen/tests/ui/err-02-enum.rs | 35 +- node/overseer/src/lib.rs | 2378 ++++++++++++++++- node/overseer/subsystems-gen/Cargo.toml | 18 + node/overseer/subsystems-gen/src/lib.rs | 179 ++ .../subsystems-gen/tests/ui/err-01-enum.rs | 13 + .../tests/ui/err-01-enum.stderr | 5 + .../tests/ui/err-01-generic-used-twice.rs | 17 + .../tests/ui/err-01-generic-used-twice.stderr | 14 + .../tests/ui/err-01-no-generic.rs | 17 + .../tests/ui/err-01-no-generics.stderr | 14 + .../tests/ui/ok-01-w-generics.rs | 17 + node/service/src/lib.rs | 9 +- node/subsystem/Cargo.toml | 1 + node/subsystem/dispatch-gen/Cargo.toml | 18 + node/subsystem/dispatch-gen/src/lib.rs | 208 ++ .../tests/ui/err-01-missing-skip.rs | 37 + .../tests/ui/err-01-missing-skip.stderr | 14 + .../tests/ui/err-02-missing-from.rs | 41 + .../tests/ui/err-02-missing-from.stderr | 10 + .../tests/ui/ok-01-with-intermediate.rs | 48 + node/subsystem/src/errors.rs | 54 - node/subsystem/src/lib.rs | 54 +- node/subsystem/src/messages.rs | 57 + 28 files changed, 3115 insertions(+), 170 deletions(-) create mode 100644 node/overseer/subsystems-gen/Cargo.toml create mode 100644 node/overseer/subsystems-gen/src/lib.rs create mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-enum.rs create mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-enum.stderr create mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.rs create mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.stderr create mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-no-generic.rs create mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-no-generics.stderr create mode 100644 node/overseer/subsystems-gen/tests/ui/ok-01-w-generics.rs create mode 100644 node/subsystem/dispatch-gen/Cargo.toml create mode 100644 node/subsystem/dispatch-gen/src/lib.rs create mode 100644 node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.rs create mode 100644 node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.stderr create mode 100644 node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.rs create mode 100644 node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.stderr create mode 100644 node/subsystem/dispatch-gen/tests/ui/ok-01-with-intermediate.rs diff --git a/node/metered-channel/src/lib.rs b/node/metered-channel/src/lib.rs index 01c3e95311f0..e2fc0d84b50f 100644 --- a/node/metered-channel/src/lib.rs +++ b/node/metered-channel/src/lib.rs @@ -59,11 +59,11 @@ impl Meter { } fn note_sent(&self) { - self.sent.fetch_add(1, Ordering::Release); + self.sent.fetch_add(1, Ordering::Relaxed); } fn retract_sent(&self) { - self.sent.fetch_sub(1, Ordering::Acquire); + self.sent.fetch_sub(1, Ordering::Relaxed); } fn note_received(&self) { diff --git a/node/network/availability-distribution/src/requester/fetch_task/mod.rs b/node/network/availability-distribution/src/requester/fetch_task/mod.rs index ecba462df175..bdb51bdd2a1c 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/mod.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/mod.rs @@ -263,7 +263,7 @@ impl RunningTask { async fn run_inner(mut self) { let mut bad_validators = Vec::new(); let mut succeeded = false; - let mut retry_count: u32 = 0; + let mut count: u32 = 0; let mut _span = self.span.child("fetch-task") .with_chunk_index(self.request.index.0) .with_relay_parent(self.relay_parent); @@ -271,10 +271,10 @@ impl RunningTask { while let Some(validator) = self.group.pop() { let _try_span = _span.child("try"); // Report retries: - if retry_count > 0 { + if count > 0 { self.metrics.on_retry(); } - retry_count +=1; + count +=1; // Send request: let resp = match self.do_request(&validator).await { @@ -319,7 +319,7 @@ impl RunningTask { _span.add_string_tag("success", "true"); break; } - _span.add_int_tag("tries", retry_count as _); + _span.add_int_tag("tries", count as _); if succeeded { self.metrics.on_fetch(SUCCEEDED); self.conclude(bad_validators).await; diff --git a/node/network/availability-distribution/src/requester/fetch_task/tests.rs b/node/network/availability-distribution/src/requester/fetch_task/tests.rs index ec0ebcb4182a..db8790435b2c 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/tests.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/tests.rs @@ -40,8 +40,7 @@ use super::*; fn task_can_be_canceled() { let (task, _rx) = get_test_running_task(); let (handle, kill) = oneshot::channel(); - drop(handle); - + std::mem::drop(handle); let running_task = task.run(kill); futures::pin_mut!(running_task); let waker = noop_waker(); diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index 6c54b703e3ee..f8eab8689a95 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -49,7 +49,6 @@ use polkadot_node_network_protocol::{ request::RequestError, }, }; -use polkadot_node_subsystem_util::metrics::Metrics; use polkadot_node_subsystem_util::request_session_info; use polkadot_erasure_coding::{branches, branch_hash, recovery_threshold, obtain_chunks_v1}; mod error; @@ -68,7 +67,6 @@ const LRU_SIZE: usize = 16; /// The Availability Recovery Subsystem. pub struct AvailabilityRecoverySubsystem { fast_path: bool, - metrics: Metrics, } struct RequestFromBackersPhase { @@ -744,13 +742,13 @@ async fn query_full_data( impl AvailabilityRecoverySubsystem { /// Create a new instance of `AvailabilityRecoverySubsystem` which starts with a fast path to request data from backers. - pub fn with_fast_path(metrics: Metrics) -> Self { - Self { fast_path: true, metrics } + pub fn with_fast_path() -> Self { + Self { fast_path: true } } /// Create a new instance of `AvailabilityRecoverySubsystem` which requests only chunks - pub fn with_chunks_only(metrics: Metrics) -> Self { - Self { fast_path: false, metrics } + pub fn with_chunks_only() -> Self { + Self { fast_path: false } } async fn run( diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml index 8de92b587c82..3ecb90f45ffa 100644 --- a/node/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -12,7 +12,7 @@ futures = "0.3.12" futures-timer = "3.0.2" polkadot-node-primitives = { package = "polkadot-node-primitives", path = "../primitives" } polkadot-node-subsystem-util = { path = "../subsystem-util" } -polkadot-overseer-gen = { path = "./overseer-gen" } +polkadot-procmacro-overseer-subsystems-gen = { path = "./subsystems-gen" } polkadot-primitives = { path = "../../primitives" } polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../subsystem" } tracing = "0.1.25" diff --git a/node/overseer/overseer-gen/tests/ui/err-02-enum.rs b/node/overseer/overseer-gen/tests/ui/err-02-enum.rs index 04cf2f519a55..7f159217252c 100644 --- a/node/overseer/overseer-gen/tests/ui/err-02-enum.rs +++ b/node/overseer/overseer-gen/tests/ui/err-02-enum.rs @@ -2,26 +2,31 @@ use polkadot_overseer_gen::overlord; -struct Msg(u8); +#[derive(Default)] +struct AwesomeSubSys; -#[derive(Default, Clone, Copy)] -struct AwesomeSub; +struct SigSigSig; -#[overlord(Wrapper)] -#[derive(Clone, AllSubsystemsGen)] +struct Event; + +#[derive(Clone)] +struct MsgStrukt(u8); + +#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] enum Overseer { - #[subsystem(Msg)] - Sub0(AwesomeSub), + #[subsystem(MsgStrukt)] + Sub0(AwesomeSubSys), } -struct Spawner; +#[derive(Debug, Clone)] +struct DummySpawner; -fn main() { - let overseer = Overseer::>::builder() - .sub0(FooSubSys::default()) - .build(Spawner); +struct DummyCtx; - // try to replace one subsystem with another that can not handle `X`. - // since it's missing the trait bound. - let overseer = overseer.replace_sub0(TequilaInABar::default()); +fn main() { + let overseer = Overseer::<_,_>::builder() + .sub0(AwesomeSubSys::default()) + .i_like_pie(std::f64::consts::PI) + .spawner(DummySpawner) + .build(|| -> DummyCtx { DummyCtx } ); } diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 54a99d2d300f..6a674fb29e35 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -92,14 +92,14 @@ pub use polkadot_subsystem::{ Subsystem, SubsystemContext, SubsystemSender, OverseerSignal, FromOverseer, SubsystemError, SubsystemResult, SpawnedSubsystem, ActiveLeavesUpdate, ActivatedLeaf, DummySubsystem, jaeger, }; -use polkadot_node_subsystem_util::{TimeoutExt, metered, Metronome}; +use polkadot_node_subsystem_util::{TimeoutExt, metrics::{self, prometheus}, metered, Metronome}; use polkadot_node_primitives::SpawnNamed; use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; - -mod metrics; - -use crate::metrics::*; +// A capacity of bounded channels inside the overseer. +const CHANNEL_CAPACITY: usize = 1024; +// The capacity of signal channels to subsystems. +const SIGNAL_CHANNEL_CAPACITY: usize = 64; // A graceful `Overseer` teardown time delay. const STOP_DELAY: u64 = 1; @@ -136,17 +136,234 @@ impl HeadSupportsParachains for Arc where } } +/// This struct is passed as an argument to create a new instance of an [`Overseer`]. +/// +/// As any entity that satisfies the interface may act as a [`Subsystem`] this allows +/// mocking in the test code: +/// +/// Each [`Subsystem`] is supposed to implement some interface that is generic over +/// message type that is specific to this [`Subsystem`]. At the moment not all +/// subsystems are implemented and the rest can be mocked with the [`DummySubsystem`]. +#[derive(Debug, Clone, AllSubsystemsGen)] +pub struct AllSubsystems< + CV = (), CB = (), CS = (), SD = (), AD = (), AR = (), BS = (), BD = (), P = (), + RA = (), AS = (), NB = (), CA = (), CG = (), CP = (), ApD = (), ApV = (), + GS = (), +> { + /// A candidate validation subsystem. + pub candidate_validation: CV, + /// A candidate backing subsystem. + pub candidate_backing: CB, + /// A candidate selection subsystem. + pub candidate_selection: CS, + /// A statement distribution subsystem. + pub statement_distribution: SD, + /// An availability distribution subsystem. + pub availability_distribution: AD, + /// An availability recovery subsystem. + pub availability_recovery: AR, + /// A bitfield signing subsystem. + pub bitfield_signing: BS, + /// A bitfield distribution subsystem. + pub bitfield_distribution: BD, + /// A provisioner subsystem. + pub provisioner: P, + /// A runtime API subsystem. + pub runtime_api: RA, + /// An availability store subsystem. + pub availability_store: AS, + /// A network bridge subsystem. + pub network_bridge: NB, + /// A Chain API subsystem. + pub chain_api: CA, + /// A Collation Generation subsystem. + pub collation_generation: CG, + /// A Collator Protocol subsystem. + pub collator_protocol: CP, + /// An Approval Distribution subsystem. + pub approval_distribution: ApD, + /// An Approval Voting subsystem. + pub approval_voting: ApV, + /// A Connection Request Issuer subsystem. + pub gossip_support: GS, +} +impl + AllSubsystems +{ + /// Create a new instance of [`AllSubsystems`]. + /// + /// Each subsystem is set to [`DummySystem`]. + /// + ///# Note + /// + /// Because of a bug in rustc it is required that when calling this function, + /// you provide a "random" type for the first generic parameter: + /// + /// ``` + /// polkadot_overseer::AllSubsystems::<()>::dummy(); + /// ``` + pub fn dummy() -> AllSubsystems< + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + > { + AllSubsystems { + candidate_validation: DummySubsystem, + candidate_backing: DummySubsystem, + candidate_selection: DummySubsystem, + statement_distribution: DummySubsystem, + availability_distribution: DummySubsystem, + availability_recovery: DummySubsystem, + bitfield_signing: DummySubsystem, + bitfield_distribution: DummySubsystem, + provisioner: DummySubsystem, + runtime_api: DummySubsystem, + availability_store: DummySubsystem, + network_bridge: DummySubsystem, + chain_api: DummySubsystem, + collation_generation: DummySubsystem, + collator_protocol: DummySubsystem, + approval_distribution: DummySubsystem, + approval_voting: DummySubsystem, + gossip_support: DummySubsystem, + } + } -impl Debug for ToOverseer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ToOverseer::SpawnJob { .. } => write!(f, "OverseerMessage::Spawn(..)"), - ToOverseer::SpawnBlockingJob { .. } => write!(f, "OverseerMessage::SpawnBlocking(..)") + fn as_ref(&self) -> AllSubsystems<&'_ CV, &'_ CB, &'_ CS, &'_ SD, &'_ AD, &'_ AR, &'_ BS, &'_ BD, &'_ P, &'_ RA, &'_ AS, &'_ NB, &'_ CA, &'_ CG, &'_ CP, &'_ ApD, &'_ ApV, &'_ GS> { + AllSubsystems { + candidate_validation: &self.candidate_validation, + candidate_backing: &self.candidate_backing, + candidate_selection: &self.candidate_selection, + statement_distribution: &self.statement_distribution, + availability_distribution: &self.availability_distribution, + availability_recovery: &self.availability_recovery, + bitfield_signing: &self.bitfield_signing, + bitfield_distribution: &self.bitfield_distribution, + provisioner: &self.provisioner, + runtime_api: &self.runtime_api, + availability_store: &self.availability_store, + network_bridge: &self.network_bridge, + chain_api: &self.chain_api, + collation_generation: &self.collation_generation, + collator_protocol: &self.collator_protocol, + approval_distribution: &self.approval_distribution, + approval_voting: &self.approval_voting, + gossip_support: &self.gossip_support, + } + } + + fn map_subsystems(self, m: M) + -> AllSubsystems< + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + > + where + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem

, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + { + AllSubsystems { + candidate_validation: m.map_subsystem(self.candidate_validation), + candidate_backing: m.map_subsystem(self.candidate_backing), + candidate_selection: m.map_subsystem(self.candidate_selection), + statement_distribution: m.map_subsystem(self.statement_distribution), + availability_distribution: m.map_subsystem(self.availability_distribution), + availability_recovery: m.map_subsystem(self.availability_recovery), + bitfield_signing: m.map_subsystem(self.bitfield_signing), + bitfield_distribution: m.map_subsystem(self.bitfield_distribution), + provisioner: m.map_subsystem(self.provisioner), + runtime_api: m.map_subsystem(self.runtime_api), + availability_store: m.map_subsystem(self.availability_store), + network_bridge: m.map_subsystem(self.network_bridge), + chain_api: m.map_subsystem(self.chain_api), + collation_generation: m.map_subsystem(self.collation_generation), + collator_protocol: m.map_subsystem(self.collator_protocol), + approval_distribution: m.map_subsystem(self.approval_distribution), + approval_voting: m.map_subsystem(self.approval_voting), + gossip_support: m.map_subsystem(self.gossip_support), } } } +type AllSubsystemsSame = AllSubsystems< + T, T, T, T, T, + T, T, T, T, T, + T, T, T, T, T, + T, T, T, +>; + +/// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. +/// +/// It wraps a system-wide [`AllMessages`] type that represents all possible +/// messages in the system. +/// +/// [`AllMessages`]: enum.AllMessages.html +/// [`Subsystem`]: trait.Subsystem.html +/// [`Overseer`]: struct.Overseer.html +enum ToOverseer { + /// A message that wraps something the `Subsystem` is desiring to + /// spawn on the overseer and a `oneshot::Sender` to signal the result + /// of the spawn. + SpawnJob { + name: &'static str, + s: BoxFuture<'static, ()>, + }, + + /// Same as `SpawnJob` but for blocking tasks to be executed on a + /// dedicated thread pool. + SpawnBlockingJob { + name: &'static str, + s: BoxFuture<'static, ()>, + }, +} + /// An event telling the `Overseer` on the particular block /// that has been imported or finalized. /// @@ -200,7 +417,59 @@ enum ExternalRequest { }, } +/// A handler used to communicate with the [`Overseer`]. +/// +/// [`Overseer`]: struct.Overseer.html +#[derive(Clone)] +pub struct OverseerHandler { + events_tx: metered::MeteredSender, +} + +impl OverseerHandler { + /// Inform the `Overseer` that that some block was imported. + #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] + pub async fn block_imported(&mut self, block: BlockInfo) { + self.send_and_log_error(Event::BlockImported(block)).await + } + + /// Send some message to one of the `Subsystem`s. + #[tracing::instrument(level = "trace", skip(self, msg), fields(subsystem = LOG_TARGET))] + pub async fn send_msg(&mut self, msg: impl Into) { + self.send_and_log_error(Event::MsgToSubsystem(msg.into())).await + } + + /// Inform the `Overseer` that some block was finalized. + #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] + pub async fn block_finalized(&mut self, block: BlockInfo) { + self.send_and_log_error(Event::BlockFinalized(block)).await + } + + /// Wait for a block with the given hash to be in the active-leaves set. + /// + /// The response channel responds if the hash was activated and is closed if the hash was deactivated. + /// Note that due the fact the overseer doesn't store the whole active-leaves set, only deltas, + /// the response channel may never return if the hash was deactivated before this call. + /// In this case, it's the caller's responsibility to ensure a timeout is set. + #[tracing::instrument(level = "trace", skip(self, response_channel), fields(subsystem = LOG_TARGET))] + pub async fn wait_for_activation(&mut self, hash: Hash, response_channel: oneshot::Sender>) { + self.send_and_log_error(Event::ExternalRequest(ExternalRequest::WaitForActivation { + hash, + response_channel + })).await + } + + /// Tell `Overseer` to shutdown. + #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] + pub async fn stop(&mut self) { + self.send_and_log_error(Event::Stop).await + } + async fn send_and_log_error(&mut self, event: Event) { + if self.events_tx.send(event).await.is_err() { + tracing::info!(target: LOG_TARGET, "Failed to send an event to Overseer"); + } + } +} /// Glues together the [`Overseer`] and `BlockchainEvents` by forwarding /// import and finality notifications into the [`OverseerHandler`]. @@ -237,8 +506,264 @@ pub async fn forward_events>( } } +impl Debug for ToOverseer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ToOverseer::SpawnJob { .. } => write!(f, "OverseerMessage::Spawn(..)"), + ToOverseer::SpawnBlockingJob { .. } => write!(f, "OverseerMessage::SpawnBlocking(..)") + } + } +} + +/// A running instance of some [`Subsystem`]. +/// +/// [`Subsystem`]: trait.Subsystem.html +struct SubsystemInstance { + tx_signal: metered::MeteredSender, + tx_bounded: metered::MeteredSender>, + meters: SubsystemMeters, + signals_received: usize, + name: &'static str, +} + +#[derive(Debug)] +struct MessagePacket { + signals_received: usize, + message: T, +} + +fn make_packet(signals_received: usize, message: T) -> MessagePacket { + MessagePacket { + signals_received, + message, + } +} + +// The channels held by every subsystem to communicate with every other subsystem. +#[derive(Debug, Clone)] +struct ChannelsOut { + candidate_validation: metered::MeteredSender>, + candidate_backing: metered::MeteredSender>, + candidate_selection: metered::MeteredSender>, + statement_distribution: metered::MeteredSender>, + availability_distribution: metered::MeteredSender>, + availability_recovery: metered::MeteredSender>, + bitfield_signing: metered::MeteredSender>, + bitfield_distribution: metered::MeteredSender>, + provisioner: metered::MeteredSender>, + runtime_api: metered::MeteredSender>, + availability_store: metered::MeteredSender>, + network_bridge: metered::MeteredSender>, + chain_api: metered::MeteredSender>, + collation_generation: metered::MeteredSender>, + collator_protocol: metered::MeteredSender>, + approval_distribution: metered::MeteredSender>, + approval_voting: metered::MeteredSender>, + gossip_support: metered::MeteredSender>, + + candidate_validation_unbounded: metered::UnboundedMeteredSender>, + candidate_backing_unbounded: metered::UnboundedMeteredSender>, + candidate_selection_unbounded: metered::UnboundedMeteredSender>, + statement_distribution_unbounded: metered::UnboundedMeteredSender>, + availability_distribution_unbounded: metered::UnboundedMeteredSender>, + availability_recovery_unbounded: metered::UnboundedMeteredSender>, + bitfield_signing_unbounded: metered::UnboundedMeteredSender>, + bitfield_distribution_unbounded: metered::UnboundedMeteredSender>, + provisioner_unbounded: metered::UnboundedMeteredSender>, + runtime_api_unbounded: metered::UnboundedMeteredSender>, + availability_store_unbounded: metered::UnboundedMeteredSender>, + network_bridge_unbounded: metered::UnboundedMeteredSender>, + chain_api_unbounded: metered::UnboundedMeteredSender>, + collation_generation_unbounded: metered::UnboundedMeteredSender>, + collator_protocol_unbounded: metered::UnboundedMeteredSender>, + approval_distribution_unbounded: metered::UnboundedMeteredSender>, + approval_voting_unbounded: metered::UnboundedMeteredSender>, + gossip_support_unbounded: metered::UnboundedMeteredSender>, +} + +impl ChannelsOut { + async fn send_and_log_error( + &mut self, + signals_received: usize, + message: AllMessages, + ) { + let res = match message { + AllMessages::CandidateValidation(msg) => { + self.candidate_validation.send(make_packet(signals_received, msg)).await + }, + AllMessages::CandidateBacking(msg) => { + self.candidate_backing.send(make_packet(signals_received, msg)).await + }, + AllMessages::CandidateSelection(msg) => { + self.candidate_selection.send(make_packet(signals_received, msg)).await + }, + AllMessages::StatementDistribution(msg) => { + self.statement_distribution.send(make_packet(signals_received, msg)).await + }, + AllMessages::AvailabilityDistribution(msg) => { + self.availability_distribution.send(make_packet(signals_received, msg)).await + }, + AllMessages::AvailabilityRecovery(msg) => { + self.availability_recovery.send(make_packet(signals_received, msg)).await + }, + AllMessages::BitfieldDistribution(msg) => { + self.bitfield_distribution.send(make_packet(signals_received, msg)).await + }, + AllMessages::BitfieldSigning(msg) => { + self.bitfield_signing.send(make_packet(signals_received, msg)).await + }, + AllMessages::Provisioner(msg) => { + self.provisioner.send(make_packet(signals_received, msg)).await + }, + AllMessages::RuntimeApi(msg) => { + self.runtime_api.send(make_packet(signals_received, msg)).await + }, + AllMessages::AvailabilityStore(msg) => { + self.availability_store.send(make_packet(signals_received, msg)).await + }, + AllMessages::NetworkBridge(msg) => { + self.network_bridge.send(make_packet(signals_received, msg)).await + }, + AllMessages::ChainApi(msg) => { + self.chain_api.send(make_packet(signals_received, msg)).await + }, + AllMessages::CollationGeneration(msg) => { + self.collation_generation.send(make_packet(signals_received, msg)).await + }, + AllMessages::CollatorProtocol(msg) => { + self.collator_protocol.send(make_packet(signals_received, msg)).await + }, + AllMessages::ApprovalDistribution(msg) => { + self.approval_distribution.send(make_packet(signals_received, msg)).await + }, + AllMessages::ApprovalVoting(msg) => { + self.approval_voting.send(make_packet(signals_received, msg)).await + }, + AllMessages::GossipSupport(msg) => { + self.gossip_support.send(make_packet(signals_received, msg)).await + }, + }; + + if res.is_err() { + tracing::debug!( + target: LOG_TARGET, + "Failed to send a message to another subsystem", + ); + } + } -type SubsystemIncomingMessages = SubsystemIncomingMessages; + + fn send_unbounded_and_log_error( + &self, + signals_received: usize, + message: AllMessages, + ) { + let res = match message { + AllMessages::CandidateValidation(msg) => { + self.candidate_validation_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::CandidateBacking(msg) => { + self.candidate_backing_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::CandidateSelection(msg) => { + self.candidate_selection_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::StatementDistribution(msg) => { + self.statement_distribution_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::AvailabilityDistribution(msg) => { + self.availability_distribution_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::AvailabilityRecovery(msg) => { + self.availability_recovery_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::BitfieldDistribution(msg) => { + self.bitfield_distribution_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::BitfieldSigning(msg) => { + self.bitfield_signing_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::Provisioner(msg) => { + self.provisioner_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::RuntimeApi(msg) => { + self.runtime_api_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::AvailabilityStore(msg) => { + self.availability_store_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::NetworkBridge(msg) => { + self.network_bridge_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::ChainApi(msg) => { + self.chain_api_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::CollationGeneration(msg) => { + self.collation_generation_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::CollatorProtocol(msg) => { + self.collator_protocol_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::ApprovalDistribution(msg) => { + self.approval_distribution_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::ApprovalVoting(msg) => { + self.approval_voting_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + AllMessages::GossipSupport(msg) => { + self.gossip_support_unbounded + .unbounded_send(make_packet(signals_received, msg)) + .map_err(|e| e.into_send_error()) + }, + }; + + if res.is_err() { + tracing::debug!( + target: LOG_TARGET, + "Failed to send a message to another subsystem", + ); + } + } +} + +type SubsystemIncomingMessages = stream::Select< + metered::MeteredReceiver>, + metered::UnboundedMeteredReceiver>, +>; #[derive(Debug, Default, Clone)] struct SignalsReceived(Arc); @@ -253,6 +778,33 @@ impl SignalsReceived { } } +/// A sender from subsystems to other subsystems. +#[derive(Debug, Clone)] +pub struct OverseerSubsystemSender { + channels: ChannelsOut, + signals_received: SignalsReceived, +} + +#[async_trait::async_trait] +impl SubsystemSender for OverseerSubsystemSender { + async fn send_message(&mut self, msg: AllMessages) { + self.channels.send_and_log_error(self.signals_received.load(), msg).await; + } + + async fn send_messages(&mut self, msgs: T) + where T: IntoIterator + Send, T::IntoIter: Send + { + // This can definitely be optimized if necessary. + for msg in msgs { + self.send_message(msg).await; + } + } + + fn send_unbounded_message(&mut self, msg: AllMessages) { + self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); + } +} + /// A context type that is given to the [`Subsystem`] upon spawning. /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s /// or to spawn it's [`SubsystemJob`]s. @@ -494,62 +1046,40 @@ struct SubsystemMeterReadouts { } /// The `Overseer` itself. -#[overlord(event=Event, signal=OverseerSignal, gen=AllMessages)] pub struct Overseer { - #[subsystem(no_dispatch, CandidateValidationMessage)] - candidate_validation: CandidateValidation, - - #[subsystem(no_dispatch, CandidateBackingMessage)] - candidate_backing: CandidateBacking, - - #[subsystem(no_dispatch, CandidateSelectionMessage)] - candidate_selection: CandidateSelection, - - #[subsystem(StatementDistributionMessage)] - statement_distribution: StatementDistribution, - - #[subsystem(no_dispatch, AvailabilityDistributionMessage)] - availability_distribution: AvailabilityDistribution, - - #[subsystem(no_dispatch, AvailabilityRecoveryMessage)] - availability_recovery: AvailabilityRecovery, - - #[subsystem(no_dispatch, BitfieldSigningMessage)] - bitfield_signing: BitfieldSigning, - - #[subsystem(BitfieldDistributionMessage)] - bitfield_distribution: BitfieldDistribution, - - #[subsystem(no_dispatch, ProvisionerMessage)] - provisioner: Provisioner, - - #[subsystem(no_dispatch, RuntimeApiMessage)] - runtime_api: RuntimeApi, - - #[subsystem(no_dispatch, blocking, AvailabilityStoreMessage)] - availability_store: AvailabilityStore, - - #[subsystem(no_dispatch, NetworkBridgeMessage)] - network_bridge: NetworkBridge, - - #[subsystem(no_dispatch, blocking, ChainApiMessage)] - chain_api: ChainApi, - - #[subsystem(no_dispatch, CollationGenerationMessage)] - collation_generation: CollationGeneration, - - #[subsystem(no_dispatch, CollatorProtocolMessage)] - collator_protocol: CollatorProtocol, - - #[subsystem(ApprovalDistributionMessage)] - approval_distribution: ApprovalDistribution, - - #[subsystem(no_dispatch, blocking, ApprovalVotingMessage)] - approval_voting: ApprovalVoting, - - #[subsystem(no_dispatch, GossipSupportMessage)] - gossip_support: GossipSupport, - + /// Handles to all subsystems. + subsystems: AllSubsystems< + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + OverseenSubsystem, + >, + + /// Spawner to spawn tasks to. + s: S, + + /// Here we keep handles to spawned subsystems to be notified when they terminate. + running_subsystems: FuturesUnordered>>, + + /// Gather running subsystems' outbound streams into one. + to_overseer_rx: Fuse>, + + /// Events that are sent to the overseer from the outside world + events_rx: metered::MeteredReceiver, /// External listeners waiting for a hash to be in the active-leave set. activation_external_listeners: HashMap>>>, @@ -572,8 +1102,178 @@ pub struct Overseer { metrics: Metrics, } +/// Overseer Prometheus metrics. +#[derive(Clone)] +struct MetricsInner { + activated_heads_total: prometheus::Counter, + deactivated_heads_total: prometheus::Counter, + messages_relayed_total: prometheus::Counter, + to_subsystem_bounded_sent: prometheus::GaugeVec, + to_subsystem_bounded_received: prometheus::GaugeVec, + to_subsystem_unbounded_sent: prometheus::GaugeVec, + to_subsystem_unbounded_received: prometheus::GaugeVec, + signals_sent: prometheus::GaugeVec, + signals_received: prometheus::GaugeVec, +} + +#[derive(Default, Clone)] +struct Metrics(Option); + +impl Metrics { + fn on_head_activated(&self) { + if let Some(metrics) = &self.0 { + metrics.activated_heads_total.inc(); + } + } + + fn on_head_deactivated(&self) { + if let Some(metrics) = &self.0 { + metrics.deactivated_heads_total.inc(); + } + } + + fn on_message_relayed(&self) { + if let Some(metrics) = &self.0 { + metrics.messages_relayed_total.inc(); + } + } + + fn channel_fill_level_snapshot( + &self, + to_subsystem: AllSubsystemsSame<(&'static str, SubsystemMeterReadouts)>, + ) { + self.0.as_ref().map(|metrics| { + to_subsystem.map_subsystems( + |(name, readouts): (_, SubsystemMeterReadouts)| { + metrics.to_subsystem_bounded_sent.with_label_values(&[name]) + .set(readouts.bounded.sent as u64); + + metrics.to_subsystem_bounded_received.with_label_values(&[name]) + .set(readouts.bounded.received as u64); + + metrics.to_subsystem_unbounded_sent.with_label_values(&[name]) + .set(readouts.unbounded.sent as u64); + + metrics.to_subsystem_unbounded_received.with_label_values(&[name]) + .set(readouts.unbounded.received as u64); + + metrics.signals_sent.with_label_values(&[name]) + .set(readouts.signals.sent as u64); + + metrics.signals_received.with_label_values(&[name]) + .set(readouts.signals.received as u64); + }); + }); + } +} + +impl metrics::Metrics for Metrics { + fn try_register(registry: &prometheus::Registry) -> Result { + let metrics = MetricsInner { + activated_heads_total: prometheus::register( + prometheus::Counter::new( + "parachain_activated_heads_total", + "Number of activated heads." + )?, + registry, + )?, + deactivated_heads_total: prometheus::register( + prometheus::Counter::new( + "parachain_deactivated_heads_total", + "Number of deactivated heads." + )?, + registry, + )?, + messages_relayed_total: prometheus::register( + prometheus::Counter::new( + "parachain_messages_relayed_total", + "Number of messages relayed by Overseer." + )?, + registry, + )?, + to_subsystem_bounded_sent: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "parachain_subsystem_bounded_sent", + "Number of elements sent to subsystems' bounded queues", + ), + &[ + "subsystem_name", + ], + )?, + registry, + )?, + to_subsystem_bounded_received: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "parachain_subsystem_bounded_received", + "Number of elements received by subsystems' bounded queues", + ), + &[ + "subsystem_name", + ], + )?, + registry, + )?, + to_subsystem_unbounded_sent: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "parachain_subsystem_unbounded_sent", + "Number of elements sent to subsystems' unbounded queues", + ), + &[ + "subsystem_name", + ], + )?, + registry, + )?, + to_subsystem_unbounded_received: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "parachain_subsystem_unbounded_received", + "Number of elements received by subsystems' unbounded queues", + ), + &[ + "subsystem_name", + ], + )?, + registry, + )?, + signals_sent: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "parachain_overseer_signals_sent", + "Number of signals sent by overseer to subsystems", + ), + &[ + "subsystem_name", + ], + )?, + registry, + )?, + signals_received: prometheus::register( + prometheus::GaugeVec::::new( + prometheus::Opts::new( + "parachain_overseer_signals_received", + "Number of signals received by subsystems from overseer", + ), + &[ + "subsystem_name", + ], + )?, + registry, + )?, + }; + Ok(Metrics(Some(metrics))) + } +} + +impl fmt::Debug for Metrics { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Metrics {{...}}") + } +} -#[overlord] impl Overseer where S: SpawnNamed, @@ -592,7 +1292,8 @@ where /// . +-----------+ +-----------+ +----------+ +---------+ . /// ............................................................... /// | - /// `spawn` a `job` + /// probably `spawn` + /// a `job` /// | /// V /// +-----------+ @@ -607,7 +1308,12 @@ where /// /// The [`Subsystems`] may be any type as long as they implement an expected interface. /// Here, we create a mock validation subsystem and a few dummy ones and start the `Overseer` with them. - /// For the sake of siroute_messagerseer::{Overseer, HeadSupportsParachains, AllSubsystems}; + /// For the sake of simplicity the termination of the example is done with a timeout. + /// ``` + /// # use std::time::Duration; + /// # use futures::{executor, pin_mut, select, FutureExt}; + /// # use futures_timer::Delay; + /// # use polkadot_overseer::{Overseer, HeadSupportsParachains, AllSubsystems}; /// # use polkadot_primitives::v1::Hash; /// # use polkadot_subsystem::{ /// # Subsystem, DummySubsystem, SpawnedSubsystem, SubsystemContext, @@ -663,22 +1369,422 @@ where /// # /// # }); } /// ``` - pub fn new( + pub fn new( leaves: impl IntoIterator, + all_subsystems: AllSubsystems, prometheus_registry: Option<&prometheus::Registry>, supports_parachains: SupportsParachains, mut s: S, ) -> SubsystemResult<(Self, OverseerHandler)> + where + CV: Subsystem> + Send, + CB: Subsystem> + Send, + CS: Subsystem> + Send, + SD: Subsystem> + Send, + AD: Subsystem> + Send, + AR: Subsystem> + Send, + BS: Subsystem> + Send, + BD: Subsystem> + Send, + P: Subsystem> + Send, + RA: Subsystem> + Send, + AS: Subsystem> + Send, + NB: Subsystem> + Send, + CA: Subsystem> + Send, + CG: Subsystem> + Send, + CP: Subsystem> + Send, + ApD: Subsystem> + Send, + ApV: Subsystem> + Send, + GS: Subsystem> + Send, { + let (events_tx, events_rx) = metered::channel(CHANNEL_CAPACITY); + + let handler = OverseerHandler { + events_tx: events_tx.clone(), + }; + + let metrics = ::register(prometheus_registry)?; + + let (to_overseer_tx, to_overseer_rx) = metered::unbounded(); + + let mut running_subsystems = FuturesUnordered::new(); + + let (candidate_validation_bounded_tx, candidate_validation_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (candidate_backing_bounded_tx, candidate_backing_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (candidate_selection_bounded_tx, candidate_selection_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (statement_distribution_bounded_tx, statement_distribution_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (availability_distribution_bounded_tx, availability_distribution_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (availability_recovery_bounded_tx, availability_recovery_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (bitfield_signing_bounded_tx, bitfield_signing_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (bitfield_distribution_bounded_tx, bitfield_distribution_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (provisioner_bounded_tx, provisioner_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (runtime_api_bounded_tx, runtime_api_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (availability_store_bounded_tx, availability_store_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (network_bridge_bounded_tx, network_bridge_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (chain_api_bounded_tx, chain_api_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (collator_protocol_bounded_tx, collator_protocol_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (collation_generation_bounded_tx, collation_generation_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (approval_distribution_bounded_tx, approval_distribution_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (approval_voting_bounded_tx, approval_voting_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + let (gossip_support_bounded_tx, gossip_support_bounded_rx) + = metered::channel(CHANNEL_CAPACITY); + + let (candidate_validation_unbounded_tx, candidate_validation_unbounded_rx) + = metered::unbounded(); + let (candidate_backing_unbounded_tx, candidate_backing_unbounded_rx) + = metered::unbounded(); + let (candidate_selection_unbounded_tx, candidate_selection_unbounded_rx) + = metered::unbounded(); + let (statement_distribution_unbounded_tx, statement_distribution_unbounded_rx) + = metered::unbounded(); + let (availability_distribution_unbounded_tx, availability_distribution_unbounded_rx) + = metered::unbounded(); + let (availability_recovery_unbounded_tx, availability_recovery_unbounded_rx) + = metered::unbounded(); + let (bitfield_signing_unbounded_tx, bitfield_signing_unbounded_rx) + = metered::unbounded(); + let (bitfield_distribution_unbounded_tx, bitfield_distribution_unbounded_rx) + = metered::unbounded(); + let (provisioner_unbounded_tx, provisioner_unbounded_rx) + = metered::unbounded(); + let (runtime_api_unbounded_tx, runtime_api_unbounded_rx) + = metered::unbounded(); + let (availability_store_unbounded_tx, availability_store_unbounded_rx) + = metered::unbounded(); + let (network_bridge_unbounded_tx, network_bridge_unbounded_rx) + = metered::unbounded(); + let (chain_api_unbounded_tx, chain_api_unbounded_rx) + = metered::unbounded(); + let (collator_protocol_unbounded_tx, collator_protocol_unbounded_rx) + = metered::unbounded(); + let (collation_generation_unbounded_tx, collation_generation_unbounded_rx) + = metered::unbounded(); + let (approval_distribution_unbounded_tx, approval_distribution_unbounded_rx) + = metered::unbounded(); + let (approval_voting_unbounded_tx, approval_voting_unbounded_rx) + = metered::unbounded(); + let (gossip_support_unbounded_tx, gossip_support_unbounded_rx) + = metered::unbounded(); + + let channels_out = ChannelsOut { + candidate_validation: candidate_validation_bounded_tx.clone(), + candidate_backing: candidate_backing_bounded_tx.clone(), + candidate_selection: candidate_selection_bounded_tx.clone(), + statement_distribution: statement_distribution_bounded_tx.clone(), + availability_distribution: availability_distribution_bounded_tx.clone(), + availability_recovery: availability_recovery_bounded_tx.clone(), + bitfield_signing: bitfield_signing_bounded_tx.clone(), + bitfield_distribution: bitfield_distribution_bounded_tx.clone(), + provisioner: provisioner_bounded_tx.clone(), + runtime_api: runtime_api_bounded_tx.clone(), + availability_store: availability_store_bounded_tx.clone(), + network_bridge: network_bridge_bounded_tx.clone(), + chain_api: chain_api_bounded_tx.clone(), + collator_protocol: collator_protocol_bounded_tx.clone(), + collation_generation: collation_generation_bounded_tx.clone(), + approval_distribution: approval_distribution_bounded_tx.clone(), + approval_voting: approval_voting_bounded_tx.clone(), + gossip_support: gossip_support_bounded_tx.clone(), + + candidate_validation_unbounded: candidate_validation_unbounded_tx.clone(), + candidate_backing_unbounded: candidate_backing_unbounded_tx.clone(), + candidate_selection_unbounded: candidate_selection_unbounded_tx.clone(), + statement_distribution_unbounded: statement_distribution_unbounded_tx.clone(), + availability_distribution_unbounded: availability_distribution_unbounded_tx.clone(), + availability_recovery_unbounded: availability_recovery_unbounded_tx.clone(), + bitfield_signing_unbounded: bitfield_signing_unbounded_tx.clone(), + bitfield_distribution_unbounded: bitfield_distribution_unbounded_tx.clone(), + provisioner_unbounded: provisioner_unbounded_tx.clone(), + runtime_api_unbounded: runtime_api_unbounded_tx.clone(), + availability_store_unbounded: availability_store_unbounded_tx.clone(), + network_bridge_unbounded: network_bridge_unbounded_tx.clone(), + chain_api_unbounded: chain_api_unbounded_tx.clone(), + collator_protocol_unbounded: collator_protocol_unbounded_tx.clone(), + collation_generation_unbounded: collation_generation_unbounded_tx.clone(), + approval_distribution_unbounded: approval_distribution_unbounded_tx.clone(), + approval_voting_unbounded: approval_voting_unbounded_tx.clone(), + gossip_support_unbounded: gossip_support_unbounded_tx.clone(), + }; + + let candidate_validation_subsystem = spawn( + &mut s, + candidate_validation_bounded_tx, + stream::select(candidate_validation_bounded_rx, candidate_validation_unbounded_rx), + candidate_validation_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.candidate_validation, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let candidate_backing_subsystem = spawn( + &mut s, + candidate_backing_bounded_tx, + stream::select(candidate_backing_bounded_rx, candidate_backing_unbounded_rx), + candidate_backing_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.candidate_backing, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let candidate_selection_subsystem = spawn( + &mut s, + candidate_selection_bounded_tx, + stream::select(candidate_selection_bounded_rx, candidate_selection_unbounded_rx), + candidate_selection_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.candidate_selection, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let statement_distribution_subsystem = spawn( + &mut s, + statement_distribution_bounded_tx, + stream::select(statement_distribution_bounded_rx, statement_distribution_unbounded_rx), + candidate_validation_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.statement_distribution, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let availability_distribution_subsystem = spawn( + &mut s, + availability_distribution_bounded_tx, + stream::select(availability_distribution_bounded_rx, availability_distribution_unbounded_rx), + availability_distribution_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.availability_distribution, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let availability_recovery_subsystem = spawn( + &mut s, + availability_recovery_bounded_tx, + stream::select(availability_recovery_bounded_rx, availability_recovery_unbounded_rx), + availability_recovery_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.availability_recovery, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let bitfield_signing_subsystem = spawn( + &mut s, + bitfield_signing_bounded_tx, + stream::select(bitfield_signing_bounded_rx, bitfield_signing_unbounded_rx), + bitfield_signing_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.bitfield_signing, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let bitfield_distribution_subsystem = spawn( + &mut s, + bitfield_distribution_bounded_tx, + stream::select(bitfield_distribution_bounded_rx, bitfield_distribution_unbounded_rx), + bitfield_distribution_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.bitfield_distribution, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let provisioner_subsystem = spawn( + &mut s, + provisioner_bounded_tx, + stream::select(provisioner_bounded_rx, provisioner_unbounded_rx), + provisioner_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.provisioner, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let runtime_api_subsystem = spawn( + &mut s, + runtime_api_bounded_tx, + stream::select(runtime_api_bounded_rx, runtime_api_unbounded_rx), + runtime_api_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.runtime_api, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let availability_store_subsystem = spawn( + &mut s, + availability_store_bounded_tx, + stream::select(availability_store_bounded_rx, availability_store_unbounded_rx), + availability_store_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.availability_store, + &metrics, + &mut running_subsystems, + TaskKind::Blocking, + )?; + + let network_bridge_subsystem = spawn( + &mut s, + network_bridge_bounded_tx, + stream::select(network_bridge_bounded_rx, network_bridge_unbounded_rx), + network_bridge_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.network_bridge, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let chain_api_subsystem = spawn( + &mut s, + chain_api_bounded_tx, + stream::select(chain_api_bounded_rx, chain_api_unbounded_rx), + chain_api_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.chain_api, + &metrics, + &mut running_subsystems, + TaskKind::Blocking, + )?; + + let collation_generation_subsystem = spawn( + &mut s, + collation_generation_bounded_tx, + stream::select(collation_generation_bounded_rx, collation_generation_unbounded_rx), + collation_generation_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.collation_generation, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let collator_protocol_subsystem = spawn( + &mut s, + collator_protocol_bounded_tx, + stream::select(collator_protocol_bounded_rx, collator_protocol_unbounded_rx), + collator_protocol_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.collator_protocol, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let approval_distribution_subsystem = spawn( + &mut s, + approval_distribution_bounded_tx, + stream::select(approval_distribution_bounded_rx, approval_distribution_unbounded_rx), + approval_distribution_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.approval_distribution, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let approval_voting_subsystem = spawn( + &mut s, + approval_voting_bounded_tx, + stream::select(approval_voting_bounded_rx, approval_voting_unbounded_rx), + approval_voting_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.approval_voting, + &metrics, + &mut running_subsystems, + TaskKind::Blocking, + )?; + + let gossip_support_subsystem = spawn( + &mut s, + gossip_support_bounded_tx, + stream::select(gossip_support_bounded_rx, gossip_support_unbounded_rx), + gossip_support_unbounded_tx.meter().clone(), + channels_out.clone(), + to_overseer_tx.clone(), + all_subsystems.gossip_support, + &metrics, + &mut running_subsystems, + TaskKind::Regular, + )?; + + let leaves = leaves + .into_iter() + .map(|BlockInfo { hash, parent_hash: _, number }| (hash, number)) + .collect(); + + let active_leaves = HashMap::new(); + let activation_external_listeners = HashMap::new(); + + let subsystems = AllSubsystems { + candidate_validation: candidate_validation_subsystem, + candidate_backing: candidate_backing_subsystem, + candidate_selection: candidate_selection_subsystem, + statement_distribution: statement_distribution_subsystem, + availability_distribution: availability_distribution_subsystem, + availability_recovery: availability_recovery_subsystem, + bitfield_signing: bitfield_signing_subsystem, + bitfield_distribution: bitfield_distribution_subsystem, + provisioner: provisioner_subsystem, + runtime_api: runtime_api_subsystem, + availability_store: availability_store_subsystem, + network_bridge: network_bridge_subsystem, + chain_api: chain_api_subsystem, + collation_generation: collation_generation_subsystem, + collator_protocol: collator_protocol_subsystem, + approval_distribution: approval_distribution_subsystem, + approval_voting: approval_voting_subsystem, + gossip_support: gossip_support_subsystem, + }; - let (overseer, handler) = Overseer::builder() - .metrics(::register(prometheus_registry)?) - .active_leaves(leaves - .into_iter() - .map(|BlockInfo { hash, parent_hash: _, number }| (hash, number)) - .collect()) - .supports_parachains(supports_parachains), - .build(s); { struct ExtractNameAndMeters; impl<'a, T: 'a> MapSubsystem<&'a OverseenSubsystem> for ExtractNameAndMeters { @@ -696,7 +1802,7 @@ where } } - let subsystem_meters = overseer.subsystems().map_subsystems(ExtractNameAndMeters); + let subsystem_meters = subsystems.as_ref().map_subsystems(ExtractNameAndMeters); let metronome_metrics = metrics.clone(); let metronome = Metronome::new(std::time::Duration::from_millis(950)) .for_each(move |_| { @@ -715,10 +1821,59 @@ where s.spawn("metrics_metronome", Box::pin(metronome)); } + let this = Self { + subsystems, + s, + running_subsystems, + to_overseer_rx: to_overseer_rx.fuse(), + events_rx, + activation_external_listeners, + leaves, + active_leaves, + metrics, + span_per_active_leaf: Default::default(), + supports_parachains, + }; Ok((this, handler)) } + // Stop the overseer. + async fn stop(mut self) { + let _ = self.subsystems.candidate_validation.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.candidate_backing.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.candidate_selection.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.statement_distribution.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.availability_distribution.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.availability_recovery.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.bitfield_signing.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.bitfield_distribution.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.provisioner.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.runtime_api.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.availability_store.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.network_bridge.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.chain_api.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.collator_protocol.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.collation_generation.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.approval_distribution.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.approval_voting.send_signal(OverseerSignal::Conclude).await; + let _ = self.subsystems.gossip_support.send_signal(OverseerSignal::Conclude).await; + + let mut stop_delay = Delay::new(Duration::from_secs(STOP_DELAY)).fuse(); + + loop { + select! { + _ = self.running_subsystems.next() => { + if self.running_subsystems.is_empty() { + break; + } + }, + _ = stop_delay => break, + complete => break, + } + } + } + /// Run the `Overseer`. #[tracing::instrument(skip(self), fields(subsystem = LOG_TARGET))] pub async fn run(mut self) -> SubsystemResult<()> { @@ -1019,6 +2174,1069 @@ where } } +enum TaskKind { + Regular, + Blocking, +} + +fn spawn( + spawner: &mut S, + message_tx: metered::MeteredSender>, + message_rx: SubsystemIncomingMessages, + unbounded_meter: metered::Meter, + to_subsystems: ChannelsOut, + to_overseer_tx: metered::UnboundedMeteredSender, + s: impl Subsystem>, + metrics: &Metrics, + futures: &mut FuturesUnordered>>, + task_kind: TaskKind, +) -> SubsystemResult> { + let (signal_tx, signal_rx) = metered::channel(SIGNAL_CHANNEL_CAPACITY); + let ctx = OverseerSubsystemContext::new( + signal_rx, + message_rx, + to_subsystems, + to_overseer_tx, + metrics.clone(), + ); + let SpawnedSubsystem { future, name } = s.start(ctx); + + let (tx, rx) = oneshot::channel(); + + let fut = Box::pin(async move { + if let Err(e) = future.await { + tracing::error!(subsystem=name, err = ?e, "subsystem exited with error"); + } else { + tracing::debug!(subsystem=name, "subsystem exited without an error"); + } + let _ = tx.send(()); + }); + + match task_kind { + TaskKind::Regular => spawner.spawn(name, fut), + TaskKind::Blocking => spawner.spawn_blocking(name, fut), + } + + futures.push(Box::pin(rx.map(|e| { tracing::warn!(err = ?e, "dropping error"); Ok(()) }))); + + let instance = Some(SubsystemInstance { + meters: SubsystemMeters { + unbounded: unbounded_meter, + bounded: message_tx.meter().clone(), + signals: signal_tx.meter().clone(), + }, + tx_signal: signal_tx, + tx_bounded: message_tx, + signals_received: 0, + name, + }); + + Ok(OverseenSubsystem { + instance, + }) +} #[cfg(test)] -mod tests; +mod tests { + use std::sync::atomic; + use std::collections::HashMap; + use futures::{executor, pin_mut, select, FutureExt, pending}; + + use polkadot_primitives::v1::{CollatorPair, CandidateHash}; + use polkadot_subsystem::{messages::RuntimeApiRequest, messages::NetworkBridgeEvent, jaeger}; + use polkadot_node_primitives::{CollationResult, CollationGenerationConfig, PoV, BlockData}; + use polkadot_node_network_protocol::{PeerId, UnifiedReputationChange}; + use polkadot_node_subsystem_util::metered; + + use sp_core::crypto::Pair as _; + use assert_matches::assert_matches; + + use super::*; + + struct TestSubsystem1(metered::MeteredSender); + + impl Subsystem for TestSubsystem1 + where C: SubsystemContext + { + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let mut sender = self.0; + SpawnedSubsystem { + name: "test-subsystem-1", + future: Box::pin(async move { + let mut i = 0; + loop { + match ctx.recv().await { + Ok(FromOverseer::Communication { .. }) => { + let _ = sender.send(i).await; + i += 1; + continue; + } + Ok(FromOverseer::Signal(OverseerSignal::Conclude)) => return Ok(()), + Err(_) => return Ok(()), + _ => (), + } + } + }), + } + } + } + + struct TestSubsystem2(metered::MeteredSender); + + impl Subsystem for TestSubsystem2 + where C: SubsystemContext + { + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let sender = self.0.clone(); + SpawnedSubsystem { + name: "test-subsystem-2", + future: Box::pin(async move { + let _sender = sender; + let mut c: usize = 0; + loop { + if c < 10 { + let (tx, _) = oneshot::channel(); + ctx.send_message( + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromChainState( + Default::default(), + PoV { + block_data: BlockData(Vec::new()), + }.into(), + tx, + ) + ) + ).await; + c += 1; + continue; + } + match ctx.try_recv().await { + Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => { + break; + } + Ok(Some(_)) => { + continue; + } + Err(_) => return Ok(()), + _ => (), + } + pending!(); + } + + Ok(()) + }), + } + } + } + + struct ReturnOnStart; + + impl Subsystem for ReturnOnStart + where C: SubsystemContext + { + fn start(self, mut _ctx: C) -> SpawnedSubsystem { + SpawnedSubsystem { + name: "test-subsystem-4", + future: Box::pin(async move { + // Do nothing and exit. + Ok(()) + }), + } + } + } + + struct MockSupportsParachains; + + impl HeadSupportsParachains for MockSupportsParachains { + fn head_supports_parachains(&self, _head: &Hash) -> bool { + true + } + } + + // Checks that a minimal configuration of two jobs can run and exchange messages. + #[test] + fn overseer_works() { + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let (s1_tx, s1_rx) = metered::channel::(64); + let (s2_tx, s2_rx) = metered::channel::(64); + + let mut s1_rx = s1_rx.fuse(); + let mut s2_rx = s2_rx.fuse(); + + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_validation(TestSubsystem1(s1_tx)) + .replace_candidate_backing(TestSubsystem2(s2_tx)); + + let (overseer, mut handler) = Overseer::new( + vec![], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + let overseer_fut = overseer.run().fuse(); + + pin_mut!(overseer_fut); + + let mut s1_results = Vec::new(); + let mut s2_results = Vec::new(); + + loop { + select! { + _ = overseer_fut => break, + s1_next = s1_rx.next() => { + match s1_next { + Some(msg) => { + s1_results.push(msg); + if s1_results.len() == 10 { + handler.stop().await; + } + } + None => break, + } + }, + s2_next = s2_rx.next() => { + match s2_next { + Some(_) => s2_results.push(s2_next), + None => break, + } + }, + complete => break, + } + } + + assert_eq!(s1_results, (0..10).collect::>()); + }); + } + + // Checks activated/deactivated metrics are updated properly. + #[test] + fn overseer_metrics_work() { + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let first_block_hash = [1; 32].into(); + let second_block_hash = [2; 32].into(); + let third_block_hash = [3; 32].into(); + + let first_block = BlockInfo { + hash: first_block_hash, + parent_hash: [0; 32].into(), + number: 1, + }; + let second_block = BlockInfo { + hash: second_block_hash, + parent_hash: first_block_hash, + number: 2, + }; + let third_block = BlockInfo { + hash: third_block_hash, + parent_hash: second_block_hash, + number: 3, + }; + + let all_subsystems = AllSubsystems::<()>::dummy(); + let registry = prometheus::Registry::new(); + let (overseer, mut handler) = Overseer::new( + vec![first_block], + all_subsystems, + Some(®istry), + MockSupportsParachains, + spawner, + ).unwrap(); + let overseer_fut = overseer.run().fuse(); + + pin_mut!(overseer_fut); + + handler.block_imported(second_block).await; + handler.block_imported(third_block).await; + handler.send_msg(AllMessages::CandidateValidation(test_candidate_validation_msg())).await; + handler.stop().await; + + select! { + res = overseer_fut => { + assert!(res.is_ok()); + let metrics = extract_metrics(®istry); + assert_eq!(metrics["activated"], 3); + assert_eq!(metrics["deactivated"], 2); + assert_eq!(metrics["relayed"], 1); + }, + complete => (), + } + }); + } + + fn extract_metrics(registry: &prometheus::Registry) -> HashMap<&'static str, u64> { + let gather = registry.gather(); + assert_eq!(gather[0].get_name(), "parachain_activated_heads_total"); + assert_eq!(gather[1].get_name(), "parachain_deactivated_heads_total"); + assert_eq!(gather[2].get_name(), "parachain_messages_relayed_total"); + let activated = gather[0].get_metric()[0].get_counter().get_value() as u64; + let deactivated = gather[1].get_metric()[0].get_counter().get_value() as u64; + let relayed = gather[2].get_metric()[0].get_counter().get_value() as u64; + let mut result = HashMap::new(); + result.insert("activated", activated); + result.insert("deactivated", deactivated); + result.insert("relayed", relayed); + result + } + + // Spawn a subsystem that immediately exits. + // + // Should immediately conclude the overseer itself. + #[test] + fn overseer_ends_on_subsystem_exit() { + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_backing(ReturnOnStart); + let (overseer, _handle) = Overseer::new( + vec![], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + + overseer.run().await.unwrap(); + }) + } + + struct TestSubsystem5(metered::MeteredSender); + + impl Subsystem for TestSubsystem5 + where C: SubsystemContext + { + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let mut sender = self.0.clone(); + + SpawnedSubsystem { + name: "test-subsystem-5", + future: Box::pin(async move { + loop { + match ctx.try_recv().await { + Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => break, + Ok(Some(FromOverseer::Signal(s))) => { + sender.send(s).await.unwrap(); + continue; + }, + Ok(Some(_)) => continue, + Err(_) => break, + _ => (), + } + pending!(); + } + + Ok(()) + }), + } + } + } + + struct TestSubsystem6(metered::MeteredSender); + + impl Subsystem for TestSubsystem6 + where C: SubsystemContext + { + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let mut sender = self.0.clone(); + + SpawnedSubsystem { + name: "test-subsystem-6", + future: Box::pin(async move { + loop { + match ctx.try_recv().await { + Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => break, + Ok(Some(FromOverseer::Signal(s))) => { + sender.send(s).await.unwrap(); + continue; + }, + Ok(Some(_)) => continue, + Err(_) => break, + _ => (), + } + pending!(); + } + + Ok(()) + }), + } + } + } + + // Tests that starting with a defined set of leaves and receiving + // notifications on imported blocks triggers expected `StartWork` and `StopWork` heartbeats. + #[test] + fn overseer_start_stop_works() { + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let first_block_hash = [1; 32].into(); + let second_block_hash = [2; 32].into(); + let third_block_hash = [3; 32].into(); + + let first_block = BlockInfo { + hash: first_block_hash, + parent_hash: [0; 32].into(), + number: 1, + }; + let second_block = BlockInfo { + hash: second_block_hash, + parent_hash: first_block_hash, + number: 2, + }; + let third_block = BlockInfo { + hash: third_block_hash, + parent_hash: second_block_hash, + number: 3, + }; + + let (tx_5, mut rx_5) = metered::channel(64); + let (tx_6, mut rx_6) = metered::channel(64); + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_validation(TestSubsystem5(tx_5)) + .replace_candidate_backing(TestSubsystem6(tx_6)); + let (overseer, mut handler) = Overseer::new( + vec![first_block], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + + let overseer_fut = overseer.run().fuse(); + pin_mut!(overseer_fut); + + let mut ss5_results = Vec::new(); + let mut ss6_results = Vec::new(); + + handler.block_imported(second_block).await; + handler.block_imported(third_block).await; + + let expected_heartbeats = vec![ + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: first_block_hash, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: [ActivatedLeaf { + hash: second_block_hash, + number: 2, + span: Arc::new(jaeger::Span::Disabled), + }].as_ref().into(), + deactivated: [first_block_hash].as_ref().into(), + }), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: [ActivatedLeaf { + hash: third_block_hash, + number: 3, + span: Arc::new(jaeger::Span::Disabled), + }].as_ref().into(), + deactivated: [second_block_hash].as_ref().into(), + }), + ]; + + loop { + select! { + res = overseer_fut => { + assert!(res.is_ok()); + break; + }, + res = rx_5.next() => { + if let Some(res) = res { + ss5_results.push(res); + } + } + res = rx_6.next() => { + if let Some(res) = res { + ss6_results.push(res); + } + } + complete => break, + } + + if ss5_results.len() == expected_heartbeats.len() && + ss6_results.len() == expected_heartbeats.len() { + handler.stop().await; + } + } + + assert_eq!(ss5_results, expected_heartbeats); + assert_eq!(ss6_results, expected_heartbeats); + }); + } + + // Tests that starting with a defined set of leaves and receiving + // notifications on imported blocks triggers expected `StartWork` and `StopWork` heartbeats. + #[test] + fn overseer_finalize_works() { + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let first_block_hash = [1; 32].into(); + let second_block_hash = [2; 32].into(); + let third_block_hash = [3; 32].into(); + + let first_block = BlockInfo { + hash: first_block_hash, + parent_hash: [0; 32].into(), + number: 1, + }; + let second_block = BlockInfo { + hash: second_block_hash, + parent_hash: [42; 32].into(), + number: 2, + }; + let third_block = BlockInfo { + hash: third_block_hash, + parent_hash: second_block_hash, + number: 3, + }; + + let (tx_5, mut rx_5) = metered::channel(64); + let (tx_6, mut rx_6) = metered::channel(64); + + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_validation(TestSubsystem5(tx_5)) + .replace_candidate_backing(TestSubsystem6(tx_6)); + + // start with two forks of different height. + let (overseer, mut handler) = Overseer::new( + vec![first_block, second_block], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + + let overseer_fut = overseer.run().fuse(); + pin_mut!(overseer_fut); + + let mut ss5_results = Vec::new(); + let mut ss6_results = Vec::new(); + + // this should stop work on both forks we started with earlier. + handler.block_finalized(third_block).await; + + let expected_heartbeats = vec![ + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: [ + ActivatedLeaf { + hash: first_block_hash, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + }, + ActivatedLeaf { + hash: second_block_hash, + number: 2, + span: Arc::new(jaeger::Span::Disabled), + }, + ].as_ref().into(), + ..Default::default() + }), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + deactivated: [first_block_hash, second_block_hash].as_ref().into(), + ..Default::default() + }), + OverseerSignal::BlockFinalized(third_block_hash, 3), + ]; + + loop { + select! { + res = overseer_fut => { + assert!(res.is_ok()); + break; + }, + res = rx_5.next() => { + if let Some(res) = res { + ss5_results.push(res); + } + } + res = rx_6.next() => { + if let Some(res) = res { + ss6_results.push(res); + } + } + complete => break, + } + + if ss5_results.len() == expected_heartbeats.len() && ss6_results.len() == expected_heartbeats.len() { + handler.stop().await; + } + } + + assert_eq!(ss5_results.len(), expected_heartbeats.len()); + assert_eq!(ss6_results.len(), expected_heartbeats.len()); + + // Notifications on finality for multiple blocks at once + // may be received in different orders. + for expected in expected_heartbeats { + assert!(ss5_results.contains(&expected)); + assert!(ss6_results.contains(&expected)); + } + }); + } + + #[test] + fn do_not_send_empty_leaves_update_on_block_finalization() { + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let imported_block = BlockInfo { + hash: Hash::random(), + parent_hash: Hash::random(), + number: 1, + }; + + let finalized_block = BlockInfo { + hash: Hash::random(), + parent_hash: Hash::random(), + number: 1, + }; + + let (tx_5, mut rx_5) = metered::channel(64); + + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_backing(TestSubsystem6(tx_5)); + + let (overseer, mut handler) = Overseer::new( + Vec::new(), + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + + let overseer_fut = overseer.run().fuse(); + pin_mut!(overseer_fut); + + let mut ss5_results = Vec::new(); + + handler.block_finalized(finalized_block.clone()).await; + handler.block_imported(imported_block.clone()).await; + + let expected_heartbeats = vec![ + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: [ + ActivatedLeaf { + hash: imported_block.hash, + number: imported_block.number, + span: Arc::new(jaeger::Span::Disabled) + } + ].as_ref().into(), + ..Default::default() + }), + OverseerSignal::BlockFinalized(finalized_block.hash, 1), + ]; + + loop { + select! { + res = overseer_fut => { + assert!(res.is_ok()); + break; + }, + res = rx_5.next() => { + if let Some(res) = dbg!(res) { + ss5_results.push(res); + } + } + } + + if ss5_results.len() == expected_heartbeats.len() { + handler.stop().await; + } + } + + assert_eq!(ss5_results.len(), expected_heartbeats.len()); + + for expected in expected_heartbeats { + assert!(ss5_results.contains(&expected)); + } + }); + } + + #[derive(Clone)] + struct CounterSubsystem { + stop_signals_received: Arc, + signals_received: Arc, + msgs_received: Arc, + } + + impl CounterSubsystem { + fn new( + stop_signals_received: Arc, + signals_received: Arc, + msgs_received: Arc, + ) -> Self { + Self { + stop_signals_received, + signals_received, + msgs_received, + } + } + } + + impl Subsystem for CounterSubsystem + where + C: SubsystemContext, + M: Send, + { + fn start(self, mut ctx: C) -> SpawnedSubsystem { + SpawnedSubsystem { + name: "counter-subsystem", + future: Box::pin(async move { + loop { + match ctx.try_recv().await { + Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => { + self.stop_signals_received.fetch_add(1, atomic::Ordering::SeqCst); + break; + }, + Ok(Some(FromOverseer::Signal(_))) => { + self.signals_received.fetch_add(1, atomic::Ordering::SeqCst); + continue; + }, + Ok(Some(FromOverseer::Communication { .. })) => { + self.msgs_received.fetch_add(1, atomic::Ordering::SeqCst); + continue; + }, + Err(_) => (), + _ => (), + } + pending!(); + } + + Ok(()) + }), + } + } + } + + fn test_candidate_validation_msg() -> CandidateValidationMessage { + let (sender, _) = oneshot::channel(); + let pov = Arc::new(PoV { block_data: BlockData(Vec::new()) }); + CandidateValidationMessage::ValidateFromChainState(Default::default(), pov, sender) + } + + fn test_candidate_backing_msg() -> CandidateBackingMessage { + let (sender, _) = oneshot::channel(); + CandidateBackingMessage::GetBackedCandidates(Default::default(), Vec::new(), sender) + } + + fn test_candidate_selection_msg() -> CandidateSelectionMessage { + CandidateSelectionMessage::default() + } + + fn test_chain_api_msg() -> ChainApiMessage { + let (sender, _) = oneshot::channel(); + ChainApiMessage::FinalizedBlockNumber(sender) + } + + fn test_collator_generation_msg() -> CollationGenerationMessage { + CollationGenerationMessage::Initialize(CollationGenerationConfig { + key: CollatorPair::generate().0, + collator: Box::new(|_, _| TestCollator.boxed()), + para_id: Default::default(), + }) + } + struct TestCollator; + + impl Future for TestCollator { + type Output = Option; + + fn poll(self: Pin<&mut Self>, _cx: &mut futures::task::Context) -> Poll { + panic!("at the Disco") + } + } + + impl Unpin for TestCollator {} + + fn test_collator_protocol_msg() -> CollatorProtocolMessage { + CollatorProtocolMessage::CollateOn(Default::default()) + } + + fn test_network_bridge_event() -> NetworkBridgeEvent { + NetworkBridgeEvent::PeerDisconnected(PeerId::random()) + } + + fn test_statement_distribution_msg() -> StatementDistributionMessage { + StatementDistributionMessage::NetworkBridgeUpdateV1(test_network_bridge_event()) + } + + fn test_availability_recovery_msg() -> AvailabilityRecoveryMessage { + let (sender, _) = oneshot::channel(); + AvailabilityRecoveryMessage::RecoverAvailableData( + Default::default(), + Default::default(), + None, + sender, + ) + } + + fn test_bitfield_distribution_msg() -> BitfieldDistributionMessage { + BitfieldDistributionMessage::NetworkBridgeUpdateV1(test_network_bridge_event()) + } + + fn test_provisioner_msg() -> ProvisionerMessage { + let (sender, _) = oneshot::channel(); + ProvisionerMessage::RequestInherentData(Default::default(), sender) + } + + fn test_runtime_api_msg() -> RuntimeApiMessage { + let (sender, _) = oneshot::channel(); + RuntimeApiMessage::Request(Default::default(), RuntimeApiRequest::Validators(sender)) + } + + fn test_availability_store_msg() -> AvailabilityStoreMessage { + let (sender, _) = oneshot::channel(); + AvailabilityStoreMessage::QueryAvailableData(CandidateHash(Default::default()), sender) + } + + fn test_network_bridge_msg() -> NetworkBridgeMessage { + NetworkBridgeMessage::ReportPeer(PeerId::random(), UnifiedReputationChange::BenefitMinor("")) + } + + fn test_approval_distribution_msg() -> ApprovalDistributionMessage { + ApprovalDistributionMessage::NewBlocks(Default::default()) + } + + fn test_approval_voting_msg() -> ApprovalVotingMessage { + let (sender, _) = oneshot::channel(); + ApprovalVotingMessage::ApprovedAncestor(Default::default(), 0, sender) + } + + // Checks that `stop`, `broadcast_signal` and `broadcast_message` are implemented correctly. + #[test] + fn overseer_all_subsystems_receive_signals_and_messages() { + const NUM_SUBSYSTEMS: usize = 18; + // -3 for BitfieldSigning, GossipSupport and AvailabilityDistribution + const NUM_SUBSYSTEMS_MESSAGED: usize = NUM_SUBSYSTEMS - 3; + + let spawner = sp_core::testing::TaskExecutor::new(); + executor::block_on(async move { + let stop_signals_received = Arc::new(atomic::AtomicUsize::new(0)); + let signals_received = Arc::new(atomic::AtomicUsize::new(0)); + let msgs_received = Arc::new(atomic::AtomicUsize::new(0)); + + let subsystem = CounterSubsystem::new( + stop_signals_received.clone(), + signals_received.clone(), + msgs_received.clone(), + ); + + let all_subsystems = AllSubsystems { + candidate_validation: subsystem.clone(), + candidate_backing: subsystem.clone(), + candidate_selection: subsystem.clone(), + collation_generation: subsystem.clone(), + collator_protocol: subsystem.clone(), + statement_distribution: subsystem.clone(), + availability_distribution: subsystem.clone(), + availability_recovery: subsystem.clone(), + bitfield_signing: subsystem.clone(), + bitfield_distribution: subsystem.clone(), + provisioner: subsystem.clone(), + runtime_api: subsystem.clone(), + availability_store: subsystem.clone(), + network_bridge: subsystem.clone(), + chain_api: subsystem.clone(), + approval_distribution: subsystem.clone(), + approval_voting: subsystem.clone(), + gossip_support: subsystem.clone(), + }; + let (overseer, mut handler) = Overseer::new( + vec![], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + let overseer_fut = overseer.run().fuse(); + + pin_mut!(overseer_fut); + + // send a signal to each subsystem + handler.block_imported(BlockInfo { + hash: Default::default(), + parent_hash: Default::default(), + number: Default::default(), + }).await; + + // send a msg to each subsystem + // except for BitfieldSigning and GossipSupport as the messages are not instantiable + handler.send_msg(AllMessages::CandidateValidation(test_candidate_validation_msg())).await; + handler.send_msg(AllMessages::CandidateBacking(test_candidate_backing_msg())).await; + handler.send_msg(AllMessages::CandidateSelection(test_candidate_selection_msg())).await; + handler.send_msg(AllMessages::CollationGeneration(test_collator_generation_msg())).await; + handler.send_msg(AllMessages::CollatorProtocol(test_collator_protocol_msg())).await; + handler.send_msg(AllMessages::StatementDistribution(test_statement_distribution_msg())).await; + handler.send_msg(AllMessages::AvailabilityRecovery(test_availability_recovery_msg())).await; + // handler.send_msg(AllMessages::BitfieldSigning(test_bitfield_signing_msg())).await; + // handler.send_msg(AllMessages::GossipSupport(test_bitfield_signing_msg())).await; + handler.send_msg(AllMessages::BitfieldDistribution(test_bitfield_distribution_msg())).await; + handler.send_msg(AllMessages::Provisioner(test_provisioner_msg())).await; + handler.send_msg(AllMessages::RuntimeApi(test_runtime_api_msg())).await; + handler.send_msg(AllMessages::AvailabilityStore(test_availability_store_msg())).await; + handler.send_msg(AllMessages::NetworkBridge(test_network_bridge_msg())).await; + handler.send_msg(AllMessages::ChainApi(test_chain_api_msg())).await; + handler.send_msg(AllMessages::ApprovalDistribution(test_approval_distribution_msg())).await; + handler.send_msg(AllMessages::ApprovalVoting(test_approval_voting_msg())).await; + + // Wait until all subsystems have received. Otherwise the messages might race against + // the conclude signal. + loop { + match (&mut overseer_fut).timeout(Duration::from_millis(100)).await { + None => { + let r = msgs_received.load(atomic::Ordering::SeqCst); + if r < NUM_SUBSYSTEMS_MESSAGED { + Delay::new(Duration::from_millis(100)).await; + } else if r > NUM_SUBSYSTEMS_MESSAGED { + panic!("too many messages received??"); + } else { + break + } + } + Some(_) => panic!("exited too early"), + } + } + + // send a stop signal to each subsystems + handler.stop().await; + + let res = overseer_fut.await; + assert_eq!(stop_signals_received.load(atomic::Ordering::SeqCst), NUM_SUBSYSTEMS); + assert_eq!(signals_received.load(atomic::Ordering::SeqCst), NUM_SUBSYSTEMS); + assert_eq!(msgs_received.load(atomic::Ordering::SeqCst), NUM_SUBSYSTEMS_MESSAGED); + + assert!(res.is_ok()); + }); + } + + #[test] + fn context_holds_onto_message_until_enough_signals_received() { + let (candidate_validation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (candidate_backing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (candidate_selection_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (statement_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (availability_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (availability_recovery_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (bitfield_signing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (bitfield_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (provisioner_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (runtime_api_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (availability_store_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (network_bridge_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (chain_api_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (collator_protocol_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (collation_generation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (approval_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (approval_voting_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (gossip_support_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + + let (candidate_validation_unbounded_tx, _) = metered::unbounded(); + let (candidate_backing_unbounded_tx, _) = metered::unbounded(); + let (candidate_selection_unbounded_tx, _) = metered::unbounded(); + let (statement_distribution_unbounded_tx, _) = metered::unbounded(); + let (availability_distribution_unbounded_tx, _) = metered::unbounded(); + let (availability_recovery_unbounded_tx, _) = metered::unbounded(); + let (bitfield_signing_unbounded_tx, _) = metered::unbounded(); + let (bitfield_distribution_unbounded_tx, _) = metered::unbounded(); + let (provisioner_unbounded_tx, _) = metered::unbounded(); + let (runtime_api_unbounded_tx, _) = metered::unbounded(); + let (availability_store_unbounded_tx, _) = metered::unbounded(); + let (network_bridge_unbounded_tx, _) = metered::unbounded(); + let (chain_api_unbounded_tx, _) = metered::unbounded(); + let (collator_protocol_unbounded_tx, _) = metered::unbounded(); + let (collation_generation_unbounded_tx, _) = metered::unbounded(); + let (approval_distribution_unbounded_tx, _) = metered::unbounded(); + let (approval_voting_unbounded_tx, _) = metered::unbounded(); + let (gossip_support_unbounded_tx, _) = metered::unbounded(); + + let channels_out = ChannelsOut { + candidate_validation: candidate_validation_bounded_tx.clone(), + candidate_backing: candidate_backing_bounded_tx.clone(), + candidate_selection: candidate_selection_bounded_tx.clone(), + statement_distribution: statement_distribution_bounded_tx.clone(), + availability_distribution: availability_distribution_bounded_tx.clone(), + availability_recovery: availability_recovery_bounded_tx.clone(), + bitfield_signing: bitfield_signing_bounded_tx.clone(), + bitfield_distribution: bitfield_distribution_bounded_tx.clone(), + provisioner: provisioner_bounded_tx.clone(), + runtime_api: runtime_api_bounded_tx.clone(), + availability_store: availability_store_bounded_tx.clone(), + network_bridge: network_bridge_bounded_tx.clone(), + chain_api: chain_api_bounded_tx.clone(), + collator_protocol: collator_protocol_bounded_tx.clone(), + collation_generation: collation_generation_bounded_tx.clone(), + approval_distribution: approval_distribution_bounded_tx.clone(), + approval_voting: approval_voting_bounded_tx.clone(), + gossip_support: gossip_support_bounded_tx.clone(), + + candidate_validation_unbounded: candidate_validation_unbounded_tx.clone(), + candidate_backing_unbounded: candidate_backing_unbounded_tx.clone(), + candidate_selection_unbounded: candidate_selection_unbounded_tx.clone(), + statement_distribution_unbounded: statement_distribution_unbounded_tx.clone(), + availability_distribution_unbounded: availability_distribution_unbounded_tx.clone(), + availability_recovery_unbounded: availability_recovery_unbounded_tx.clone(), + bitfield_signing_unbounded: bitfield_signing_unbounded_tx.clone(), + bitfield_distribution_unbounded: bitfield_distribution_unbounded_tx.clone(), + provisioner_unbounded: provisioner_unbounded_tx.clone(), + runtime_api_unbounded: runtime_api_unbounded_tx.clone(), + availability_store_unbounded: availability_store_unbounded_tx.clone(), + network_bridge_unbounded: network_bridge_unbounded_tx.clone(), + chain_api_unbounded: chain_api_unbounded_tx.clone(), + collator_protocol_unbounded: collator_protocol_unbounded_tx.clone(), + collation_generation_unbounded: collation_generation_unbounded_tx.clone(), + approval_distribution_unbounded: approval_distribution_unbounded_tx.clone(), + approval_voting_unbounded: approval_voting_unbounded_tx.clone(), + gossip_support_unbounded: gossip_support_unbounded_tx.clone(), + }; + + let (mut signal_tx, signal_rx) = metered::channel(CHANNEL_CAPACITY); + let (mut bounded_tx, bounded_rx) = metered::channel(CHANNEL_CAPACITY); + let (unbounded_tx, unbounded_rx) = metered::unbounded(); + let (to_overseer_tx, _to_overseer_rx) = metered::unbounded(); + + let mut ctx = OverseerSubsystemContext::<()>::new_unmetered( + signal_rx, + stream::select(bounded_rx, unbounded_rx), + channels_out, + to_overseer_tx, + ); + + assert_eq!(ctx.signals_received.load(), 0); + + let test_fut = async move { + signal_tx.send(OverseerSignal::Conclude).await.unwrap(); + assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Signal(OverseerSignal::Conclude)); + + assert_eq!(ctx.signals_received.load(), 1); + bounded_tx.send(MessagePacket { + signals_received: 2, + message: (), + }).await.unwrap(); + unbounded_tx.unbounded_send(MessagePacket { + signals_received: 2, + message: (), + }).unwrap(); + + match poll!(ctx.recv()) { + Poll::Pending => {} + Poll::Ready(_) => panic!("ready too early"), + }; + + assert!(ctx.pending_incoming.is_some()); + + signal_tx.send(OverseerSignal::Conclude).await.unwrap(); + assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Signal(OverseerSignal::Conclude)); + assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Communication { msg: () }); + assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Communication { msg: () }); + assert!(ctx.pending_incoming.is_none()); + }; + + futures::executor::block_on(test_fut); + } +} diff --git a/node/overseer/subsystems-gen/Cargo.toml b/node/overseer/subsystems-gen/Cargo.toml new file mode 100644 index 000000000000..2720663c293d --- /dev/null +++ b/node/overseer/subsystems-gen/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "polkadot-procmacro-overseer-subsystems-gen" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" +description = "Small proc macro to create mocking level iface type helpers" + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0.60", features = ["full", "extra-traits"] } +quote = "1.0.9" +proc-macro2 = "1.0.24" +assert_matches = "1.5.0" + +[dev-dependencies] +trybuild = "1.0.41" diff --git a/node/overseer/subsystems-gen/src/lib.rs b/node/overseer/subsystems-gen/src/lib.rs new file mode 100644 index 000000000000..c15d08bb04f4 --- /dev/null +++ b/node/overseer/subsystems-gen/src/lib.rs @@ -0,0 +1,179 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use std::collections::HashSet; + +use proc_macro2::TokenStream; +use quote::quote; + +use syn::{Error, GenericParam, Ident, Result, Type, parse2}; + +#[proc_macro_derive(AllSubsystemsGen)] +pub fn subsystems_gen(item: proc_macro::TokenStream) -> proc_macro::TokenStream { + let item: TokenStream = item.into(); + impl_subsystems_gen(item).unwrap_or_else(|err| err.to_compile_error()).into() +} + +fn impl_subsystems_gen(item: TokenStream) -> Result { + let span = proc_macro2::Span::call_site(); + let ds = parse2::(item.clone())?; + + match ds.fields { + syn::Fields::Named(named) => { + #[derive(Clone)] + struct NameTyTup { + field: Ident, + ty: Type, + } + let mut orig_generics = ds.generics; + // remove default types + orig_generics.params = orig_generics.params.into_iter().map(|mut generic| { + match generic { + GenericParam::Type(ref mut param) => { + param.eq_token = None; + param.default = None; + } + _ => {} + } + generic + }).collect(); + + // prepare a hashmap of generic type to member that uses it + let generic_types = orig_generics.params.iter().filter_map(|generic| { + if let GenericParam::Type(param) = generic { + Some(param.ident.clone()) + } else { + None + } + }).collect::>(); + + let strukt_ty = ds.ident; + + if generic_types.is_empty() { + return Err(Error::new(strukt_ty.span(), "struct must have at least one generic parameter.")) + } + + // collect all fields that exist, and all fields that are replaceable + let mut replacable_items = Vec::::with_capacity(64); + let mut all_fields = replacable_items.clone(); + + + let mut duplicate_generic_detection = HashSet::::with_capacity(64); + + for field in named.named { + let field_ident = field.ident.clone().ok_or_else(|| Error::new(span, "Member field must have a name."))?; + let ty = field.ty.clone(); + let ntt = NameTyTup { field: field_ident, ty }; + + replacable_items.push(ntt.clone()); + + + // assure every generic is used exactly once + let ty_ident = match field.ty { + Type::Path(path) => path.path.get_ident().cloned().ok_or_else(|| { + Error::new(proc_macro2::Span::call_site(), "Expected an identifier, but got a path.") + }), + _ => return Err(Error::new(proc_macro2::Span::call_site(), "Must be path.")) + }?; + + if generic_types.contains(&ty_ident) { + if let Some(previous) = duplicate_generic_detection.replace(ty_ident) { + return Err(Error::new(previous.span(), "Generic type parameters may only be used for exactly one field, but is used more than once.")) + } + } + + all_fields.push(ntt); + } + + + let msg = "Generated by #[derive(AllSubsystemsGen)] derive proc-macro."; + let mut additive = TokenStream::new(); + + // generate an impl of `fn replace_#name` + for NameTyTup { field: replacable_item, ty: replacable_item_ty } in replacable_items { + let keeper = all_fields.iter().filter(|ntt| ntt.field != replacable_item).map(|ntt| ntt.field.clone()); + let strukt_ty = strukt_ty.clone(); + let fname = Ident::new(&format!("replace_{}", replacable_item), span); + // adjust the generics such that the appropriate member type is replaced + let mut modified_generics = orig_generics.clone(); + modified_generics.params = modified_generics.params.into_iter().map(|mut generic| { + match generic { + GenericParam::Type(ref mut param) => { + param.eq_token = None; + param.default = None; + if match &replacable_item_ty { + Type::Path(path) => + path.path.get_ident().filter(|&ident| ident == ¶m.ident).is_some(), + _ => false + } { + param.ident = Ident::new("NEW", span); + } + } + _ => {} + } + generic + }).collect(); + + additive.extend(quote! { + impl #orig_generics #strukt_ty #orig_generics { + #[doc = #msg] + pub fn #fname < NEW > (self, replacement: NEW) -> #strukt_ty #modified_generics { + #strukt_ty :: #modified_generics { + #replacable_item: replacement, + #( + #keeper: self.#keeper, + )* + } + } + } + }); + } + + Ok(additive) + } + syn::Fields::Unit => Err(Error::new(span, "Must be a struct with named fields. Not an unit struct.")), + syn::Fields::Unnamed(_) => { + Err(Error::new(span, "Must be a struct with named fields. Not an unnamed fields struct.")) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic() { + let item = quote! { + pub struct AllSubsystems { + pub a: A, + pub beee: B, + pub dj: CD, + } + }; + + let output = impl_subsystems_gen(item).expect("Simple example always works. qed"); + println!("//generated:"); + println!("{}", output); + } + + #[test] + fn ui() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/err-*.rs"); + t.pass("tests/ui/ok-*.rs"); + } +} diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-enum.rs b/node/overseer/subsystems-gen/tests/ui/err-01-enum.rs new file mode 100644 index 000000000000..318636279ea5 --- /dev/null +++ b/node/overseer/subsystems-gen/tests/ui/err-01-enum.rs @@ -0,0 +1,13 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; + +#[derive(Clone, AllSubsystemsGen)] +enum AllSubsystems { + A(A), + B(B), +} + +fn main() { + let all = AllSubsystems::::A(0u8); +} diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-enum.stderr b/node/overseer/subsystems-gen/tests/ui/err-01-enum.stderr new file mode 100644 index 000000000000..5f61df1057cb --- /dev/null +++ b/node/overseer/subsystems-gen/tests/ui/err-01-enum.stderr @@ -0,0 +1,5 @@ +error: expected `struct` + --> $DIR/err-01-enum.rs:6:1 + | +6 | enum AllSubsystems { + | ^^^^ diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.rs b/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.rs new file mode 100644 index 000000000000..f89939d5c306 --- /dev/null +++ b/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.rs @@ -0,0 +1,17 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; + +#[derive(Clone, AllSubsystemsGen)] +struct AllSubsystems { + a: X, + b: X, +} + +fn main() { + let all = AllSubsystems:: { + a: 0_u16, + b: 1_u16, + }; + let _all = all.replace_a(77u8); +} diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.stderr b/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.stderr new file mode 100644 index 000000000000..23e1404ff822 --- /dev/null +++ b/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.stderr @@ -0,0 +1,14 @@ +error: Generic type parameters may only be used for exactly one field, but is used more than once. + --> $DIR/err-01-generic-used-twice.rs:7:5 + | +7 | a: X, + | ^ + +error[E0599]: no method named `replace_a` found for struct `AllSubsystems` in the current scope + --> $DIR/err-01-generic-used-twice.rs:16:17 + | +6 | struct AllSubsystems { + | ----------------------- method `replace_a` not found for this +... +16 | let _all = all.replace_a(77u8); + | ^^^^^^^^^ method not found in `AllSubsystems` diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-no-generic.rs b/node/overseer/subsystems-gen/tests/ui/err-01-no-generic.rs new file mode 100644 index 000000000000..0466eb444cd9 --- /dev/null +++ b/node/overseer/subsystems-gen/tests/ui/err-01-no-generic.rs @@ -0,0 +1,17 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; + +#[derive(Clone, AllSubsystemsGen)] +struct AllSubsystems { + a: f32, + b: u16, +} + +fn main() { + let all = AllSubsystems { + a: 0_f32, + b: 1_u16, + }; + let _all = all.replace_a(77u8); +} diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-no-generics.stderr b/node/overseer/subsystems-gen/tests/ui/err-01-no-generics.stderr new file mode 100644 index 000000000000..1de880ae433c --- /dev/null +++ b/node/overseer/subsystems-gen/tests/ui/err-01-no-generics.stderr @@ -0,0 +1,14 @@ +error: Generic type parameters may only be used once have at least one generic parameter. + --> $DIR/err-01-no-generics.rs:7:5 + | +7 | a: X, + | ^ + +error[E0599]: no method named `replace_a` found for struct `AllSubsystems` in the current scope + --> $DIR/err-01-no-generics.rs:16:17 + | +6 | struct AllSubsystems { + | ----------------------- method `replace_a` not found for this +... +16 | let _all = all.replace_a(77u8); + | ^^^^^^^^^ method not found in `AllSubsystems` diff --git a/node/overseer/subsystems-gen/tests/ui/ok-01-w-generics.rs b/node/overseer/subsystems-gen/tests/ui/ok-01-w-generics.rs new file mode 100644 index 000000000000..1519990a0a55 --- /dev/null +++ b/node/overseer/subsystems-gen/tests/ui/ok-01-w-generics.rs @@ -0,0 +1,17 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; + +#[derive(Clone, AllSubsystemsGen)] +struct AllSubsystems { + a: A, + b: B, +} + +fn main() { + let all = AllSubsystems:: { + a: 0u8, + b: 1u16, + }; + let _all: AllSubsystems<_,_> = all.replace_a::(777_777u32); +} diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index ed0e8192c1aa..fde49cacd831 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -420,7 +420,7 @@ fn new_partial( } #[cfg(feature = "full-node")] -fn real_overseer( +fn real_overseer( leaves: impl IntoIterator, keystore: Arc, runtime_client: Arc, @@ -434,12 +434,13 @@ fn real_overseer spawner: Spawner, is_collator: IsCollator, candidate_validation_config: CandidateValidationConfig, -) -> Result<(Overseer, SubsystemSelection>, OverseerHandler), Error> +) -> Result<(Overseer>, OverseerHandler), Error> where RuntimeClient: 'static + ProvideRuntimeApi + HeaderBackend + AuxStore, RuntimeClient::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, Spawner: 'static + SpawnNamed + Clone + Unpin, { + use polkadot_node_subsystem_util::metrics::Metrics; use polkadot_availability_distribution::AvailabilityDistributionSubsystem; use polkadot_node_core_av_store::AvailabilityStoreSubsystem; @@ -674,7 +675,7 @@ where /// This is an advanced feature and not recommended for general use. Generally, `build_full` is /// a better choice. #[cfg(feature = "full-node")] -pub fn new_full( +pub fn new_full( mut config: Configuration, is_collator: IsCollator, grandpa_pause: Option<(u32, u32)>, @@ -863,7 +864,7 @@ pub fn new_full( .and_then(move |k| authority_discovery_service.map(|a| (a, k))); let overseer_handler = if let Some((authority_discovery_service, keystore)) = maybe_params { - let (overseer, overseer_handler) = real_overseer::<_,_,SubsystemSelection>( + let (overseer, overseer_handler) = real_overseer( active_leaves, keystore, overseer_client.clone(), diff --git a/node/subsystem/Cargo.toml b/node/subsystem/Cargo.toml index 7b2e8e6055ce..590af20343e8 100644 --- a/node/subsystem/Cargo.toml +++ b/node/subsystem/Cargo.toml @@ -22,6 +22,7 @@ polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-primitives = { path = "../../primitives" } polkadot-statement-table = { path = "../../statement-table" } polkadot-node-jaeger = { path = "../jaeger" } +polkadot-procmacro-subsystem-dispatch-gen = { path = "dispatch-gen" } sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } smallvec = "1.6.1" sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/subsystem/dispatch-gen/Cargo.toml b/node/subsystem/dispatch-gen/Cargo.toml new file mode 100644 index 000000000000..09de1362c92c --- /dev/null +++ b/node/subsystem/dispatch-gen/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "polkadot-procmacro-subsystem-dispatch-gen" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" +description = "Small proc macro to create the distribution code for network events" + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0.60", features = ["full"] } +quote = "1.0.9" +proc-macro2 = "1.0.24" +assert_matches = "1.5.0" + +[dev-dependencies] +trybuild = "1.0.41" diff --git a/node/subsystem/dispatch-gen/src/lib.rs b/node/subsystem/dispatch-gen/src/lib.rs new file mode 100644 index 000000000000..737712639cff --- /dev/null +++ b/node/subsystem/dispatch-gen/src/lib.rs @@ -0,0 +1,208 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use proc_macro2::TokenStream; +use quote::{quote, ToTokens}; +use std::fmt; +use syn::{parse2, Error, Fields, FieldsNamed, FieldsUnnamed, Ident, ItemEnum, Path, Result, Type, Variant}; + +#[proc_macro_attribute] +pub fn subsystem_dispatch_gen(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream { + let attr: TokenStream = attr.into(); + let item: TokenStream = item.into(); + let mut backup = item.clone(); + impl_subsystem_dispatch_gen(attr.into(), item).unwrap_or_else(|err| { + backup.extend(err.to_compile_error()); + backup + }).into() +} + +/// An enum variant without base type. +#[derive(Clone)] +struct EnumVariantDispatchWithTy { + // enum ty name + ty: Ident, + // variant + variant: EnumVariantDispatch, +} + +impl fmt::Debug for EnumVariantDispatchWithTy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}::{:?}", self.ty, self.variant) + } +} + +impl ToTokens for EnumVariantDispatchWithTy { + fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { + if let Some(inner) = &self.variant.inner { + let enum_name = &self.ty; + let variant_name = &self.variant.name; + + let quoted = quote! { + #enum_name::#variant_name(#inner::from(event)) + }; + quoted.to_tokens(tokens); + } + } +} + +/// An enum variant without the base type, contains the relevant inner type. +#[derive(Clone)] +struct EnumVariantDispatch { + /// variant name + name: Ident, + /// The inner type for which a `From::from` impl is anticipated from the input type. + /// No code will be generated for this enum variant if `inner` is `None`. + inner: Option, +} + +impl fmt::Debug for EnumVariantDispatch { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}(..)", self.name) + } +} + +fn prepare_enum_variant(variant: &mut Variant) -> Result { + let skip = variant.attrs.iter().find(|attr| attr.path.is_ident("skip")).is_some(); + variant.attrs = variant.attrs.iter().filter(|attr| !attr.path.is_ident("skip")).cloned().collect::>(); + + let variant = variant.clone(); + let span = variant.ident.span(); + let inner = match variant.fields.clone() { + // look for one called inner + Fields::Named(FieldsNamed { brace_token: _, named }) if !skip => named + .iter() + .find_map( + |field| { + if let Some(ident) = &field.ident { + if ident == "inner" { + return Some(Some(field.ty.clone())) + } + } + None + }, + ) + .ok_or_else(|| { + Error::new(span, "To dispatch with struct enum variant, one element must named `inner`") + })?, + + // technically, if it has no inner types we cound not require the #[skip] annotation, but better make it consistent + Fields::Unnamed(FieldsUnnamed { paren_token: _, unnamed }) if !skip => unnamed + .first() + .map(|field| Some(field.ty.clone())) + .ok_or_else(|| Error::new(span, "Must be annotated with skip, even if no inner types exist."))?, + _ if skip => None, + Fields::Unit => { + return Err(Error::new( + span, + "Must be annotated with #[skip].", + )) + } + Fields::Unnamed(_) => { + return Err(Error::new( + span, + "Must be annotated with #[skip] or have in `inner` element which impls `From<_>`.", + )) + } + Fields::Named(_) => { + return Err(Error::new( + span, + "Must be annotated with #[skip] or the first wrapped type must impl `From<_>`.", + )) + } + }; + + Ok(EnumVariantDispatch { name: variant.ident, inner }) +} + +fn impl_subsystem_dispatch_gen(attr: TokenStream, item: TokenStream) -> Result { + let event_ty = parse2::(attr)?; + + let mut ie = parse2::(item)?; + + let message_enum = ie.ident.clone(); + let variants = ie.variants.iter_mut().try_fold(Vec::::new(), |mut acc, variant| { + let variant = prepare_enum_variant(variant)?; + if variant.inner.is_some() { + acc.push(EnumVariantDispatchWithTy { ty: message_enum.clone(), variant }) + } + Ok::<_, syn::Error>(acc) + })?; + + let mut orig = ie.to_token_stream(); + + let msg = "Generated by #[subsystem_dispatch_gen] proc-macro."; + + orig.extend(quote! { + impl #message_enum { + #[doc = #msg] + pub fn dispatch_iter(event: #event_ty) -> impl Iterator + Send { + let mut iter = None.into_iter(); + + #( + let mut iter = iter.chain(std::iter::once(event.focus().ok().map(|event| { + #variants + }))); + )* + iter.filter_map(|x| x) + } + } + }); + Ok(orig) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic() { + let attr = quote! { + NetEvent + }; + + let item = quote! { + /// Documentation. + #[derive(Clone)] + enum AllMessages { + + Sub1(Inner1), + + #[skip] + /// D3 + Sub3, + + /// D4 + #[skip] + Sub4(Inner2), + + /// D2 + Sub2(Inner2), + } + }; + + let output = impl_subsystem_dispatch_gen(attr, item).expect("Simple example always works. qed"); + println!("//generated:"); + println!("{}", output); + } + + #[test] + fn ui() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/err-*.rs"); + t.pass("tests/ui/ok-*.rs"); + } +} diff --git a/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.rs b/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.rs new file mode 100644 index 000000000000..7248a7181e49 --- /dev/null +++ b/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.rs @@ -0,0 +1,37 @@ +#![allow(dead_code)] + +use polkadot_procmacro_subsystem_dispatch_gen::subsystem_dispatch_gen; + +/// The event type in question. +#[derive(Clone, Copy)] +enum Event { + Smth, + Else, +} + +impl Event { + fn focus(&self) -> std::result::Result { + unimplemented!("foo") + } +} + +/// This should have a `From` impl but does not. +#[derive(Clone)] +enum Inner { + Foo, + Bar(Event), +} + +#[subsystem_dispatch_gen(Event)] +#[derive(Clone)] +enum AllMessages { + /// Foo + Vvvvvv(Inner), + + /// Missing a `#[skip]` annotation + Uuuuu, +} + +fn main() { + let _x = AllMessages::dispatch_iter(Event::Else); +} diff --git a/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.stderr b/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.stderr new file mode 100644 index 000000000000..855521d2c4ef --- /dev/null +++ b/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.stderr @@ -0,0 +1,14 @@ +error: Must be annotated with #[skip]. + --> $DIR/err-01-missing-skip.rs:32:5 + | +32 | Uuuuu, + | ^^^^^ + +error[E0599]: no variant or associated item named `dispatch_iter` found for enum `AllMessages` in the current scope + --> $DIR/err-01-missing-skip.rs:36:27 + | +27 | enum AllMessages { + | ---------------- variant or associated item `dispatch_iter` not found here +... +36 | let _x = AllMessages::dispatch_iter(Event::Else); + | ^^^^^^^^^^^^^ variant or associated item not found in `AllMessages` diff --git a/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.rs b/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.rs new file mode 100644 index 000000000000..a7abef2c8709 --- /dev/null +++ b/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.rs @@ -0,0 +1,41 @@ +#![allow(dead_code)] + +use polkadot_procmacro_subsystem_dispatch_gen::subsystem_dispatch_gen; + +/// The event type in question. +#[derive(Clone, Copy, Debug)] +enum Event { + Smth, + Else, +} + +impl Event { + fn focus(&self) -> std::result::Result { + Ok(Intermediate(self.clone())) + } +} + +#[derive(Debug, Clone)] +struct Intermediate(Event); + + +/// This should have a `From` impl but does not. +#[derive(Debug, Clone)] +enum Inner { + Foo, + Bar(Intermediate), +} + +#[subsystem_dispatch_gen(Event)] +#[derive(Clone)] +enum AllMessages { + /// Foo + Vvvvvv(Inner), + + #[skip] + Uuuuu, +} + +fn main() { + let _x = AllMessages::dispatch_iter(Event::Else); +} diff --git a/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.stderr b/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.stderr new file mode 100644 index 000000000000..bf82201a7e40 --- /dev/null +++ b/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.stderr @@ -0,0 +1,10 @@ +error[E0308]: mismatched types + --> $DIR/err-02-missing-from.rs:29:1 + | +29 | #[subsystem_dispatch_gen(Event)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | | + | expected enum `Inner`, found struct `Intermediate` + | help: try using a variant of the expected enum: `Inner::Bar(#[subsystem_dispatch_gen(Event)])` + | + = note: this error originates in an attribute macro (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/node/subsystem/dispatch-gen/tests/ui/ok-01-with-intermediate.rs b/node/subsystem/dispatch-gen/tests/ui/ok-01-with-intermediate.rs new file mode 100644 index 000000000000..b160bf9ce1c1 --- /dev/null +++ b/node/subsystem/dispatch-gen/tests/ui/ok-01-with-intermediate.rs @@ -0,0 +1,48 @@ +#![allow(dead_code)] + +use polkadot_procmacro_subsystem_dispatch_gen::subsystem_dispatch_gen; + +/// The event type in question. +#[derive(Clone, Copy, Debug)] +enum Event { + Smth, + Else, +} + +impl Event { + fn focus(&self) -> std::result::Result { + Ok(Intermediate(self.clone())) + } +} + + +#[derive(Debug, Clone)] +struct Intermediate(Event); + + +/// This should have a `From` impl but does not. +#[derive(Clone, Debug)] +enum Inner { + Foo, + Bar(Intermediate), +} + +impl From for Inner { + fn from(src: Intermediate) -> Self { + Inner::Bar(src) + } +} + +#[subsystem_dispatch_gen(Event)] +#[derive(Clone)] +enum AllMessages { + /// Foo + Vvvvvv(Inner), + + #[skip] + Uuuuu, +} + +fn main() { + let _x = AllMessages::dispatch_iter(Event::Else); +} diff --git a/node/subsystem/src/errors.rs b/node/subsystem/src/errors.rs index 6b7832bf1e7a..acd33cff1dfb 100644 --- a/node/subsystem/src/errors.rs +++ b/node/subsystem/src/errors.rs @@ -77,57 +77,3 @@ impl std::fmt::Display for RecoveryError { } impl std::error::Error for RecoveryError {} - - - - -/// An error type that describes faults that may happen -/// -/// These are: -/// * Channels being closed -/// * Subsystems dying when they are not expected to -/// * Subsystems not dying when they are told to die -/// * etc. -#[derive(thiserror::Error, Debug)] -#[allow(missing_docs)] -pub enum SubsystemError { - #[error(transparent)] - NotifyCancellation(#[from] oneshot::Canceled), - - #[error(transparent)] - QueueError(#[from] mpsc::SendError), - - #[error(transparent)] - TaskSpawn(#[from] futures::task::SpawnError), - - #[error(transparent)] - Infallible(#[from] std::convert::Infallible), - - #[error(transparent)] - Prometheus(#[from] substrate_prometheus_endpoint::PrometheusError), - - #[error(transparent)] - Jaeger(#[from] JaegerError), - - #[error("Failed to {0}")] - Context(String), - - #[error("Subsystem stalled: {0}")] - SubsystemStalled(&'static str), - - /// Per origin (or subsystem) annotations to wrap an error. - #[error("Error originated in {origin}")] - FromOrigin { - /// An additional anotation tag for the origin of `source`. - origin: &'static str, - /// The wrapped error. Marked as source for tracking the error chain. - #[source] source: Box - }, -} - -impl SubsystemError { - /// Adds a `str` as `origin` to the given error `err`. - pub fn with_origin(origin: &'static str, err: E) -> Self { - Self::FromOrigin { origin, source: Box::new(err) } - } -} diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 4dd30c2aef75..3cc4894a493e 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -39,7 +39,6 @@ pub use polkadot_node_jaeger as jaeger; pub use jaeger::*; use self::messages::AllMessages; -pub use self::errors::SubsystemError; /// How many slots are stack-reserved for active leaves updates /// @@ -133,7 +132,7 @@ pub enum OverseerSignal { /// /// It is generic over over the message type `M` that a particular `Subsystem` may use. #[derive(Debug)] -pub enum FromOverseer { +pub enum FromOverseer { /// Signal from the `Overseer`. Signal(OverseerSignal), @@ -145,6 +144,57 @@ pub enum FromOverseer { } +/// An error type that describes faults that may happen +/// +/// These are: +/// * Channels being closed +/// * Subsystems dying when they are not expected to +/// * Subsystems not dying when they are told to die +/// * etc. +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum SubsystemError { + #[error(transparent)] + NotifyCancellation(#[from] oneshot::Canceled), + + #[error(transparent)] + QueueError(#[from] mpsc::SendError), + + #[error(transparent)] + TaskSpawn(#[from] futures::task::SpawnError), + + #[error(transparent)] + Infallible(#[from] std::convert::Infallible), + + #[error(transparent)] + Prometheus(#[from] substrate_prometheus_endpoint::PrometheusError), + + #[error(transparent)] + Jaeger(#[from] JaegerError), + + #[error("Failed to {0}")] + Context(String), + + #[error("Subsystem stalled: {0}")] + SubsystemStalled(&'static str), + + /// Per origin (or subsystem) annotations to wrap an error. + #[error("Error originated in {origin}")] + FromOrigin { + /// An additional anotation tag for the origin of `source`. + origin: &'static str, + /// The wrapped error. Marked as source for tracking the error chain. + #[source] source: Box + }, +} + +impl SubsystemError { + /// Adds a `str` as `origin` to the given error `err`. + pub fn with_origin(origin: &'static str, err: E) -> Self { + Self::FromOrigin { origin, source: Box::new(err) } + } +} + /// An asynchronous subsystem task.. /// /// In essence it's just a newtype wrapping a `BoxFuture`. diff --git a/node/subsystem/src/messages.rs b/node/subsystem/src/messages.rs index 6728f2fd7db3..a0d274ec6fd4 100644 --- a/node/subsystem/src/messages.rs +++ b/node/subsystem/src/messages.rs @@ -674,6 +674,63 @@ pub enum ApprovalDistributionMessage { pub enum GossipSupportMessage { } +/// A message type tying together all message types that are used across Subsystems. +#[subsystem_dispatch_gen(NetworkBridgeEvent)] +#[derive(Debug, derive_more::From)] +pub enum AllMessages { + /// Message for the validation subsystem. + #[skip] + CandidateValidation(CandidateValidationMessage), + /// Message for the candidate backing subsystem. + #[skip] + CandidateBacking(CandidateBackingMessage), + /// Message for the candidate selection subsystem. + #[skip] + CandidateSelection(CandidateSelectionMessage), + /// Message for the Chain API subsystem. + #[skip] + ChainApi(ChainApiMessage), + /// Message for the Collator Protocol subsystem. + #[skip] + CollatorProtocol(CollatorProtocolMessage), + /// Message for the statement distribution subsystem. + StatementDistribution(StatementDistributionMessage), + /// Message for the availability distribution subsystem. + #[skip] + AvailabilityDistribution(AvailabilityDistributionMessage), + /// Message for the availability recovery subsystem. + #[skip] + AvailabilityRecovery(AvailabilityRecoveryMessage), + /// Message for the bitfield distribution subsystem. + BitfieldDistribution(BitfieldDistributionMessage), + /// Message for the bitfield signing subsystem. + #[skip] + BitfieldSigning(BitfieldSigningMessage), + /// Message for the Provisioner subsystem. + #[skip] + Provisioner(ProvisionerMessage), + /// Message for the Runtime API subsystem. + #[skip] + RuntimeApi(RuntimeApiMessage), + /// Message for the availability store subsystem. + #[skip] + AvailabilityStore(AvailabilityStoreMessage), + /// Message for the network bridge subsystem. + #[skip] + NetworkBridge(NetworkBridgeMessage), + /// Message for the Collation Generation subsystem. + #[skip] + CollationGeneration(CollationGenerationMessage), + /// Message for the Approval Voting subsystem. + #[skip] + ApprovalVoting(ApprovalVotingMessage), + /// Message for the Approval Distribution subsystem. + ApprovalDistribution(ApprovalDistributionMessage), + /// Message for the Gossip Support subsystem. + #[skip] + GossipSupport(GossipSupportMessage), +} + impl From> for AvailabilityDistributionMessage { fn from(req: IncomingRequest) -> Self { Self::PoVFetchingRequest(req) From cc4544df036d5985facee1c70762b42991b0b7d4 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 12 May 2021 18:14:28 +0200 Subject: [PATCH 029/161] fixins --- Cargo.lock | 27 +- node/overseer/overseer-gen/Cargo.toml | 4 +- node/overseer/overseer-gen/examples/dummy.rs | 36 ++ .../proc-macro/src/impl_channels_out.rs | 20 +- .../proc-macro/src/impl_dispatch.rs | 2 +- .../proc-macro/src/impl_message_wrapper.rs | 6 +- .../proc-macro/src/impl_overseer.rs | 351 ++++++++++++---- .../proc-macro/src/impl_replace.rs | 13 +- .../overseer-gen/proc-macro/src/inc.rs | 9 - .../overseer-gen/proc-macro/src/lib.rs | 88 ++-- .../overseer-gen/proc-macro/src/parse.rs | 379 ------------------ .../overseer-gen/proc-macro/src/parse_attr.rs | 174 ++++++++ .../proc-macro/src/parse_struct.rs | 284 +++++++++++++ .../overseer-gen/proc-macro/src/tests.rs | 58 ++- node/overseer/overseer-gen/src/lib.rs | 91 ++++- node/overseer/overseer-gen/src/tests.rs | 3 +- .../tests/ui/err-01-duplicate-consumer.rs | 37 ++ .../tests/ui/err-01-replace-w-inadequate.rs | 38 -- .../tests/ui/err-03-subsys-twice.rs | 43 +- .../overseer-gen/tests/ui/ok-01-boring.rs | 28 +- .../overseer-gen/tests/ui/ok-02-w-generic.rs | 28 +- .../tests/ui/ok-03-no-dispatch.rs | 28 +- .../overseer-gen/tests/ui/ok-04-blocking.rs | 34 ++ 23 files changed, 1120 insertions(+), 661 deletions(-) create mode 100644 node/overseer/overseer-gen/examples/dummy.rs delete mode 100644 node/overseer/overseer-gen/proc-macro/src/inc.rs delete mode 100644 node/overseer/overseer-gen/proc-macro/src/parse.rs create mode 100644 node/overseer/overseer-gen/proc-macro/src/parse_attr.rs create mode 100644 node/overseer/overseer-gen/proc-macro/src/parse_struct.rs create mode 100644 node/overseer/overseer-gen/tests/ui/err-01-duplicate-consumer.rs delete mode 100644 node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs create mode 100644 node/overseer/overseer-gen/tests/ui/ok-04-blocking.rs diff --git a/Cargo.lock b/Cargo.lock index fb014bedec01..dbc0736c8f2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6173,6 +6173,7 @@ dependencies = [ "polkadot-node-primitives", "polkadot-node-subsystem-test-helpers", "polkadot-primitives", + "polkadot-procmacro-subsystem-dispatch-gen", "polkadot-statement-table", "sc-network", "smallvec 1.6.1", @@ -6250,8 +6251,8 @@ dependencies = [ "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-util", - "polkadot-overseer-gen", "polkadot-primitives", + "polkadot-procmacro-overseer-subsystems-gen", "sc-client-api", "sp-api", "sp-core", @@ -6262,10 +6263,12 @@ dependencies = [ name = "polkadot-overseer-gen" version = "0.1.0" dependencies = [ + "async-trait", "futures 0.3.14", "metered-channel", "polkadot-overseer-gen-proc-macro", "sp-core", + "thiserror", "tracing", "trybuild", ] @@ -6326,6 +6329,28 @@ dependencies = [ "thiserror", ] +[[package]] +name = "polkadot-procmacro-overseer-subsystems-gen" +version = "0.1.0" +dependencies = [ + "assert_matches", + "proc-macro2", + "quote", + "syn", + "trybuild", +] + +[[package]] +name = "polkadot-procmacro-subsystem-dispatch-gen" +version = "0.1.0" +dependencies = [ + "assert_matches", + "proc-macro2", + "quote", + "syn", + "trybuild", +] + [[package]] name = "polkadot-rpc" version = "0.9.1" diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index 317ef0d1ce84..9de6a41a686b 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -8,11 +8,13 @@ description = "Generate an overseer including builder pattern and message wrappe [dependencies] tracing = "0.1" futures = "0.3" - +async-trait = "0.1" +thiserror = "1" overseer-gen-proc-macro = { package = "polkadot-overseer-gen-proc-macro", path = "./proc-macro" } metered = { package = "metered-channel", path = "../../metered-channel" } # trait SpawnNamed sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } + [dev-dependencies] trybuild = "1.0.41" diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs new file mode 100644 index 000000000000..021a2bbaca45 --- /dev/null +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -0,0 +1,36 @@ +//! A dummy to be used with cargo expand + +use polkadot_overseer_gen::*; + +#[derive(Default)] +struct AwesomeSubSys; + +struct SigSigSig; + +struct Event; + +struct Yikes; + +impl std::error::Error for Yikes {} + +#[derive(Clone)] +struct MsgStrukt(u8); + +#[overlord(signal=SigSigSig, event=Event, error=Yikes, gen=AllMessages)] +struct Overseer { + #[subsystem(MsgStrukt)] + sub0: AwesomeSubSys, +} + +#[derive(Debug, Clone)] +struct DummySpawner; + +struct DummyCtx; + +fn main() { + let overseer = Overseer::<_,_>::builder() + .sub0(AwesomeSubSys::default()) + .i_like_pie(std::f64::consts::PI) + .spawner(DummySpawner) + .build(|| -> DummyCtx { DummyCtx } ); +} diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index e319e649e6b6..f7af13faf0ec 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -19,14 +19,15 @@ pub(crate) fn impl_channels_out_struct( let ts = quote! { pub struct ChannelsOut { #( - pub #channel_name: ::metered::MeteredSender>, + pub #channel_name: ::polkadot_overseer_gen::metered::MeteredSender>, )* #( - pub #channel_name_unbounded: ::metered::UnboundedMeteredSender>, + pub #channel_name_unbounded: ::polkadot_overseer_gen::metered::UnboundedMeteredSender>, )* } impl ChannelsOut { + /// Send a message via a bounded channel. pub async fn send_and_log_error( &mut self, signals_received: usize, @@ -34,22 +35,23 @@ pub(crate) fn impl_channels_out_struct( ) { let res = match message { #( - #message_wrapper :: #consumes (msg) => { + #message_wrapper :: #consumes ( message ) => { self. #channel_name .send( - make_packet(signals_received, msg) + ::polkadot_overseer_gen::make_packet(signals_received, message) ).await }, )* }; if res.is_err() { - tracing::debug!( + ::polkadot_overseer_gen::tracing::debug!( target: LOG_TARGET, "Failed to send a message to another subsystem", ); } } + /// Send a message to another subsystem via an unbounded channel. pub fn send_unbounded_and_log_error( &self, signals_received: usize, @@ -57,16 +59,16 @@ pub(crate) fn impl_channels_out_struct( ) { let res = match message { #( - #message_wrapper :: #consumes (msg) => { + #message_wrapper :: #consumes (message) => { self. #channel_name_unbounded .send( - make_packet(signals_received, msg) - ).await + make_packet(signals_received, message) + ) }, )* }; if res.is_err() { - tracing::debug!( + ::polkadot_overseer_gen::tracing::debug!( target: LOG_TARGET, "Failed to send a message to another subsystem", ); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index 33ba9aa4132a..d158bdbb4b44 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -36,7 +36,7 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { #( let mut iter = iter.chain( - std::iter::once( + ::std::iter::once( event.focus().ok().map(|event| { #message_wrapper :: #dispatchable ( #dispatchable :: from( event ) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 6bc6b8cde3e2..7911e48dae46 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -22,9 +22,9 @@ pub(crate) fn impl_message_wrapper_enum( } #( - impl ::std::from::From<#consumes> for #message_wrapper { + impl ::std::convert::From<#consumes> for #message_wrapper { fn from(src: #consumes) -> Self { - #message_wrapper :: #consumes ( #consumes ) + #message_wrapper :: #consumes ( src ) } } )* @@ -35,7 +35,7 @@ pub(crate) fn impl_message_wrapper_enum( signals_received: SignalsReceived, } - #[async_trait::async_trait] + #[::polkadot_overseer_gen::async_trait] impl SubsystemSender for OverseerSubsystemSender { async fn send_message(&mut self, msg: #message_wrapper) { self.channels.send_and_log_error(self.signals_received.load(), msg).await; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index f5745d5ab571..ba9bcea78201 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -7,7 +7,7 @@ pub(crate) fn impl_overseer_struct( info: &OverseerInfo, ) -> Result { let message_wrapper = &info.message_wrapper.clone(); - let overseer_name = info.overseer_name.clone(); + let overseer_name = dbg!(info.overseer_name.clone()); let subsystem_name = &info.subsystem_names(); let subsystem_generic_ty = &info.subsystem_generic_types(); @@ -24,19 +24,23 @@ pub(crate) fn impl_overseer_struct( let where_clause = quote! { where Ctx: SubsystemContext, - S: SpawnNamed, + S: ::polkadot_overseer_gen::SpawnNamed, #( #subsystem_generic_ty : Subsystem, )* }; let consumes = &info.consumes(); + let signal_ty = &info.extern_event_ty; + let message_channel_capacity = info.message_channel_capacity; let signal_channel_capacity = info.signal_channel_capacity; + let log_target = syn::LitStr::new(overseer_name.to_string().to_lowercase().as_str(), overseer_name.span()); + let mut ts = quote! { const CHANNEL_CAPACITY: usize = #message_channel_capacity; const SIGNAL_CHANNEL_CAPACITY: usize = #signal_channel_capacity; - + const LOG_TARGET: &'static str = #log_target; pub struct #overseer_name #generics { // Subsystem instances. #( @@ -52,35 +56,22 @@ pub(crate) fn impl_overseer_struct( spawner: S, /// The set of running subsystems. - running_subsystems: FuturesUnordered>>, + running_subsystems: ::polkadot_overseer_gen::FuturesUnordered>>, /// Gather running subsystems' outbound streams into one. - to_overseer_rx: Fuse>, + to_overseer_rx: ::polkadot_overseer_gen::Fuse>, /// Events that are sent to the overseer from the outside world. - events_rx: metered::MeteredReceiver, + events_rx: ::polkadot_overseer_gen::metered::MeteredReceiver, } impl #generics #overseer_name #generics #where_clause { pub async fn stop(mut self) { - #( - let _ = self. #subsystem_name .send_signal(OverseerSignal::Conclude).await; - )* - loop { - select! { - _ = self.running_subsystems.next() => { - if self.running_subsystems.is_empty() { - break; - } - }, - _ = stop_delay => break, - complete => break, - } - } + unimplemented!("Stopping is not yet implemented") } } - pub async fn broadcast_signal(&mut self, signal: OverseerSignal) -> SubsystemResult<()> { + pub async fn broadcast_signal(&mut self, signal: #signal_ty) -> SubsystemResult<()> { #( self. #subsystem_name .send_signal(signal.clone()).await; )* @@ -101,28 +92,9 @@ pub(crate) fn impl_overseer_struct( ts.extend(impl_builder(info)?); ts.extend(impl_subsystem_instance(info)?); - - Ok(ts) -} - - -pub(crate) fn impl_subsystem_instance(info: &OverseerInfo) -> Result { - let extern_signal_ty = &info.extern_signal_ty; - - let ts = quote::quote! { - /// A running instance of some [`Subsystem`]. - /// - /// [`Subsystem`]: trait.Subsystem.html - /// - /// `M` here is the inner message type, and _not_ the generated `enum AllMessages`. - pub struct SubsystemInstance { - tx_signal: metered::MeteredSender< #extern_signal_ty >, - tx_bounded: metered::MeteredSender>, - meters: SubsystemMeters, - signals_received: usize, - name: &'static str, - } - }; + ts.extend(impl_overseen_subsystem(info)?); + ts.extend(impl_trait_subsystem(info)?); + ts.extend(impl_trait_subsystem_sender(info)?); Ok(ts) } @@ -167,12 +139,14 @@ pub(crate) fn impl_builder( let where_clause = quote! { where Ctx: SubsystemContext, - S: SpawnNamed, + S: ::polkadot_overseer_gen::SpawnNamed, #( #subsystem_generic_ty : Subsystem, )* }; let consumes = &info.consumes(); let message_wrapper = &info.message_wrapper; + let event = &info.extern_event_ty; + let signal = &info.extern_signal_ty; let ts = quote! { @@ -182,6 +156,9 @@ pub(crate) fn impl_builder( } } + + pub type #handler = ::polkadot_overseer_gen::metered::UnboundedMeteredSender< #event >; + #[derive(Debug, Clone, Default)] struct #builder #builder_generics { #( @@ -195,66 +172,67 @@ pub(crate) fn impl_builder( impl #builder_generics #builder #builder_generics #where_clause { #( - fn #subsystem_name (mut self, subsystem: #subsystem_generic_ty ) -> #builder { - self.#subsystem_name = Some( new ); + pub fn #subsystem_name (mut self, subsystem: #subsystem_generic_ty ) -> Self { + self. #subsystem_name = Some( subsystem ); self } )* - fn build(mut self, create_subsystem_ctx: F) -> (#overseer_name #generics, #handler) + pub fn build(mut self, create_subsystem_ctx: F) -> (#overseer_name #generics, #handler) where F: FnMut( - metered::MeteredReceiver, - meteted::SubsystemIncomingMessages< #message_wrapper >, + ::polkadot_overseer_gen::metered::MeteredReceiver< #signal >, + SubsystemIncomingMessages< #message_wrapper >, ChannelsOut, - metered::UnboundedMeteredSender, + ::polkadot_overseer_gen::metered::UnboundedMeteredSender, ) -> Ctx, { - let (events_tx, events_rx) = ::metered::channel(SIGNAL_CHANNEL_CAPACITY); - - let handler = #handler { - events_tx: events_tx.clone(), - }; + let (events_tx, events_rx) = ::polkadot_overseer_gen:metered::channel(SIGNAL_CHANNEL_CAPACITY); + let handler: #handler = events_tx.clone(); - let (to_overseer_tx, to_overseer_rx) = metered::unbounded(); + let (to_overseer_tx, to_overseer_rx) = ::polkadot_overseer_gen::metered::unbounded(); let channels_out = { #( - let (#channel_name_tx, #channel_name_rx) = ::metered::channel::>(CHANNEL_CAPACITY); + let (#channel_name_tx, #channel_name_rx) = ::polkadot_overseer_gen::metered::channel::>(CHANNEL_CAPACITY); )* #( - let (#channel_name_unbounded_tx, #channel_name_unbounded_rx) = ::metered::unbounded::>(); + let (#channel_name_unbounded_tx, #channel_name_unbounded_rx) = ::polkadot_overseer_gen::metered::unbounded::>(); )* ChannelsOut { #( - #channel_name: #channel_name_tx .clone(), + #channel_name: #channel_name_tx, )* #( - #channel_name_unbounded: #channel_name_unbounded_tx .clone(), + #channel_name_unbounded: #channel_name_unbounded_tx, )* } }; let spawner = &mut overseer.spawner; - let mut running_subsystems = FuturesUnordered::>>::new(); + let mut running_subsystems = ::polkadot_overseer_gen::FuturesUnordered::>>::new(); #( + // FIXME generate a builder pattern that ensures this + let #subsystem_name = self. #subsystem_name .expect("All subsystem must exist with the builder pattern."); + let #subsystem_name: OverseenSubsystem< #message_wrapper > = { - let unbounded_meter = #channel_name_unbounded_tx .meter().clone(); + let unbounded_meter = channels_out. #channel_name .meter().clone(); - let message_tx: metered::MeteredSender> = #channel_name_tx; + let message_tx: ::polkadot_overseer_gen::metered::MeteredSender> = channels_out. #channel_name .clone(); - let message_rx: SubsystemIncomingMessages< #message_wrapper > = stream::select( - #channel_name_tx, - #channel_name_unbounded_tx, - ); + let message_rx: SubsystemIncomingMessages< #message_wrapper > = + ::polkadot_overseer_gen::select( + channels_out. #channel_name .clone(), + channels_out. #channel_name_unbounded .clone(), + ); - let (signal_tx, signal_rx) = metered::channel(SIGNAL_CHANNEL_CAPACITY); + let (signal_tx, signal_rx) = ::polkadot_overseer_gen::metered::channel(SIGNAL_CHANNEL_CAPACITY); let ctx = create_subsystem_ctx( signal_rx, @@ -263,15 +241,15 @@ pub(crate) fn impl_builder( to_overseer_tx.clone(), ); - let SpawnedSubsystem { future, name } = #subsystem_name .start(ctx); + let ::polkadot_overseer_gen::SpawnedSubsystem { future, name } = #subsystem_name .start(ctx); let (terminated_tx, terminated_rx) = oneshot::channel(); let fut = Box::pin(async move { if let Err(e) = future.await { - tracing::error!(subsystem=name, err = ?e, "subsystem exited with error"); + ::polkadot_overseer_gen::tracing::error!(subsystem=name, err = ?e, "subsystem exited with error"); } else { - tracing::debug!(subsystem=name, "subsystem exited without an error"); + ::polkadot_overseer_gen::tracing::debug!(subsystem=name, "subsystem exited without an error"); } let _ = terminated_tx.send(()); }); @@ -282,19 +260,21 @@ pub(crate) fn impl_builder( spawner.spawn(name, fut); } - futures.push(Box::pin(terminated_rx.map(|e| { tracing::warn!(err = ?e, "dropping error"); Ok(()) }))); - - let instance = Some(SubsystemInstance::< #message_wrapper > { - meters: SubsystemMeters { - unbounded: unbounded_meter, - bounded: message_tx.meter().clone(), - signals: signal_tx.meter().clone(), - }, - tx_signal: signal_tx, - tx_bounded: message_tx, - signals_received: 0, - name, - }); + running_subsystems.push(Box::pin(terminated_rx.map(|e| { tracing::warn!(err = ?e, "dropping error"); Ok(()) }))); + + let instance = Some( + SubsystemInstance::< #message_wrapper > { + meters: SubsystemMeters { + unbounded: unbounded_meter, + bounded: message_tx.meter().clone(), + signals: signal_tx.meter().clone(), + }, + tx_signal: signal_tx, + tx_bounded: message_tx, + signals_received: 0, + name, + } + ); OverseenSubsystem::< #message_wrapper > { instance, @@ -312,7 +292,7 @@ pub(crate) fn impl_builder( )* spawner, - running_instance, + running_subsystems, events_rx, to_overseer_rx, }; @@ -323,3 +303,204 @@ pub(crate) fn impl_builder( }; Ok(ts) } + + + + + +pub(crate) fn impl_subsystem_instance(info: &OverseerInfo) -> Result { + let signal = &info.extern_signal_ty; + + let ts = quote::quote! { + /// A running instance of some [`Subsystem`]. + /// + /// [`Subsystem`]: trait.Subsystem.html + /// + /// `M` here is the inner message type, and _not_ the generated `enum AllMessages`. + pub struct SubsystemInstance { + tx_signal: ::polkadot_overseer_gen::metered::MeteredSender< #signal >, + tx_bounded: ::polkadot_overseer_gen::metered::MeteredSender>, + meters: SubsystemMeters, + signals_received: usize, + name: &'static str, + } + }; + + Ok(ts) +} + + +pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { + let signal = &info.extern_signal_ty; + + let ts = quote::quote! { + + + /// A subsystem that we oversee. + /// + /// Ties together the [`Subsystem`] itself and it's running instance + /// (which may be missing if the [`Subsystem`] is not running at the moment + /// for whatever reason). + /// + /// [`Subsystem`]: trait.Subsystem.html + pub struct OverseenSubsystem { + pub instance: std::option::Option>, + } + + impl OverseenSubsystem { + /// Send a message to the wrapped subsystem. + /// + /// If the inner `instance` is `None`, nothing is happening. + pub async fn send_message(&mut self, msg: M) -> SubsystemResult<()> { + const MESSAGE_TIMEOUT: Duration = Duration::from_secs(10); + + if let Some(ref mut instance) = self.instance { + match instance.tx_bounded.send(MessagePacket { + signals_received: instance.signals_received, + message: msg.into() + }).timeout(MESSAGE_TIMEOUT).await + { + None => { + ::polkadot_overseer_gen::tracing::error!(target: crate::LOG_TARGET, "Subsystem {} appears unresponsive.", instance.name); + Err(SubsystemError::SubsystemStalled(instance.name)) + } + Some(res) => res.map_err(Into::into), + } + } else { + Ok(()) + } + } + + /// Send a signal to the wrapped subsystem. + /// + /// If the inner `instance` is `None`, nothing is happening. + pub async fn send_signal(&mut self, signal: #signal) -> SubsystemResult<()> { + const SIGNAL_TIMEOUT: Duration = Duration::from_secs(10); + + if let Some(ref mut instance) = self.instance { + match instance.tx_signal.send(signal).timeout(SIGNAL_TIMEOUT).await { + None => { + ::polkadot_overseer_gen::tracing::error!(target: crate::LOG_TARGET, "Subsystem {} appears unresponsive.", instance.name); + Err(SubsystemError::SubsystemStalled(instance.name)) + } + Some(res) => { + let res = res.map_err(Into::into); + if res.is_ok() { + instance.signals_received += 1; + } + res + } + } + } else { + Ok(()) + } + } + } + }; + Ok(ts) +} + + +pub(crate) fn impl_trait_subsystem_sender(info: &OverseerInfo) -> Result { + let message_wrapper = &info.message_wrapper; + + let ts = quote! { + #[::polkadot_overseer_gen::async_trait] + pub trait SubsystemSender: Send + Clone + 'static { + /// Send a direct message to some other `Subsystem`, routed based on message type. + async fn send_message(&mut self, msg: #message_wrapper); + + /// Send multiple direct messages to other `Subsystem`s, routed based on message type. + async fn send_messages(&mut self, msgs: T) + where T: IntoIterator + Send, T::IntoIter: Send; + + /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message + /// type. + /// + /// This function should be used only when there is some other bounding factor on the messages + /// sent with it. Otherwise, it risks a memory leak. + fn send_unbounded_message(&mut self, msg: #message_wrapper); + } + }; + Ok(ts) +} + + + +pub(crate) fn impl_trait_subsystem(info: &OverseerInfo) -> Result { + let message_wrapper = &info.message_wrapper; + let error = &info.extern_error_ty; + + let ts = quote! { + /// A message type that a subsystem receives from an overseer. + /// It wraps signals from an overseer and messages that are circulating + /// between subsystems. + /// + /// It is generic over over the message type `M` that a particular `Subsystem` may use. + #[derive(Debug)] + pub enum FromOverseer { + /// Signal from the `Overseer`. + Signal(Signal), + + /// Some other `Subsystem`'s message. + Communication { + /// Contained message + msg: M, + }, + } + + /// A context type that is given to the [`Subsystem`] upon spawning. + /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s + /// or spawn jobs. + /// + /// [`Overseer`]: struct.Overseer.html + /// [`SubsystemJob`]: trait.SubsystemJob.html + #[::polkadot_overseer_gen::async_trait] + pub trait SubsystemContext: Send + 'static { + /// The message type of this context. Subsystems launched with this context will expect + /// to receive messages of this type. + type Message: Send; + type Signal: Send; + + /// Try to asynchronously receive a message. + /// + /// This has to be used with caution, if you loop over this without + /// using `pending!()` macro you will end up with a busy loop! + async fn try_recv(&mut self) -> Result>, ()>; + + /// Receive a message. + async fn recv(&mut self) -> SubsystemResult>; + + /// Spawn a child task on the executor. + async fn spawn(&mut self, name: &'static str, s: ::std::pin::Pin + Send>>) -> SubsystemResult<()>; + + /// Spawn a blocking child task on the executor's dedicated thread pool. + async fn spawn_blocking( + &mut self, + name: &'static str, + s: ::std::pin::Pin + Send>>, + ) -> SubsystemResult<()>; + + /// Send a direct message to some other `Subsystem`, routed based on message type. + async fn send_message(&mut self, msg: #message_wrapper); + + /// Send multiple direct messages to other `Subsystem`s, routed based on message type. + async fn send_messages(&mut self, msgs: T) + where T: IntoIterator + Send, T::IntoIter: Send; + } + + /// A trait that describes the [`Subsystem`]s that can run on the [`Overseer`]. + /// + /// It is generic over the message type circulating in the system. + /// The idea that we want some type contaning persistent state that + /// can spawn actually running subsystems when asked to. + /// + /// [`Overseer`]: struct.Overseer.html + /// [`Subsystem`]: trait.Subsystem.html + pub trait Subsystem { + /// Start this `Subsystem` and return `SpawnedSubsystem`. + fn start(self, ctx: Ctx) -> SpawnedSubsystem < #error >; + } + }; + Ok(ts) +} diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs index fc43792418d3..714947fdbc9d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs @@ -1,5 +1,5 @@ //! Replace a subsystem -use proc_macro2::{Span, TokenStream}; +use proc_macro2::TokenStream; use quote::quote; use syn::{Ident, Result}; @@ -10,7 +10,6 @@ pub(crate) fn impl_replacable_subsystem( ) -> Result { let msg = "Generated by #[overlord] derive proc-macro."; - let span = Span::call_site(); let overseer_name = &info.overseer_name; let field_ty = &info.subsystem_generic_types(); @@ -25,7 +24,7 @@ pub(crate) fn impl_replacable_subsystem( let where_clause = quote! { where Ctx: SubsystemContext, - S: SpawnNamed, + S: ::polkadot_overseer_gen::SpawnNamed, #( #field_ty : Subsystem, )* }; @@ -35,10 +34,10 @@ pub(crate) fn impl_replacable_subsystem( for SubSysField { name: replacable_item, ty: _replacable_item_ty, generic, .. } in info.subsystems.iter() { let keeper = info.subsystems.iter().filter(|&ssf| ssf.generic != *generic).map(|ssf| ssf.name.clone()); - let fn_name = Ident::new(&format!("replace_{}", replacable_item), span); + let fn_name = Ident::new(&format!("replace_{}", replacable_item), replacable_item.span()); // adjust the generics such that the appropriate member type is replaced - let new = Ident::new("NEW", span); + let new = Ident::new("NEW", replacable_item.span()); let modified_generics = &info.subsystems .iter() .map(|ssf| if ssf.generic != *generic { ssf.generic.clone() } else { new.clone() }) @@ -58,10 +57,10 @@ pub(crate) fn impl_replacable_subsystem( #overseer_name :: #modified_generics { #replacable_item: replacement, #( - #keeper: self.#keeper, + #keeper: self. #keeper, )* #( - #baggage_name: self.#baggage_name, + #baggage_name: self. #baggage_name, )* } } diff --git a/node/overseer/overseer-gen/proc-macro/src/inc.rs b/node/overseer/overseer-gen/proc-macro/src/inc.rs deleted file mode 100644 index 7a2b189feac1..000000000000 --- a/node/overseer/overseer-gen/proc-macro/src/inc.rs +++ /dev/null @@ -1,9 +0,0 @@ -use syn::Result; - -pub(crate) fn include_static_rs() -> Result { - use std::str::FromStr; - - let s = include_str!("./inc/static.rs"); - let ts = proc_macro2::TokenStream::from_str(s)?; - Ok(ts) -} diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index b5305e7feea2..ea163c4a84f7 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -15,10 +15,12 @@ // along with Polkadot. If not, see . use proc_macro2::TokenStream; -use syn::{parse2, Error, GenericParam, Result}; +use syn::{parse2, Error, Result}; +use syn::spanned::Spanned; use std::collections::HashSet; -mod parse; +mod parse_struct; +mod parse_attr; mod impl_overseer; mod impl_replace; mod impl_channels_out; @@ -26,7 +28,8 @@ mod impl_message_wrapper; mod impl_dispatch; // mod inc; -use parse::*; +use parse_struct::*; +use parse_attr::*; use impl_overseer::*; use impl_replace::*; use impl_channels_out::*; @@ -41,65 +44,30 @@ pub fn overlord(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> } pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result { - let args: AttrArgs = syn::parse2(attr)?; + let args: AttrArgs = parse2(attr)?; let message_wrapper = args.message_wrapper; - let span = proc_macro2::Span::call_site(); - let ds = parse2::(orig.clone())?; - match ds.fields { - syn::Fields::Named(named) => { - let overseer_name = ds.ident.clone(); - - // collect the indepedentent subsystem generics - // which need to be carried along, there are the non-generated ones - let mut orig_generics = ds.generics; - - // remove default types - let mut baggage_generic_idents = HashSet::with_capacity(orig_generics.params.len()); - orig_generics.params = orig_generics - .params - .into_iter() - .map(|mut generic| { - match generic { - GenericParam::Type(ref mut param) => { - baggage_generic_idents.insert(param.ident.clone()); - param.eq_token = None; - param.default = None; - } - _ => {} - } - generic - }) - .collect(); - - let (subsystems, baggage) = parse_overseer_struct_field(baggage_generic_idents, named)?; - let info = OverseerInfo { - subsystems, - baggage, - overseer_name, - message_wrapper, - message_channel_capacity: args.message_channel_capacity, - signal_channel_capacity: args.signal_channel_capacity, - extern_event_ty: args.extern_event_ty, - extern_signal_ty: args.extern_signal_ty, - }; - - let mut additive = impl_overseer_struct(&info)?; - - additive.extend(impl_message_wrapper_enum(&info)?); - additive.extend(impl_channels_out_struct(&info)?); - additive.extend(impl_replacable_subsystem(&info)?); - additive.extend(impl_dispatch(&info)?); - - // additive.extend(inc::include_static_rs()?); - - Ok(additive) - } - syn::Fields::Unit => Err(Error::new(span, "Must be a struct with named fields. Not an unit struct.")), - syn::Fields::Unnamed(_) => { - Err(Error::new(span, "Must be a struct with named fields. Not an unnamed fields struct.")) - } - } + let of: OverseerGuts = parse2(orig)?; + + let info = OverseerInfo { + subsystems: of.subsystems, + baggage: of.baggage, + overseer_name: of.name, + message_wrapper, + message_channel_capacity: args.message_channel_capacity, + signal_channel_capacity: args.signal_channel_capacity, + extern_event_ty: args.extern_event_ty, + extern_signal_ty: args.extern_signal_ty, + extern_error_ty: args.extern_error_ty, + }; + + let mut additive = impl_overseer_struct(&info)?; + additive.extend(impl_message_wrapper_enum(&info)?); + additive.extend(impl_channels_out_struct(&info)?); + additive.extend(impl_replacable_subsystem(&info)?); + additive.extend(impl_dispatch(&info)?); + + Ok(additive) } #[cfg(test)] diff --git a/node/overseer/overseer-gen/proc-macro/src/parse.rs b/node/overseer/overseer-gen/proc-macro/src/parse.rs deleted file mode 100644 index bb7f6fa7b6c6..000000000000 --- a/node/overseer/overseer-gen/proc-macro/src/parse.rs +++ /dev/null @@ -1,379 +0,0 @@ -use proc_macro2::Span; -use std::collections::{HashMap, HashSet, hash_map::RandomState}; -use syn::{AttrStyle, Path}; -use syn::punctuated::Punctuated; -use syn::parse::Parse; -use syn::token::Paren; -use syn::Token; -use syn::Field; -use syn::FieldsNamed; -use syn::Result; -use syn::Ident; -use syn::spanned::Spanned; -use syn::Error; -use syn::Attribute; -use syn::Type; - -use syn::LitInt; -/// A field of the struct annotated with -/// `#[subsystem(no_dispatch, , A | B | C)]` -#[derive(Clone, Debug)] -pub(crate) struct SubSysField { - /// Name of the field. - pub(crate) name: Ident, - /// Generate generic type name for the `AllSubsystems` type. - pub(crate) generic: Ident, - /// Type of the subsystem. - pub(crate) ty: Ident, - /// Type to be consumed by the subsystem. - pub(crate) consumes: Ident, - /// If `no_dispatch` is present, if the message is incomming via - /// an extern `Event`, it will not be dispatched to all subsystems. - pub(crate) no_dispatch: bool, - /// If the subsystem imlementation is blocking execution and hence - /// has to be spawned on a separate thread or thread pool. - pub(crate) blocking: bool, -} - -fn try_type_to_ident(ty: Type, span: Span) -> Result { - match ty { - Type::Path(path) => { - path.path.get_ident().cloned().ok_or_else(|| Error::new(span, "Expected an identifier, but got a path.")) - } - _ => Err(Error::new(span, "Type must be a path expression.")), - } -} - - -use syn::parse::ParseBuffer; - -#[derive(Clone, Debug)] -enum AttrItem { - ExternEventType(Path), - ExternOverseerSignalType(Path), - MessageWrapperName(Ident), - SignalChannelCapacity(LitInt), - MessageChannelCapacity(LitInt), -} - -impl Spanned for AttrItem { - fn span(&self) -> Span { - match self { - AttrItem::ExternEventType(x) => x.span(), - AttrItem::MessageWrapperName(x) => x.span(), - AttrItem::SignalChannelCapacity(x) => x.span(), - AttrItem::MessageChannelCapacity(x) => x.span(), - AttrItem::ExternOverseerSignalType(x) => x.span(), - } - } -} - -impl AttrItem { - fn key(&self) -> &'static str { - match self { - AttrItem::ExternEventType(_) => "event", - AttrItem::ExternOverseerSignalType(_) => "signal", - AttrItem::MessageWrapperName(_) => "gen", - AttrItem::SignalChannelCapacity(_) => "signal_capacity", - AttrItem::MessageChannelCapacity(_) => "message_capacity", - } - } -} - -impl Parse for AttrItem { - fn parse(input: &ParseBuffer) -> Result { - let key = input.parse::()?; - let span = Span::call_site(); - let _ = input.parse::()?; - Ok(if key == "signal" { - let path = input.parse::()?; - AttrItem::ExternOverseerSignalType(path) - } else if key == "event" { - let path = input.parse::()?; - AttrItem::ExternEventType(path) - } else if key == "gen" { - let wrapper_message = input.parse::()?; - AttrItem::MessageWrapperName(wrapper_message) - } else if key == "signal_capacity" { - let value = input.parse::()?; - AttrItem::SignalChannelCapacity(value) - } else if key == "message_capacity" { - let value = input.parse::()?; - AttrItem::MessageChannelCapacity(value) - } else { - return Err(Error::new(span, "Expected one of `gen`, `signal_capacity`, or `message_capacity`.")) - }) - } -} - -/// Attribute arguments -#[derive(Clone, Debug)] -pub(crate) struct AttrArgs { - pub(crate) message_wrapper: Ident, - pub(crate) extern_event_ty: Path, - pub(crate) extern_signal_ty: Path, - pub(crate) signal_channel_capacity: u64, - pub(crate) message_channel_capacity: u64, -} - -impl Parse for AttrArgs { - fn parse(input: &ParseBuffer) -> Result { - let span = Span::call_site(); - - let content; - let _paren = syn::parenthesized!(content in input); - let items: Punctuated<_, Token![,]> = content.parse_terminated(AttrItem::parse)?; - - let mut unique = HashMap::<&str, AttrItem, RandomState>::default(); - for item in items { - if let Some(first) = unique.insert(item.key(), item.clone()) { - let mut e = Error::new(item.span(), "Duplicate definition found"); - e.combine(Error::new(first.span(), "previously defined here.")); - return Err(e) - } - } - - let signal_channel_capacity = if let Some(item) = unique.get("signal_capacity") { - if let AttrItem::SignalChannelCapacity(lit) = item { - lit.base10_parse::()? - } else { - unreachable!() - } - } else { - 64 - }; - - let message_channel_capacity = if let Some(item) = unique.get("message_capacity") { - if let AttrItem::MessageChannelCapacity(lit) = item { - lit.base10_parse::()? - } else { - unreachable!() - } - } else { - 1024 - }; - let extern_signal_ty = unique.get("signal") - .map(|x| if let AttrItem::ExternOverseerSignalType(x) = x { x.clone() } else { unreachable!() } ) - .ok_or_else(|| { - Error::new(span, "Must declare the overseer signals type via `signal=..`.") - })?; - - let extern_event_ty = unique.get("event") - .map(|x| if let AttrItem::ExternEventType(x) = x { x.clone() } else { unreachable!() } ) - .ok_or_else(|| { - Error::new(span, "Must declare the external event type via `event=..`.") - })?; - - let message_wrapper = unique.get("gen") - .map(|x| if let AttrItem::MessageWrapperName(x) = x { x.clone() } else { unreachable!() } ) - .ok_or_else(|| { - Error::new(span, "Must declare the generated type via `gen=..`.") - })?; - - Ok(AttrArgs { - signal_channel_capacity, - message_channel_capacity, - extern_event_ty, - extern_signal_ty, - message_wrapper, - }) - } -} - -pub(crate) struct SubSystemTag { - #[allow(dead_code)] - pub(crate) attrs: Vec, - #[allow(dead_code)] - pub(crate) paren_token: Paren, - pub(crate) no_dispatch: bool, - pub(crate) blocking: bool, - pub(crate) consumes: Punctuated, -} - -impl Parse for SubSystemTag { - fn parse(input: syn::parse::ParseStream) -> Result { - let attrs = Attribute::parse_outer(input)?; - - let content; - let paren_token = syn::parenthesized!(content in input); - - let parse_tags = || -> Result> { - if content.peek(Ident) && content.peek2(Token![,]) { - let ident = content.parse::()?; - let _ = content.parse::()?; - Ok(Some(ident)) - } else { - Ok(None) - } - }; - - let mut unique = HashSet::<_, RandomState>::default(); - while let Some(ident) = parse_tags()? { - if ident != "no_dispatch" && ident != "blocking" { - return Err(Error::new(ident.span(), "Allowed tags are only `no_dispatch` or `blocking`.")) - } - if !unique.insert(ident.to_string()) { - return Err(Error::new(ident.span(), "Allowed tags are only `no_dispatch` or `blocking`.")) - } - } - let no_dispatch = unique.get("no_dispatch").is_some(); - let blocking = unique.get("blocking").is_some(); - - - let consumes = content.parse_terminated(Ident::parse)?; - - Ok(Self { - attrs, - paren_token, - no_dispatch, - blocking, - consumes, - }) - } -} - - -/// Fields that are _not_ subsystems. -#[derive(Debug, Clone)] -pub(crate) struct BaggageField { - pub(crate) field_name: Ident, - pub(crate) field_ty: Ident, - pub(crate) generic: bool, -} - - -#[derive(Clone, Debug)] -pub(crate) struct OverseerInfo { - /// Fields annotated with `#[subsystem(..)]`. - pub(crate) subsystems: Vec, - /// Fields that do not define a subsystem, - /// but are mere baggage. - pub(crate) baggage: Vec, - /// Name of the wrapping enum for all messages, defaults to `AllMessages`. - pub(crate) message_wrapper: Ident, - /// Name of the overseer struct, used as a prefix for - /// almost all generated types. - pub(crate) overseer_name: Ident, - - /// Size of the bounded channel. - pub(crate) message_channel_capacity: u64, - /// Size of the bounded signal channel. - pub(crate) signal_channel_capacity: u64, - - /// Signals to be sent, sparse information that is used intermittendly. - pub(crate) extern_signal_ty: Path, - - /// Incoming event type from the outer world, commonly from the network. - pub(crate) extern_event_ty: Path, -} - -impl OverseerInfo { - pub(crate) fn subsystems(&self) -> &[SubSysField] { - self.subsystems.as_slice() - } - - pub(crate) fn subsystem_names(&self) -> Vec { - self.subsystems.iter().map(|ssf| ssf.name.clone()).collect::>() - } - - #[allow(dead_code)] - // FIXME use as the defaults - pub(crate) fn subsystem_types(&self) -> Vec { - self.subsystems.iter().map(|ssf| ssf.ty.clone()).collect::>() - } - - pub(crate) fn baggage_names(&self) -> Vec { - self.baggage.iter().map(|bag| bag.field_name.clone()).collect::>() - } - pub(crate) fn baggage_types(&self) -> Vec { - self.baggage.iter().map(|bag| bag.field_ty.clone()).collect::>() - } - - - pub(crate) fn subsystem_generic_types(&self) -> Vec { - self.subsystems.iter().map(|sff| sff.generic.clone()).collect::>() - } - - pub(crate) fn baggage_generic_types(&self) -> Vec { - self.baggage.iter().filter(|bag| bag.generic).map(|bag| bag.field_ty.clone()).collect::>() - } - - pub(crate) fn channel_names(&self, suffix: &'static str) -> Vec { - self.subsystems.iter() - .map(|ssf| Ident::new(&(ssf.name.to_string() + suffix), ssf.name.span())) - .collect::>() - } - - pub(crate) fn consumes(&self) -> Vec { - self.subsystems.iter() - .map(|ssf| ssf.consumes.clone()) - .collect::>() - } -} - - -/// Creates a list of generic identifiers used for the subsystems -pub(crate) fn parse_overseer_struct_field( - baggage_generics: HashSet, - fields: FieldsNamed, -) -> Result<(Vec, Vec)> { - let _span = Span::call_site(); - let n = fields.named.len(); - let mut subsystems = Vec::with_capacity(n); - let mut baggage = Vec::with_capacity(n); - for (idx, Field { attrs, vis: _, ident, ty, .. }) in fields.named.into_iter().enumerate() { - let mut consumes = attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| { - let span = attr.path.span(); - attr.path.get_ident().filter(|ident| *ident == "subsystem").map(move |_ident| { - let attr_tokens = attr.tokens.clone(); - (attr_tokens, span) - }) - }); - let ident = ident.ok_or_else(|| { - Error::new(ty.span(), "Missing identifier for member. BUG") - })?; - - if let Some((attr_tokens, span)) = consumes.next() { - if let Some((_attr_tokens2, span2)) = consumes.next() { - return Err({ - let mut err = Error::new(span, "The first subsystem annotation is at"); - err.combine( - Error::new(span2, "but another here for the same field.") - ); - err - }) - } - let mut consumes_idents = Vec::with_capacity(attrs.len()); - - let variant = syn::parse2::(attr_tokens.clone())?; - if variant.consumes.len() != 1 { - return Err(Error::new(attr_tokens.span(), "Exactly one message can be consumed per subsystem.")) - } - consumes_idents.extend(variant.consumes.into_iter()); - - - if consumes_idents.is_empty() { - return Err( - Error::new(span, "Subsystem must consume at least one message") - ) - } - - subsystems.push(SubSysField { - name: ident, - generic: Ident::new(format!("Sub{}", idx).as_str(), Span::call_site()), - ty: try_type_to_ident(ty, span)?, - consumes: consumes_idents[0].clone(), - no_dispatch: variant.no_dispatch, - blocking: variant.blocking, - }); - } else { - let field_ty = try_type_to_ident(ty, Span::call_site())?; - baggage.push(BaggageField { - field_name: ident, - generic: !baggage_generics.contains(&field_ty), - field_ty, - }); - } - } - Ok((subsystems, baggage)) -} diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs new file mode 100644 index 000000000000..58c05fb6b42f --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs @@ -0,0 +1,174 @@ +use proc_macro2::Span; +use std::collections::{HashMap, hash_map::RandomState}; +use syn::Path; +use syn::Error; +use syn::Ident; +use syn::LitInt; +use syn::parse::Parse; +use syn::parse::ParseBuffer; +use syn::punctuated::Punctuated; +use syn::Result; +use syn::spanned::Spanned; +use syn::Token; + + +#[derive(Clone, Debug)] +enum AttrItem { + ExternEventType(Path), + ExternOverseerSignalType(Path), + ExternErrorType(Path), + MessageWrapperName(Ident), + SignalChannelCapacity(LitInt), + MessageChannelCapacity(LitInt), +} + +impl Spanned for AttrItem { + fn span(&self) -> Span { + match self { + AttrItem::ExternEventType(x) => x.span(), + AttrItem::ExternOverseerSignalType(x) => x.span(), + AttrItem::ExternErrorType(x) => x.span(), + AttrItem::MessageWrapperName(x) => x.span(), + AttrItem::SignalChannelCapacity(x) => x.span(), + AttrItem::MessageChannelCapacity(x) => x.span(), + } + } +} + +const TAG_EXT_EVENT_TY: &str = "event"; +const TAG_EXT_SIGNAL_TY: &str = "signal"; +const TAG_EXT_ERROR_TY: &str = "error"; +const TAG_GEN_TY: &str = "gen"; +const TAG_SIGNAL_CAPACITY: &str = "signal_capacity"; +const TAG_MESSAGE_CAPACITY: &str = "message_capacity"; + +impl AttrItem { + fn key(&self) -> &'static str { + match self { + AttrItem::ExternEventType(_) => TAG_EXT_EVENT_TY, + AttrItem::ExternOverseerSignalType(_) => TAG_EXT_SIGNAL_TY, + AttrItem::ExternErrorType(_) => TAG_EXT_ERROR_TY, + AttrItem::MessageWrapperName(_) => TAG_GEN_TY, + AttrItem::SignalChannelCapacity(_) => TAG_SIGNAL_CAPACITY, + AttrItem::MessageChannelCapacity(_) => TAG_MESSAGE_CAPACITY, + } + } +} + +impl Parse for AttrItem { + fn parse(input: &ParseBuffer) -> Result { + let key = input.parse::()?; + let span = key.span(); + let _ = input.parse::()?; + Ok(if key == TAG_EXT_SIGNAL_TY { + let path = input.parse::()?; + AttrItem::ExternOverseerSignalType(path) + } else if key == TAG_EXT_EVENT_TY { + let path = input.parse::()?; + AttrItem::ExternEventType(path) + } else if key == TAG_EXT_ERROR_TY { + let path = input.parse::()?; + AttrItem::ExternErrorType(path) + } else if key == TAG_GEN_TY { + let wrapper_message = input.parse::()?; + AttrItem::MessageWrapperName(wrapper_message) + } else if key == TAG_SIGNAL_CAPACITY { + let value = input.parse::()?; + AttrItem::SignalChannelCapacity(value) + } else if key == TAG_MESSAGE_CAPACITY { + let value = input.parse::()?; + AttrItem::MessageChannelCapacity(value) + } else { + return Err(Error::new(span, "Expected one of `gen`, `signal_capacity`, or `message_capacity`.")) + }) + } +} + +/// Attribute arguments +#[derive(Clone, Debug)] +pub(crate) struct AttrArgs { + pub(crate) message_wrapper: Ident, + pub(crate) extern_event_ty: Path, + pub(crate) extern_signal_ty: Path, + pub(crate) extern_error_ty: Path, + pub(crate) signal_channel_capacity: u64, + pub(crate) message_channel_capacity: u64, +} + +impl Parse for AttrArgs { + fn parse(input: &ParseBuffer) -> Result { + let span = input.span(); + let items: Punctuated = input.parse_terminated(AttrItem::parse)?; + + let mut unique = HashMap::<&str, AttrItem, RandomState>::default(); + for item in items { + if let Some(first) = unique.insert(item.key(), item.clone()) { + let mut e = Error::new(item.span(), format!("Duplicate definition of `{}` found", item.key())); + e.combine(Error::new(first.span(), "previously defined here.")); + return Err(e) + } + } + + let signal_channel_capacity = if let Some(item) = unique.remove(TAG_SIGNAL_CAPACITY) { + if let AttrItem::SignalChannelCapacity(lit) = item { + lit.base10_parse::()? + } else { + unreachable!() + } + } else { + 64 + }; + + let message_channel_capacity = if let Some(item) = unique.remove(TAG_MESSAGE_CAPACITY) { + if let AttrItem::MessageChannelCapacity(lit) = item { + lit.base10_parse::()? + } else { + unreachable!() + } + } else { + 1024 + }; + let extern_error_ty = unique.remove(TAG_EXT_ERROR_TY) + .map(|x| if let AttrItem::ExternErrorType(x) = x { x.clone() } else { unreachable!() } ) + .ok_or_else(|| { + Error::new(span, format!("Must declare the overseer signals type via `{}=..`.", TAG_EXT_ERROR_TY)) + })?; + + let extern_signal_ty = unique.remove(TAG_EXT_SIGNAL_TY) + .map(|x| if let AttrItem::ExternOverseerSignalType(x) = x { x.clone() } else { unreachable!() } ) + .ok_or_else(|| { + Error::new(span, format!("Must declare the overseer signals type via `{}=..`.", TAG_EXT_SIGNAL_TY)) + })?; + + let extern_event_ty = unique.remove(TAG_EXT_EVENT_TY) + .map(|x| if let AttrItem::ExternEventType(x) = x { x.clone() } else { unreachable!() } ) + .ok_or_else(|| { + Error::new(span, format!("Must declare the external event type via `{}=..`.", TAG_EXT_EVENT_TY)) + + })?; + + let message_wrapper = unique.remove(TAG_GEN_TY) + .map(|x| if let AttrItem::MessageWrapperName(x) = x { x.clone() } else { unreachable!() } ) + .ok_or_else(|| { + Error::new(span, format!("Must declare the generated type via `{}=..`.", TAG_GEN_TY)) + })?; + + if !unique.is_empty() { + let v = unique.into_iter().map(|(tag, _attr)| -> String { + format!("`{}`", tag) + }).collect::>(); + let s = v.join(", "); + + return Err(Error::new(span, format!("Found unknown arguments to the overseer macro {}.", s))) + } + + Ok(AttrArgs { + signal_channel_capacity, + message_channel_capacity, + extern_event_ty, + extern_signal_ty, + extern_error_ty, + message_wrapper, + }) + } +} diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs new file mode 100644 index 000000000000..66dbe3f67f32 --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -0,0 +1,284 @@ +use proc_macro2::Span; +use std::collections::{HashSet, hash_map::RandomState}; +use syn::{AttrStyle, Path}; +use syn::Attribute; +use syn::Field; +use syn::FieldsNamed; +use syn::Ident; +use syn::parse::Parse; +use syn::punctuated::Punctuated; +use syn::spanned::Spanned; +use syn::Token; +use syn::Type; +use syn::{parse2, ItemStruct, Error, GenericParam, Result}; +use syn::parse::ParseStream; + +/// A field of the struct annotated with +/// `#[subsystem(no_dispatch, , A | B | C)]` +#[derive(Clone, Debug)] +pub(crate) struct SubSysField { + /// Name of the field. + pub(crate) name: Ident, + /// Generate generic type name for the `AllSubsystems` type. + pub(crate) generic: Ident, + /// Type of the subsystem. + pub(crate) ty: Ident, + /// Type to be consumed by the subsystem. + pub(crate) consumes: Ident, + /// If `no_dispatch` is present, if the message is incomming via + /// an extern `Event`, it will not be dispatched to all subsystems. + pub(crate) no_dispatch: bool, + /// If the subsystem imlementation is blocking execution and hence + /// has to be spawned on a separate thread or thread pool. + pub(crate) blocking: bool, +} + +fn try_type_to_ident(ty: Type, span: Span) -> Result { + match ty { + Type::Path(path) => { + path.path.get_ident().cloned().ok_or_else(|| Error::new(span, "Expected an identifier, but got a path.")) + } + _ => Err(Error::new(span, "Type must be a path expression.")), + } +} + +pub(crate) struct SubSystemTag { + #[allow(dead_code)] + pub(crate) attrs: Vec, + #[allow(dead_code)] + pub(crate) no_dispatch: bool, + pub(crate) blocking: bool, + pub(crate) consumes: Punctuated, +} + +impl Parse for SubSystemTag { + fn parse(input: syn::parse::ParseStream) -> Result { + let attrs = Attribute::parse_outer(input)?; + + let input = dbg!(input); + let content; + let _ = syn::parenthesized!(content in input); + let parse_tags = || -> Result> { + if content.peek(Ident) && content.peek2(Token![,]) { + let ident = content.parse::()?; + let _ = content.parse::()?; + Ok(Some(ident)) + } else { + Ok(None) + } + }; + + let mut unique = HashSet::<_, RandomState>::default(); + while let Some(ident) = parse_tags()? { + if ident != "no_dispatch" && ident != "blocking" { + return Err(Error::new(ident.span(), "Allowed tags are only `no_dispatch` or `blocking`.")) + } + if !unique.insert(ident.to_string()) { + return Err(Error::new(ident.span(), "Found duplicate tag.")) + } + } + let no_dispatch = unique.take("no_dispatch").is_some(); + let blocking = unique.take("blocking").is_some(); + + let consumes = content.parse_terminated(Ident::parse)?; + + Ok(Self { + attrs, + no_dispatch, + blocking, + consumes, + }) + } +} + + +/// Fields that are _not_ subsystems. +#[derive(Debug, Clone)] +pub(crate) struct BaggageField { + pub(crate) field_name: Ident, + pub(crate) field_ty: Ident, + pub(crate) generic: bool, +} + + +#[derive(Clone, Debug)] +pub(crate) struct OverseerInfo { + /// Fields annotated with `#[subsystem(..)]`. + pub(crate) subsystems: Vec, + /// Fields that do not define a subsystem, + /// but are mere baggage. + pub(crate) baggage: Vec, + /// Name of the wrapping enum for all messages, defaults to `AllMessages`. + pub(crate) message_wrapper: Ident, + /// Name of the overseer struct, used as a prefix for + /// almost all generated types. + pub(crate) overseer_name: Ident, + + /// Size of the bounded channel. + pub(crate) message_channel_capacity: u64, + /// Size of the bounded signal channel. + pub(crate) signal_channel_capacity: u64, + + /// Signals to be sent, sparse information that is used intermittendly. + pub(crate) extern_signal_ty: Path, + + /// Incoming event type from the outer world, commonly from the network. + pub(crate) extern_event_ty: Path, + + /// Incoming event type from the outer world, commonly from the network. + pub(crate) extern_error_ty: Path, +} + +impl OverseerInfo { + pub(crate) fn subsystems(&self) -> &[SubSysField] { + self.subsystems.as_slice() + } + + pub(crate) fn subsystem_names(&self) -> Vec { + self.subsystems.iter().map(|ssf| ssf.name.clone()).collect::>() + } + + #[allow(dead_code)] + // FIXME use as the defaults + pub(crate) fn subsystem_types(&self) -> Vec { + self.subsystems.iter().map(|ssf| ssf.ty.clone()).collect::>() + } + + pub(crate) fn baggage_names(&self) -> Vec { + self.baggage.iter().map(|bag| bag.field_name.clone()).collect::>() + } + pub(crate) fn baggage_types(&self) -> Vec { + self.baggage.iter().map(|bag| bag.field_ty.clone()).collect::>() + } + + + pub(crate) fn subsystem_generic_types(&self) -> Vec { + self.subsystems.iter().map(|sff| sff.generic.clone()).collect::>() + } + + pub(crate) fn baggage_generic_types(&self) -> Vec { + self.baggage.iter().filter(|bag| bag.generic).map(|bag| bag.field_ty.clone()).collect::>() + } + + pub(crate) fn channel_names(&self, suffix: &'static str) -> Vec { + self.subsystems.iter() + .map(|ssf| Ident::new(&(ssf.name.to_string() + suffix), ssf.name.span())) + .collect::>() + } + + pub(crate) fn consumes(&self) -> Vec { + self.subsystems.iter() + .map(|ssf| ssf.consumes.clone()) + .collect::>() + } +} + +/// Internals of the overseer. +#[derive(Debug, Clone)] +pub(crate) struct OverseerGuts { + pub(crate) name: Ident, + pub(crate) subsystems: Vec, + pub(crate) baggage: Vec, +} + +impl OverseerGuts { + pub(crate) fn parse_fields(name: Ident, baggage_generics: HashSet, fields: FieldsNamed) -> Result { + let n = fields.named.len(); + let mut subsystems = Vec::with_capacity(n); + let mut baggage = Vec::with_capacity(n); + for (idx, Field { attrs, vis: _, ident, ty, .. }) in fields.named.into_iter().enumerate() { + let mut consumes = attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| { + let span = attr.path.span(); + attr.path.get_ident().filter(|ident| *ident == "subsystem").map(move |_ident| { + let attr_tokens = attr.tokens.clone(); + (attr_tokens, span) + }) + }); + let ident = ident.ok_or_else(|| { + Error::new(ty.span(), "Missing identifier for member. BUG") + })?; + + if let Some((attr_tokens, span)) = consumes.next() { + if let Some((_attr_tokens2, span2)) = consumes.next() { + return Err({ + let mut err = Error::new(span, "The first subsystem annotation is at"); + err.combine( + Error::new(span2, "but another here for the same field.") + ); + err + }) + } + let mut consumes_idents = Vec::with_capacity(attrs.len()); + let attr_tokens = dbg!(attr_tokens.clone()); + let variant: SubSystemTag = syn::parse2(attr_tokens.clone())?; + if variant.consumes.len() != 1 { + return Err(Error::new(attr_tokens.span(), "Exactly one message can be consumed per subsystem.")) + } + consumes_idents.extend(variant.consumes.into_iter()); + + + if consumes_idents.is_empty() { + return Err( + Error::new(span, "Subsystem must consume at least one message") + ) + } + + subsystems.push(SubSysField { + name: ident, + generic: Ident::new(format!("Sub{}", idx).as_str(), span), + ty: try_type_to_ident(ty, span)?, + consumes: consumes_idents[0].clone(), + no_dispatch: variant.no_dispatch, + blocking: variant.blocking, + }); + } else { + let field_ty = try_type_to_ident(ty, ident.span())?; + baggage.push(BaggageField { + field_name: ident, + generic: !baggage_generics.contains(&field_ty), + field_ty, + }); + } + } + Ok( Self { name, subsystems, baggage }) + } +} + +impl Parse for OverseerGuts { + fn parse(input: ParseStream) -> Result { + let ds: ItemStruct = input.parse()?; + match ds.fields { + syn::Fields::Named(named) => { + let name = ds.ident.clone(); + + // collect the indepedentent subsystem generics + // which need to be carried along, there are the non-generated ones + let mut orig_generics = ds.generics; + + // remove defaults from types + let mut baggage_generic_idents = HashSet::with_capacity(orig_generics.params.len()); + orig_generics.params = orig_generics + .params + .into_iter() + .map(|mut generic| { + match generic { + GenericParam::Type(ref mut param) => { + baggage_generic_idents.insert(param.ident.clone()); + param.eq_token = None; + param.default = None; + } + _ => {} + } + generic + }) + .collect(); + + Self::parse_fields(name, baggage_generic_idents, named) + } + syn::Fields::Unit => Err(Error::new(ds.fields.span(), "Must be a struct with named fields. Not an unit struct.")), + syn::Fields::Unnamed(unnamed) => { + Err(Error::new(unnamed.span(), "Must be a struct with named fields. Not an unnamed fields struct.")) + } + } + } +} diff --git a/node/overseer/overseer-gen/proc-macro/src/tests.rs b/node/overseer/overseer-gen/proc-macro/src/tests.rs index dd5695601c76..98f96bfb3ae3 100644 --- a/node/overseer/overseer-gen/proc-macro/src/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/tests.rs @@ -3,10 +3,12 @@ use quote::quote; use assert_matches::assert_matches; use syn::parse_quote; + + #[test] -fn basic() { +fn print() { let attr = quote! { - (event=OverseerSignal,signal=SigSigSig, gen=AllMessages) + gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, }; let item = quote! { @@ -34,9 +36,43 @@ fn basic() { #[test] -fn attr_parsing_works() { +fn struct_parse_full() { + let item: OverseerGuts = parse_quote! { + pub struct Ooooh where X: Secrit { + #[subsystem(no_dispatch, Foo)] + sub0: FooSubsystem, + + #[subsystem(blocking, Bar)] + yyy: BaersBuyBilliardBalls, + + #[subsystem(no_dispatch, blocking, Twain)] + fff: Beeeeep, + + #[subsystem(Rope)] + mc: MountainCave, + + metrics: Metrics, + } + }; + let _ = dbg!(item); +} + +#[test] +fn struct_parse_basic() { + let item: OverseerGuts = parse_quote! { + pub struct Ooooh { + #[subsystem(Foo)] + sub0: FooSubsystem, + } + }; + let _ = dbg!(item); +} + + +#[test] +fn attr_full() { let attr: AttrArgs = parse_quote! { - (gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222,) + gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, }; assert_matches!(attr, AttrArgs { message_channel_capacity, @@ -47,3 +83,17 @@ fn attr_parsing_works() { assert_eq!(signal_channel_capacity, 111); }); } + + +#[test] +fn attr_partial() { + let attr: AttrArgs = parse_quote! { + gen=AllMessage, event=::some::why::ExternEvent, signal=::foo::SigSigSig, + }; + assert_matches!(attr, AttrArgs { + message_channel_capacity, + signal_channel_capacity, + .. + } => { + }); +} diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 003460774793..de81d5af8231 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -60,12 +60,29 @@ #![warn(missing_docs)] pub use overseer_gen_proc_macro::*; +#[doc(hidden)] pub use tracing; +#[doc(hidden)] pub use metered; +#[doc(hidden)] pub use sp_core::traits::SpawnNamed; -pub use futures::future::BoxFuture; +#[doc(hidden)] +pub use futures::{ + select, + future::{ + Fuse, Future, BoxFuture + }, + stream::{ + select, FuturesUnordered + }, + channel::{mpsc, oneshot}, +}; +#[doc(hidden)] +pub use async_trait::async_trait; +#[doc(hidden)] +pub use std::time::Duration; use std::sync::atomic::{self, AtomicUsize}; use std::sync::Arc; @@ -128,8 +145,8 @@ pub fn make_packet(signals_received: usize, message: T) -> MessagePacket { /// Incoming messages from both the bounded and unbounded channel. pub type SubsystemIncomingMessages = ::futures::stream::Select< - ::metered::MeteredReceiver>, - ::metered::UnboundedMeteredReceiver>, + self::metered::MeteredReceiver>, + self::metered::UnboundedMeteredReceiver>, >; @@ -152,6 +169,72 @@ impl SignalsReceived { } + +/// A trait to support the origin annotation +/// such that errors across subsystems can be easier tracked. +pub trait AnnotateErrorOrigin: 'static + Send + Sync + std::error::Error { + fn with_origin(self, origin: &'static str) -> Self; +} + + +/// An asynchronous subsystem task.. +/// +/// In essence it's just a newtype wrapping a `BoxFuture`. +pub struct SpawnedSubsystem +where +E: std::error::Error + + 'static + + Send + + From +{ + /// Name of the subsystem being spawned. + pub name: &'static str, + /// The task of the subsystem being spawned. + pub future: BoxFuture<'static, Result<(), E>>, +} + + +/// An error type that describes faults that may happen +/// +/// These are: +/// * Channels being closed +/// * Subsystems dying when they are not expected to +/// * Subsystems not dying when they are told to die +/// * etc. +// FIXME XXX make generic over the source error of FromOrigin +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum SubsystemError { + #[error(transparent)] + NotifyCancellation(#[from] oneshot::Canceled), + + #[error(transparent)] + QueueError(#[from] mpsc::SendError), + + #[error(transparent)] + TaskSpawn(#[from] futures::task::SpawnError), + + #[error(transparent)] + Infallible(#[from] std::convert::Infallible), + + #[error("Failed to {0}")] + Context(String), + + #[error("Subsystem stalled: {0}")] + SubsystemStalled(&'static str), + + /// Per origin (or subsystem) annotations to wrap an error. + #[error("Error originated in {origin}")] + FromOrigin { + /// An additional anotation tag for the origin of `source`. + origin: &'static str, + /// The wrapped error. Marked as source for tracking the error chain. + #[source] source: Box + }, +} + +pub type SubsystemResult = std::result::Result; + /// Collection of meters related to a subsystem. #[derive(Clone)] pub struct SubsystemMeters { @@ -185,7 +268,5 @@ pub struct SubsystemMeterReadouts { pub signals: metered::Readout, } - - #[cfg(test)] mod tests; diff --git a/node/overseer/overseer-gen/src/tests.rs b/node/overseer/overseer-gen/src/tests.rs index 36d0089c1b7c..a5e77d1e7077 100644 --- a/node/overseer/overseer-gen/src/tests.rs +++ b/node/overseer/overseer-gen/src/tests.rs @@ -1,8 +1,9 @@ +use super::*; #[test] fn ui_pass() { let t = trybuild::TestCases::new(); - t.pass("tests/ui/ok-*.rs"); + t.pass("tests/ui/ok-01*.rs"); } #[test] fn ui_compile_fail() { diff --git a/node/overseer/overseer-gen/tests/ui/err-01-duplicate-consumer.rs b/node/overseer/overseer-gen/tests/ui/err-01-duplicate-consumer.rs new file mode 100644 index 000000000000..48560852464b --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/err-01-duplicate-consumer.rs @@ -0,0 +1,37 @@ +#![allow(dead_code)] + +use polkadot_overseer_gen::overlord; + +#[derive(Default)] +struct AwesomeSubSys; + +#[derive(Default)] +struct AwesomeSubSys2; + +struct SigSigSig; + +struct Event; + +#[derive(Clone)] +struct MsgStrukt(u8); + +#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] +struct Overseer { + #[subsystem(MsgStrukt)] + sub0: AwesomeSubSys, + + #[subsystem(MsgStrukt)] + sub1: AwesomeSubSys2, +} + +#[derive(Debug, Clone)] +struct DummySpawner; + +struct DummyCtx; + +fn main() { + let overseer = Overseer::<_,_>::builder() + .sub0(AwesomeSubSys::default()) + .spawner(DummySpawner) + .build(|| -> DummyCtx { DummyCtx } ); +} diff --git a/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs b/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs deleted file mode 100644 index d51728f41fe6..000000000000 --- a/node/overseer/overseer-gen/tests/ui/err-01-replace-w-inadequate.rs +++ /dev/null @@ -1,38 +0,0 @@ -#![allow(dead_code)] - -use polkadot_overseer_gen::overlord; - -struct X; - -struct Orange; - -#[derive(Default, Clone, Copy)] -struct AwesomeSubSys; - - -#[derive(Default, Clone, Copy)] -struct TequilaInABar; - - -#[overlord(Wrapper)] -struct Overseer { - #[subsystem(X)] - sub0: AwesomeSubSys, - - #[subsystem(Orange)] - shots_of: TequilaInABar, - - other: Stuff, -} - -struct Spawner; - -fn main() { - let overseer = Overseer::>::builder() - .sub0(FooSubSys::default()) - .build(Spawner); - - // try to replace one subsystem with another that can not handle `X`. - // since it's missing the trait bound. - let overseer = overseer.replace_sub0(TequilaInABar::default()); -} diff --git a/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs b/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs index 4fbbdb247626..c8979d7a3adf 100644 --- a/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs +++ b/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs @@ -1,39 +1,38 @@ #![allow(dead_code)] use polkadot_overseer_gen::overlord; -struct X; -struct Z; -struct Orange; - -#[derive(Default, Clone, Copy)] +#[derive(Default)] struct AwesomeSubSys; +struct SigSigSig; + +struct Event; -#[derive(Default, Clone, Copy)] -struct TequilaInABar; +#[derive(Clone)] +struct MsgStrukt(u8); +#[derive(Clone)] +struct MsgStrukt2(f64); -#[overlord(Wrapper)] +#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] struct Overseer { - #[subsystem(X)] - #[subsystem(Z)] + #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, - #[subsystem(Orange)] - shots_of: TequilaInABar, - - other: Stuff, + #[subsystem(MsgStrukt2)] + sub1: AwesomeSubSys, } -struct Spawner; +#[derive(Debug, Clone)] +struct DummySpawner; -fn main() { - let overseer = Overseer::>::builder() - .sub0(FooSubSys::default()) - .build(Spawner); +struct DummyCtx; - // try to replace one subsystem with another that can not handle `X`. - // since it's missing the trait bound. - let overseer = overseer.replace_sub0(TequilaInABar::default()); +fn main() { + let overseer = Overseer::<_,_>::builder() + .sub0(AwesomeSubSys::default()) + .i_like_pie(std::f64::consts::PI) + .spawner(DummySpawner) + .build(|| -> DummyCtx { DummyCtx } ); } diff --git a/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs b/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs index 4b73ad27e667..8f6a436c5743 100644 --- a/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs @@ -2,29 +2,31 @@ use polkadot_overseer_gen::overlord; -struct X; - -#[derive(Default, Clone, Copy)] +#[derive(Default)] struct AwesomeSubSys; -#[derive(Default, Clone, Copy)] struct SigSigSig; -#[derive(Default, Clone, Copy)] -struct EventX; +struct Event; + +#[derive(Clone)] +struct MsgStrukt(u8); -#[overlord(signal=self::SigSigSig, event=self::EventX, gen=AllMessages)] +#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] struct Overseer { - #[subsystem(X)] + #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, } -struct Spawner; +#[derive(Debug, Clone)] +struct DummySpawner; + +struct DummyCtx; fn main() { - let overseer = Overseer::>::builder() + let overseer = Overseer::<_,_>::builder() .sub0(AwesomeSubSys::default()) - .build(Spawner); - - let overseer = overseer.replace_sub0(TequilaInABar::default()); + .i_like_pie(std::f64::consts::PI) + .spawner(DummySpawner) + .build(|| -> DummyCtx { DummyCtx } ); } diff --git a/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs b/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs index 4b3b7b46e725..f92df3abcb06 100644 --- a/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs @@ -1,27 +1,35 @@ #![allow(dead_code)] use polkadot_overseer_gen::overlord; -trait MetaMeta {} -#[derive(Debug)] -struct MsgStrukt(u8); - -#[derive(Default, Clone, Copy)] +#[derive(Default)] struct AwesomeSubSys; -#[overlord(Wrapper)] +struct SigSigSig; + +struct Event; + #[derive(Clone)] -struct Overseer { +struct MsgStrukt(u8); + +#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] +struct Overseer { #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, something_else: T, } -struct Spawner; +#[derive(Debug, Clone)] +struct DummySpawner; + +struct DummyCtx; + fn main() { - let overseer = Overseer::>::builder() + let overseer = Overseer::<_,_,_>::builder() .sub0(AwesomeSubSys::default()) - .build(Spawner); + .something_else(7777u32) + .spawner(DummySpawner) + .build(|| -> DummyCtx { DummyCtx } ); } diff --git a/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs b/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs index 2e61bf7fbd46..3fe6d0fa75ba 100644 --- a/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs +++ b/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs @@ -2,31 +2,33 @@ use polkadot_overseer_gen::overlord; -#[derive(Debug)] -struct MsgStrukt(u8); - -#[derive(Default, Clone, Copy)] +#[derive(Default)] struct AwesomeSubSys; -#[derive(Default, Clone, Copy)] struct SigSigSig; -#[overlord(signal=SigSigSig, gen=AllMessages)] -struct Overseer { + +struct Event; + +#[derive(Clone)] +struct MsgStrukt(u8); + +#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] +struct Overseer { #[subsystem(no_dispatch, MsgStrukt)] sub0: AwesomeSubSys, - something_else: T, + i_like_pie: f64, } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] struct DummySpawner; -struct Ctx; +struct DummyCtx; fn main() { - let overseer = Overseer::<_>::builder() + let overseer = Overseer::<_,_>::builder() .sub0(AwesomeSubSys::default()) - .something_else(7777u32) + .i_like_pie(std::f64::consts::PI) .spawner(DummySpawner) - .build(|| -> Ctx { Ctx } ); + .build(|| -> DummyCtx { DummyCtx } ); } diff --git a/node/overseer/overseer-gen/tests/ui/ok-04-blocking.rs b/node/overseer/overseer-gen/tests/ui/ok-04-blocking.rs new file mode 100644 index 000000000000..921ee5c3e19d --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/ok-04-blocking.rs @@ -0,0 +1,34 @@ +#![allow(dead_code)] + +use polkadot_overseer_gen::overlord; + +#[derive(Default)] +struct BlockingSubSys; + +struct SigSigSig; + +struct Event; + +#[derive(Clone)] +struct MsgStrukt(u8); + +#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] +struct Overseer { + #[subsystem(blocking, MsgStrukt)] + sub0: BlockingSubSys, + + i_like_pie: f64, +} + +#[derive(Debug, Clone)] +struct DummySpawner; + +struct DummyCtx; + +fn main() { + let overseer = Overseer::<_,_>::builder() + .sub0(BlockingSubSys::default()) + .i_like_pie(std::f64::consts::PI) + .spawner(DummySpawner) + .build(|| -> DummyCtx { DummyCtx } ); +} From 16f704d4b5c585c8270c786b38049931e8cffbe4 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 12 May 2021 18:32:12 +0200 Subject: [PATCH 030/161] almost there --- node/overseer/overseer-gen/examples/dummy.rs | 4 +- .../proc-macro/src/impl_overseer.rs | 39 +++++++----- .../proc-macro/src/impl_replace.rs | 2 +- .../overseer-gen/proc-macro/src/lib.rs | 2 +- .../proc-macro/src/parse_struct.rs | 6 +- node/subsystem/src/errors.rs | 63 +++++++++++++++++++ 6 files changed, 95 insertions(+), 21 deletions(-) diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index 021a2bbaca45..d2f677ad3d2c 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -17,7 +17,7 @@ impl std::error::Error for Yikes {} struct MsgStrukt(u8); #[overlord(signal=SigSigSig, event=Event, error=Yikes, gen=AllMessages)] -struct Overseer { +struct Xxx { #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, } @@ -28,7 +28,7 @@ struct DummySpawner; struct DummyCtx; fn main() { - let overseer = Overseer::<_,_>::builder() + let overseer = Xxx::builder() .sub0(AwesomeSubSys::default()) .i_like_pie(std::f64::consts::PI) .spawner(DummySpawner) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index ba9bcea78201..d37a1177a7cf 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -10,7 +10,7 @@ pub(crate) fn impl_overseer_struct( let overseer_name = dbg!(info.overseer_name.clone()); let subsystem_name = &info.subsystem_names(); - let subsystem_generic_ty = &info.subsystem_generic_types(); + let builder_generic_ty = &info.builder_generic_types(); let baggage_name = &info.baggage_names(); let baggage_ty = &info.baggage_types(); @@ -25,7 +25,6 @@ pub(crate) fn impl_overseer_struct( where Ctx: SubsystemContext, S: ::polkadot_overseer_gen::SpawnNamed, - #( #subsystem_generic_ty : Subsystem, )* }; let consumes = &info.consumes(); @@ -108,7 +107,7 @@ pub(crate) fn impl_builder( let handler = Ident::new(&(overseer_name.to_string() + "Handler"), overseer_name.span()); let subsystem_name = &info.subsystem_names(); - let subsystem_generic_ty = &info.subsystem_generic_types(); + let builder_generic_ty = &info.builder_generic_types(); let channel_name = &info.channel_names(""); let channel_name_unbounded = &info.channel_names("_unbounded"); @@ -129,18 +128,25 @@ pub(crate) fn impl_builder( }; let builder_generics = quote! { - < Ctx, S, #( #baggage_generic_ty, )* #( #subsystem_generic_ty, )* > + < Ctx, S, #( #baggage_generic_ty, )* #( #builder_generic_ty, )* > }; let builder_additional_generics = quote! { - < #( #subsystem_generic_ty, )* > + < #( #builder_generic_ty, )* > }; let where_clause = quote! { where Ctx: SubsystemContext, S: ::polkadot_overseer_gen::SpawnNamed, - #( #subsystem_generic_ty : Subsystem, )* + }; + + + let builder_where_clause = quote! { + where + Ctx: SubsystemContext, + S: ::polkadot_overseer_gen::SpawnNamed, + #( #builder_generic_ty : Subsystem, )* }; let consumes = &info.consumes(); @@ -162,7 +168,7 @@ pub(crate) fn impl_builder( #[derive(Debug, Clone, Default)] struct #builder #builder_generics { #( - #subsystem_name : ::std::option::Option< #subsystem_generic_ty >, + #subsystem_name : ::std::option::Option< #builder_generic_ty >, )* #( #baggage_name : ::std::option::Option< #baggage_name >, @@ -170,9 +176,9 @@ pub(crate) fn impl_builder( spawner: ::std::option::Option< S >, } - impl #builder_generics #builder #builder_generics #where_clause { + impl #builder_generics #builder #builder_generics #builder_where_clause { #( - pub fn #subsystem_name (mut self, subsystem: #subsystem_generic_ty ) -> Self { + pub fn #subsystem_name (mut self, subsystem: #builder_generic_ty ) -> Self { self. #subsystem_name = Some( subsystem ); self } @@ -187,7 +193,7 @@ pub(crate) fn impl_builder( ) -> Ctx, { - let (events_tx, events_rx) = ::polkadot_overseer_gen:metered::channel(SIGNAL_CHANNEL_CAPACITY); + let (events_tx, events_rx) = ::polkadot_overseer_gen::metered::channel(SIGNAL_CHANNEL_CAPACITY); let handler: #handler = events_tx.clone(); @@ -212,7 +218,7 @@ pub(crate) fn impl_builder( } }; - let spawner = &mut overseer.spawner; + let spawner = &mut self.spawner; let mut running_subsystems = ::polkadot_overseer_gen::FuturesUnordered::>>::new(); @@ -220,7 +226,7 @@ pub(crate) fn impl_builder( // FIXME generate a builder pattern that ensures this let #subsystem_name = self. #subsystem_name .expect("All subsystem must exist with the builder pattern."); - let #subsystem_name: OverseenSubsystem< #message_wrapper > = { + let #subsystem_name: OverseenSubsystem< #message_wrapper > = { let unbounded_meter = channels_out. #channel_name .meter().clone(); @@ -243,7 +249,7 @@ pub(crate) fn impl_builder( let ::polkadot_overseer_gen::SpawnedSubsystem { future, name } = #subsystem_name .start(ctx); - let (terminated_tx, terminated_rx) = oneshot::channel(); + let (terminated_tx, terminated_rx) = ::polkadot_overseer_gen::oneshot::channel(); let fut = Box::pin(async move { if let Err(e) = future.await { @@ -282,13 +288,18 @@ pub(crate) fn impl_builder( }; )* + + #( + let #baggage_name = self. #baggage_name .expect("Baggage must initialize"); + )* + let overseer = #overseer_name :: #generics { #( #subsystem_name, )* #( - #baggage_name : self. #baggage_name .unwrap(), + #baggage_name, )* spawner, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs index 714947fdbc9d..29162226e99b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs @@ -12,7 +12,7 @@ pub(crate) fn impl_replacable_subsystem( let overseer_name = &info.overseer_name; - let field_ty = &info.subsystem_generic_types(); + let field_ty = &info.builder_generic_types(); let baggage_generic_ty = &info.baggage_generic_types(); let baggage_name = &info.baggage_names(); diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index ea163c4a84f7..088819f7363b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -64,7 +64,7 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< let mut additive = impl_overseer_struct(&info)?; additive.extend(impl_message_wrapper_enum(&info)?); additive.extend(impl_channels_out_struct(&info)?); - additive.extend(impl_replacable_subsystem(&info)?); + // additive.extend(impl_replacable_subsystem(&info)?); additive.extend(impl_dispatch(&info)?); Ok(additive) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 66dbe3f67f32..2ad8629ea06d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -151,13 +151,13 @@ impl OverseerInfo { self.baggage.iter().map(|bag| bag.field_ty.clone()).collect::>() } - - pub(crate) fn subsystem_generic_types(&self) -> Vec { + /// Generic types per subsystem, in the form `Sub#N`. + pub(crate) fn builder_generic_types(&self) -> Vec { self.subsystems.iter().map(|sff| sff.generic.clone()).collect::>() } pub(crate) fn baggage_generic_types(&self) -> Vec { - self.baggage.iter().filter(|bag| bag.generic).map(|bag| bag.field_ty.clone()).collect::>() + self.baggage.iter().filter(|bag| bag.generic).map(|bag| dbg!(bag.field_ty.clone())).collect::>() } pub(crate) fn channel_names(&self, suffix: &'static str) -> Vec { diff --git a/node/subsystem/src/errors.rs b/node/subsystem/src/errors.rs index acd33cff1dfb..89c21426a53d 100644 --- a/node/subsystem/src/errors.rs +++ b/node/subsystem/src/errors.rs @@ -77,3 +77,66 @@ impl std::fmt::Display for RecoveryError { } impl std::error::Error for RecoveryError {} + + + + +/// An error type that describes faults that may happen +/// +/// These are: +/// * Channels being closed +/// * Subsystems dying when they are not expected to +/// * Subsystems not dying when they are told to die +/// * etc. +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum SubsystemError { + #[error(transparent)] + NotifyCancellation(#[from] oneshot::Canceled), + + #[error(transparent)] + QueueError(#[from] mpsc::SendError), + + #[error(transparent)] + TaskSpawn(#[from] futures::task::SpawnError), + + #[error(transparent)] + Infallible(#[from] std::convert::Infallible), + + #[error(transparent)] + Prometheus(#[from] substrate_prometheus_endpoint::PrometheusError), + + #[error(transparent)] + Jaeger(#[from] JaegerError), + + #[error("Failed to {0}")] + Context(String), + + #[error("Subsystem stalled: {0}")] + SubsystemStalled(&'static str), + + /// Per origin (or subsystem) annotations to wrap an error. + #[error("Error originated in {origin}")] + FromOrigin { + /// An additional anotation tag for the origin of `source`. + origin: &'static str, + /// The wrapped error. Marked as source for tracking the error chain. + #[source] source: Box + }, +} + +// impl AnnotateErrorOrigin for SubsystemError { +// fn with_origin(self, origin: &'static str) -> Self { +// Self::FromOrigin { +// origin, +// source: Box::new(self), +// } +// } +// } + +impl SubsystemError { + /// Adds a `str` as `origin` to the given error `err`. + pub fn with_origin(origin: &'static str, err: E) -> Self { + Self::FromOrigin { origin, source: Box::new(err) } + } +} From 705bc2f0a4f1da7247c78f5a2589d6b499d6b3b6 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 17 May 2021 15:27:31 +0200 Subject: [PATCH 031/161] fixins --- Cargo.lock | 4 +-- node/overseer/overseer-gen/examples/dummy.rs | 16 +++++++++++ .../proc-macro/src/impl_overseer.rs | 28 +++++++++---------- .../overseer-gen/proc-macro/src/lib.rs | 1 - node/overseer/overseer-gen/src/lib.rs | 12 ++++++-- 5 files changed, 42 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dbc0736c8f2d..f1e1c77553aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6986,9 +6986,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" dependencies = [ "unicode-xid", ] diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index d2f677ad3d2c..968c0f0c18fc 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -5,14 +5,30 @@ use polkadot_overseer_gen::*; #[derive(Default)] struct AwesomeSubSys; +#[derive(Debug)] struct SigSigSig; + +#[derive(Debug)] struct Event; +#[derive(Debug)] struct Yikes; +impl std::fmt::Display for Yikes { + fn fmt(f: std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "yikes!") + } +} + impl std::error::Error for Yikes {} +impl From for Yikes { + fn from(_: polkadot_overseer_gen::SubsystemError) -> Yikes { + Yikes + } +} + #[derive(Clone)] struct MsgStrukt(u8); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index d37a1177a7cf..0a53d0996baf 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -68,24 +68,24 @@ pub(crate) fn impl_overseer_struct( pub async fn stop(mut self) { unimplemented!("Stopping is not yet implemented") } - } - - pub async fn broadcast_signal(&mut self, signal: #signal_ty) -> SubsystemResult<()> { - #( - self. #subsystem_name .send_signal(signal.clone()).await; - )* - let _ = signal; - - Ok(()) - } - pub async fn route_message(&mut self, msg: #message_wrapper) -> SubsystemResult<()> { - match msg { + pub async fn broadcast_signal(&mut self, signal: #signal_ty) -> SubsystemResult<()> { #( - #message_wrapper :: #consumes (msg) => self. #subsystem_name .send_message(msg).await?, + self. #subsystem_name .send_signal(signal.clone()).await; )* + let _ = signal; + + Ok(()) + } + + pub async fn route_message(&mut self, msg: #message_wrapper) -> SubsystemResult<()> { + match msg { + #( + #message_wrapper :: #consumes (msg) => self. #subsystem_name .send_message(msg).await?, + )* + } + Ok(()) } - Ok(()) } }; diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 088819f7363b..97d696759ef1 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -64,7 +64,6 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< let mut additive = impl_overseer_struct(&info)?; additive.extend(impl_message_wrapper_enum(&info)?); additive.extend(impl_channels_out_struct(&info)?); - // additive.extend(impl_replacable_subsystem(&info)?); additive.extend(impl_dispatch(&info)?); Ok(additive) diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index de81d5af8231..f1b09eacfd8b 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -49,7 +49,7 @@ //! V V //! ..................| Overseer "runs" these |....................... //! . +--------------------+ +---------------------+ . -//! . | SubsystemInstance1 | | SubsystemInstance2 | . +//! . | SubsystemInstance1 | <-- bidir --> | SubsystemInstance2 | . //! . +--------------------+ +---------------------+ . //! .................................................................. //! ``` @@ -57,7 +57,7 @@ // #![deny(unused_results)] // unused dependencies can not work for test and examples at the same time // yielding false positives -#![warn(missing_docs)] +#![deny(missing_docs)] pub use overseer_gen_proc_macro::*; #[doc(hidden)] @@ -173,6 +173,13 @@ impl SignalsReceived { /// A trait to support the origin annotation /// such that errors across subsystems can be easier tracked. pub trait AnnotateErrorOrigin: 'static + Send + Sync + std::error::Error { + /// Annotate the error with a origin `str`. + /// + /// Commonly this is used to create nested enum variants. + /// + /// ```rust,ignore + /// E::WithOrigin("I am originally from Cowtown.", E::Variant) + /// ``` fn with_origin(self, origin: &'static str) -> Self; } @@ -233,6 +240,7 @@ pub enum SubsystemError { }, } +/// Alias for a result with error type `SubsystemError`. pub type SubsystemResult = std::result::Result; /// Collection of meters related to a subsystem. From ebf71f0ed07dd1750f71698ed9ff2aba81c11c96 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 18 May 2021 14:08:43 +0200 Subject: [PATCH 032/161] fffffixins --- Cargo.lock | 12 -- node/overseer/Cargo.toml | 1 - node/overseer/overseer-gen/examples/dummy.rs | 3 +- .../proc-macro/src/impl_channels_out.rs | 1 + .../proc-macro/src/impl_overseer.rs | 73 ++++--- node/overseer/subsystems-gen/Cargo.toml | 18 -- node/overseer/subsystems-gen/src/lib.rs | 179 ------------------ .../subsystems-gen/tests/ui/err-01-enum.rs | 13 -- .../tests/ui/err-01-enum.stderr | 5 - .../tests/ui/err-01-generic-used-twice.rs | 17 -- .../tests/ui/err-01-generic-used-twice.stderr | 14 -- .../tests/ui/err-01-no-generic.rs | 17 -- .../tests/ui/err-01-no-generics.stderr | 14 -- .../tests/ui/ok-01-w-generics.rs | 17 -- 14 files changed, 53 insertions(+), 331 deletions(-) delete mode 100644 node/overseer/subsystems-gen/Cargo.toml delete mode 100644 node/overseer/subsystems-gen/src/lib.rs delete mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-enum.rs delete mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-enum.stderr delete mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.rs delete mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.stderr delete mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-no-generic.rs delete mode 100644 node/overseer/subsystems-gen/tests/ui/err-01-no-generics.stderr delete mode 100644 node/overseer/subsystems-gen/tests/ui/ok-01-w-generics.rs diff --git a/Cargo.lock b/Cargo.lock index f1e1c77553aa..9556d473ba76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6252,7 +6252,6 @@ dependencies = [ "polkadot-node-subsystem", "polkadot-node-subsystem-util", "polkadot-primitives", - "polkadot-procmacro-overseer-subsystems-gen", "sc-client-api", "sp-api", "sp-core", @@ -6329,17 +6328,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "polkadot-procmacro-overseer-subsystems-gen" -version = "0.1.0" -dependencies = [ - "assert_matches", - "proc-macro2", - "quote", - "syn", - "trybuild", -] - [[package]] name = "polkadot-procmacro-subsystem-dispatch-gen" version = "0.1.0" diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml index 3ecb90f45ffa..674c22eb8069 100644 --- a/node/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -12,7 +12,6 @@ futures = "0.3.12" futures-timer = "3.0.2" polkadot-node-primitives = { package = "polkadot-node-primitives", path = "../primitives" } polkadot-node-subsystem-util = { path = "../subsystem-util" } -polkadot-procmacro-overseer-subsystems-gen = { path = "./subsystems-gen" } polkadot-primitives = { path = "../../primitives" } polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../subsystem" } tracing = "0.1.25" diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index 968c0f0c18fc..3e2f1ac4d829 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -16,7 +16,7 @@ struct Event; struct Yikes; impl std::fmt::Display for Yikes { - fn fmt(f: std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, mut f: std::fmt::Formatter) -> std::fmt::Result { writeln!(f, "yikes!") } } @@ -41,6 +41,7 @@ struct Xxx { #[derive(Debug, Clone)] struct DummySpawner; +#[derive(Debug, Clone)] struct DummyCtx; fn main() { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index f7af13faf0ec..9c1a93254081 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -17,6 +17,7 @@ pub(crate) fn impl_channels_out_struct( let consumes = &info.consumes(); let ts = quote! { + #[derive(Debug, Clone)] pub struct ChannelsOut { #( pub #channel_name: ::polkadot_overseer_gen::metered::MeteredSender>, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 0a53d0996baf..c0dd2d15d8f8 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -18,14 +18,14 @@ pub(crate) fn impl_overseer_struct( let baggage_generic_ty = &info.baggage_generic_types(); let generics = quote! { - < Ctx, S, #( #baggage_generic_ty, )* > + < S, #( #baggage_generic_ty, )* > }; let where_clause = quote! { where - Ctx: SubsystemContext, S: ::polkadot_overseer_gen::SpawnNamed, }; + // FIXME add where clauses for baggage types let consumes = &info.consumes(); @@ -37,9 +37,17 @@ pub(crate) fn impl_overseer_struct( let log_target = syn::LitStr::new(overseer_name.to_string().to_lowercase().as_str(), overseer_name.span()); let mut ts = quote! { + /// Capacity of a bounded message channel between overseer and subsystem + /// but also for bounded channels between two subsystems. const CHANNEL_CAPACITY: usize = #message_channel_capacity; + + /// Capacity of a signal channel between a subsystem and the overseer. const SIGNAL_CHANNEL_CAPACITY: usize = #signal_channel_capacity; + + /// The log target tag. const LOG_TARGET: &'static str = #log_target; + + /// The overseer. pub struct #overseer_name #generics { // Subsystem instances. #( @@ -98,7 +106,8 @@ pub(crate) fn impl_overseer_struct( Ok(ts) } -/// Implement a builder pattern. +/// Implement a builder pattern for the `Overseer`-type, +/// which acts as the gateway to constructing the overseer. pub(crate) fn impl_builder( info: &OverseerInfo, ) -> Result { @@ -124,21 +133,21 @@ pub(crate) fn impl_builder( let blocking = &info.subsystems().iter().map(|x| x.blocking).collect::>(); let generics = quote! { - < Ctx, S, #( #baggage_generic_ty, )* > + < S, #( #baggage_generic_ty, )* > + }; + let where_clause = quote! { + where + S: ::polkadot_overseer_gen::SpawnNamed, }; let builder_generics = quote! { - < Ctx, S, #( #baggage_generic_ty, )* #( #builder_generic_ty, )* > + }; + // all subsystems must have the same context + // even if the overseer does not impose such a limit. let builder_additional_generics = quote! { - < #( #builder_generic_ty, )* > - }; - - let where_clause = quote! { - where - Ctx: SubsystemContext, - S: ::polkadot_overseer_gen::SpawnNamed, + < Ctx, #( #builder_generic_ty, )* > }; @@ -157,7 +166,9 @@ pub(crate) fn impl_builder( let ts = quote! { impl #generics #overseer_name #generics #where_clause { - fn builder::< #builder_additional_generics >() -> #builder #builder_generics { + fn builder #builder_additional_generics () -> #builder #builder_generics + #builder_where_clause + { #builder :: default() } } @@ -165,7 +176,7 @@ pub(crate) fn impl_builder( pub type #handler = ::polkadot_overseer_gen::metered::UnboundedMeteredSender< #event >; - #[derive(Debug, Clone, Default)] + #[derive(Debug, Clone)] struct #builder #builder_generics { #( #subsystem_name : ::std::option::Option< #builder_generic_ty >, @@ -174,6 +185,22 @@ pub(crate) fn impl_builder( #baggage_name : ::std::option::Option< #baggage_name >, )* spawner: ::std::option::Option< S >, + _phantom_ctx: ::std::marker::PhantomData< Ctx >, + } + + impl #builder_generics Default for #builder #builder_generics { + fn default() -> Self { + Self { + #( + #subsystem_name : ::std::option::Option< #builder_generic_ty >, + )* + #( + #baggage_name : ::std::option::Option< #baggage_name >, + )* + spawner: None, + _phantom_ctx: ::std::marker::PhantomData, + } + } } impl #builder_generics #builder #builder_generics #builder_where_clause { @@ -185,12 +212,13 @@ pub(crate) fn impl_builder( )* pub fn build(mut self, create_subsystem_ctx: F) -> (#overseer_name #generics, #handler) - where F: FnMut( - ::polkadot_overseer_gen::metered::MeteredReceiver< #signal >, - SubsystemIncomingMessages< #message_wrapper >, - ChannelsOut, - ::polkadot_overseer_gen::metered::UnboundedMeteredSender, - ) -> Ctx, + where + F: FnMut( + ::polkadot_overseer_gen::metered::MeteredReceiver< #signal >, + SubsystemIncomingMessages< #message_wrapper >, + ChannelsOut, + ::polkadot_overseer_gen::metered::UnboundedMeteredSender, + ) -> Ctx, { let (events_tx, events_rx) = ::polkadot_overseer_gen::metered::channel(SIGNAL_CHANNEL_CAPACITY); @@ -288,12 +316,11 @@ pub(crate) fn impl_builder( }; )* - #( - let #baggage_name = self. #baggage_name .expect("Baggage must initialize"); + let #baggage_name = self. #baggage_name .expect("Baggage must initialized"); )* - let overseer = #overseer_name :: #generics { + let overseer = #overseer_name { #( #subsystem_name, )* diff --git a/node/overseer/subsystems-gen/Cargo.toml b/node/overseer/subsystems-gen/Cargo.toml deleted file mode 100644 index 2720663c293d..000000000000 --- a/node/overseer/subsystems-gen/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "polkadot-procmacro-overseer-subsystems-gen" -version = "0.1.0" -authors = ["Parity Technologies "] -edition = "2018" -description = "Small proc macro to create mocking level iface type helpers" - -[lib] -proc-macro = true - -[dependencies] -syn = { version = "1.0.60", features = ["full", "extra-traits"] } -quote = "1.0.9" -proc-macro2 = "1.0.24" -assert_matches = "1.5.0" - -[dev-dependencies] -trybuild = "1.0.41" diff --git a/node/overseer/subsystems-gen/src/lib.rs b/node/overseer/subsystems-gen/src/lib.rs deleted file mode 100644 index c15d08bb04f4..000000000000 --- a/node/overseer/subsystems-gen/src/lib.rs +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use std::collections::HashSet; - -use proc_macro2::TokenStream; -use quote::quote; - -use syn::{Error, GenericParam, Ident, Result, Type, parse2}; - -#[proc_macro_derive(AllSubsystemsGen)] -pub fn subsystems_gen(item: proc_macro::TokenStream) -> proc_macro::TokenStream { - let item: TokenStream = item.into(); - impl_subsystems_gen(item).unwrap_or_else(|err| err.to_compile_error()).into() -} - -fn impl_subsystems_gen(item: TokenStream) -> Result { - let span = proc_macro2::Span::call_site(); - let ds = parse2::(item.clone())?; - - match ds.fields { - syn::Fields::Named(named) => { - #[derive(Clone)] - struct NameTyTup { - field: Ident, - ty: Type, - } - let mut orig_generics = ds.generics; - // remove default types - orig_generics.params = orig_generics.params.into_iter().map(|mut generic| { - match generic { - GenericParam::Type(ref mut param) => { - param.eq_token = None; - param.default = None; - } - _ => {} - } - generic - }).collect(); - - // prepare a hashmap of generic type to member that uses it - let generic_types = orig_generics.params.iter().filter_map(|generic| { - if let GenericParam::Type(param) = generic { - Some(param.ident.clone()) - } else { - None - } - }).collect::>(); - - let strukt_ty = ds.ident; - - if generic_types.is_empty() { - return Err(Error::new(strukt_ty.span(), "struct must have at least one generic parameter.")) - } - - // collect all fields that exist, and all fields that are replaceable - let mut replacable_items = Vec::::with_capacity(64); - let mut all_fields = replacable_items.clone(); - - - let mut duplicate_generic_detection = HashSet::::with_capacity(64); - - for field in named.named { - let field_ident = field.ident.clone().ok_or_else(|| Error::new(span, "Member field must have a name."))?; - let ty = field.ty.clone(); - let ntt = NameTyTup { field: field_ident, ty }; - - replacable_items.push(ntt.clone()); - - - // assure every generic is used exactly once - let ty_ident = match field.ty { - Type::Path(path) => path.path.get_ident().cloned().ok_or_else(|| { - Error::new(proc_macro2::Span::call_site(), "Expected an identifier, but got a path.") - }), - _ => return Err(Error::new(proc_macro2::Span::call_site(), "Must be path.")) - }?; - - if generic_types.contains(&ty_ident) { - if let Some(previous) = duplicate_generic_detection.replace(ty_ident) { - return Err(Error::new(previous.span(), "Generic type parameters may only be used for exactly one field, but is used more than once.")) - } - } - - all_fields.push(ntt); - } - - - let msg = "Generated by #[derive(AllSubsystemsGen)] derive proc-macro."; - let mut additive = TokenStream::new(); - - // generate an impl of `fn replace_#name` - for NameTyTup { field: replacable_item, ty: replacable_item_ty } in replacable_items { - let keeper = all_fields.iter().filter(|ntt| ntt.field != replacable_item).map(|ntt| ntt.field.clone()); - let strukt_ty = strukt_ty.clone(); - let fname = Ident::new(&format!("replace_{}", replacable_item), span); - // adjust the generics such that the appropriate member type is replaced - let mut modified_generics = orig_generics.clone(); - modified_generics.params = modified_generics.params.into_iter().map(|mut generic| { - match generic { - GenericParam::Type(ref mut param) => { - param.eq_token = None; - param.default = None; - if match &replacable_item_ty { - Type::Path(path) => - path.path.get_ident().filter(|&ident| ident == ¶m.ident).is_some(), - _ => false - } { - param.ident = Ident::new("NEW", span); - } - } - _ => {} - } - generic - }).collect(); - - additive.extend(quote! { - impl #orig_generics #strukt_ty #orig_generics { - #[doc = #msg] - pub fn #fname < NEW > (self, replacement: NEW) -> #strukt_ty #modified_generics { - #strukt_ty :: #modified_generics { - #replacable_item: replacement, - #( - #keeper: self.#keeper, - )* - } - } - } - }); - } - - Ok(additive) - } - syn::Fields::Unit => Err(Error::new(span, "Must be a struct with named fields. Not an unit struct.")), - syn::Fields::Unnamed(_) => { - Err(Error::new(span, "Must be a struct with named fields. Not an unnamed fields struct.")) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn basic() { - let item = quote! { - pub struct AllSubsystems { - pub a: A, - pub beee: B, - pub dj: CD, - } - }; - - let output = impl_subsystems_gen(item).expect("Simple example always works. qed"); - println!("//generated:"); - println!("{}", output); - } - - #[test] - fn ui() { - let t = trybuild::TestCases::new(); - t.compile_fail("tests/ui/err-*.rs"); - t.pass("tests/ui/ok-*.rs"); - } -} diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-enum.rs b/node/overseer/subsystems-gen/tests/ui/err-01-enum.rs deleted file mode 100644 index 318636279ea5..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/err-01-enum.rs +++ /dev/null @@ -1,13 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; - -#[derive(Clone, AllSubsystemsGen)] -enum AllSubsystems { - A(A), - B(B), -} - -fn main() { - let all = AllSubsystems::::A(0u8); -} diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-enum.stderr b/node/overseer/subsystems-gen/tests/ui/err-01-enum.stderr deleted file mode 100644 index 5f61df1057cb..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/err-01-enum.stderr +++ /dev/null @@ -1,5 +0,0 @@ -error: expected `struct` - --> $DIR/err-01-enum.rs:6:1 - | -6 | enum AllSubsystems { - | ^^^^ diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.rs b/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.rs deleted file mode 100644 index f89939d5c306..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.rs +++ /dev/null @@ -1,17 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; - -#[derive(Clone, AllSubsystemsGen)] -struct AllSubsystems { - a: X, - b: X, -} - -fn main() { - let all = AllSubsystems:: { - a: 0_u16, - b: 1_u16, - }; - let _all = all.replace_a(77u8); -} diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.stderr b/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.stderr deleted file mode 100644 index 23e1404ff822..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/err-01-generic-used-twice.stderr +++ /dev/null @@ -1,14 +0,0 @@ -error: Generic type parameters may only be used for exactly one field, but is used more than once. - --> $DIR/err-01-generic-used-twice.rs:7:5 - | -7 | a: X, - | ^ - -error[E0599]: no method named `replace_a` found for struct `AllSubsystems` in the current scope - --> $DIR/err-01-generic-used-twice.rs:16:17 - | -6 | struct AllSubsystems { - | ----------------------- method `replace_a` not found for this -... -16 | let _all = all.replace_a(77u8); - | ^^^^^^^^^ method not found in `AllSubsystems` diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-no-generic.rs b/node/overseer/subsystems-gen/tests/ui/err-01-no-generic.rs deleted file mode 100644 index 0466eb444cd9..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/err-01-no-generic.rs +++ /dev/null @@ -1,17 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; - -#[derive(Clone, AllSubsystemsGen)] -struct AllSubsystems { - a: f32, - b: u16, -} - -fn main() { - let all = AllSubsystems { - a: 0_f32, - b: 1_u16, - }; - let _all = all.replace_a(77u8); -} diff --git a/node/overseer/subsystems-gen/tests/ui/err-01-no-generics.stderr b/node/overseer/subsystems-gen/tests/ui/err-01-no-generics.stderr deleted file mode 100644 index 1de880ae433c..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/err-01-no-generics.stderr +++ /dev/null @@ -1,14 +0,0 @@ -error: Generic type parameters may only be used once have at least one generic parameter. - --> $DIR/err-01-no-generics.rs:7:5 - | -7 | a: X, - | ^ - -error[E0599]: no method named `replace_a` found for struct `AllSubsystems` in the current scope - --> $DIR/err-01-no-generics.rs:16:17 - | -6 | struct AllSubsystems { - | ----------------------- method `replace_a` not found for this -... -16 | let _all = all.replace_a(77u8); - | ^^^^^^^^^ method not found in `AllSubsystems` diff --git a/node/overseer/subsystems-gen/tests/ui/ok-01-w-generics.rs b/node/overseer/subsystems-gen/tests/ui/ok-01-w-generics.rs deleted file mode 100644 index 1519990a0a55..000000000000 --- a/node/overseer/subsystems-gen/tests/ui/ok-01-w-generics.rs +++ /dev/null @@ -1,17 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; - -#[derive(Clone, AllSubsystemsGen)] -struct AllSubsystems { - a: A, - b: B, -} - -fn main() { - let all = AllSubsystems:: { - a: 0u8, - b: 1u16, - }; - let _all: AllSubsystems<_,_> = all.replace_a::(777_777u32); -} From 4a6ded320641737e7578160cd897bbe72996eb7e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 18 May 2021 14:32:49 +0200 Subject: [PATCH 033/161] foo --- node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index c0dd2d15d8f8..99537c7c4069 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -254,7 +254,7 @@ pub(crate) fn impl_builder( // FIXME generate a builder pattern that ensures this let #subsystem_name = self. #subsystem_name .expect("All subsystem must exist with the builder pattern."); - let #subsystem_name: OverseenSubsystem< #message_wrapper > = { + let #subsystem_name: OverseenSubsystem< #message_wrapper > = { let unbounded_meter = channels_out. #channel_name .meter().clone(); @@ -294,7 +294,10 @@ pub(crate) fn impl_builder( spawner.spawn(name, fut); } - running_subsystems.push(Box::pin(terminated_rx.map(|e| { tracing::warn!(err = ?e, "dropping error"); Ok(()) }))); + running_subsystems.push(Box::pin(terminated_rx.map(|e| { + ::polkadot_overseer_gen::tracing::warn!(err = ?e, "dropping error"); + Ok(()) + }))); let instance = Some( SubsystemInstance::< #message_wrapper > { From af5bdbb4781eb27955db0d3b9c6491df1b4ef29d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 18 May 2021 16:21:52 +0200 Subject: [PATCH 034/161] fixins --- node/overseer/overseer-gen/examples/dummy.rs | 18 ++++++++++++----- .../proc-macro/src/impl_channels_out.rs | 1 + .../proc-macro/src/impl_overseer.rs | 20 ++++++++++--------- .../overseer-gen/proc-macro/src/parse_attr.rs | 12 +++++------ .../proc-macro/src/parse_struct.rs | 15 +++++++------- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index 3e2f1ac4d829..e05a83ef9ab3 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -5,18 +5,18 @@ use polkadot_overseer_gen::*; #[derive(Default)] struct AwesomeSubSys; -#[derive(Debug)] +#[derive(Debug, Clone)] struct SigSigSig; -#[derive(Debug)] -struct Event; +#[derive(Debug, Clone)] +struct EvX; #[derive(Debug)] struct Yikes; impl std::fmt::Display for Yikes { - fn fmt(&self, mut f: std::fmt::Formatter) -> std::fmt::Result { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { writeln!(f, "yikes!") } } @@ -32,10 +32,17 @@ impl From for Yikes { #[derive(Clone)] struct MsgStrukt(u8); -#[overlord(signal=SigSigSig, event=Event, error=Yikes, gen=AllMessages)] +#[derive(Clone, Copy)] +struct Plinko; + + +#[overlord(signal=SigSigSig, event=EvX, error=Yikes, gen=AllMessages)] struct Xxx { #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, + + #[subsystem(Plinko)] + plinkos: AwesomeSubSys, } #[derive(Debug, Clone)] @@ -47,6 +54,7 @@ struct DummyCtx; fn main() { let overseer = Xxx::builder() .sub0(AwesomeSubSys::default()) + .plinkos(AwesomeSubSys::default()) .i_like_pie(std::f64::consts::PI) .spawner(DummySpawner) .build(|| -> DummyCtx { DummyCtx } ); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 9c1a93254081..31032b354bae 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -22,6 +22,7 @@ pub(crate) fn impl_channels_out_struct( #( pub #channel_name: ::polkadot_overseer_gen::metered::MeteredSender>, )* + #( pub #channel_name_unbounded: ::polkadot_overseer_gen::metered::UnboundedMeteredSender>, )* diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 99537c7c4069..70d5bce2d88a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -29,7 +29,9 @@ pub(crate) fn impl_overseer_struct( let consumes = &info.consumes(); - let signal_ty = &info.extern_event_ty; + let signal_ty = &info.extern_signal_ty; + + let event_ty = &info.extern_event_ty; let message_channel_capacity = info.message_channel_capacity; let signal_channel_capacity = info.signal_channel_capacity; @@ -69,7 +71,7 @@ pub(crate) fn impl_overseer_struct( to_overseer_rx: ::polkadot_overseer_gen::Fuse>, /// Events that are sent to the overseer from the outside world. - events_rx: ::polkadot_overseer_gen::metered::MeteredReceiver, + events_rx: ::polkadot_overseer_gen::metered::MeteredReceiver< #event_ty >, } impl #generics #overseer_name #generics #where_clause { @@ -192,10 +194,10 @@ pub(crate) fn impl_builder( fn default() -> Self { Self { #( - #subsystem_name : ::std::option::Option< #builder_generic_ty >, + #subsystem_name: None, )* #( - #baggage_name : ::std::option::Option< #baggage_name >, + #baggage_name: None, )* spawner: None, _phantom_ctx: ::std::marker::PhantomData, @@ -229,7 +231,8 @@ pub(crate) fn impl_builder( let channels_out = { #( - let (#channel_name_tx, #channel_name_rx) = ::polkadot_overseer_gen::metered::channel::>(CHANNEL_CAPACITY); + let (#channel_name_tx, #channel_name_rx) = ::polkadot_overseer_gen::metered::channel::< + MessagePacket< #consumes >>(CHANNEL_CAPACITY); )* #( @@ -246,7 +249,7 @@ pub(crate) fn impl_builder( } }; - let spawner = &mut self.spawner; + let spawner = self.spawner.expect("Spawner is set. qed"); let mut running_subsystems = ::polkadot_overseer_gen::FuturesUnordered::>>::new(); @@ -258,7 +261,7 @@ pub(crate) fn impl_builder( let unbounded_meter = channels_out. #channel_name .meter().clone(); - let message_tx: ::polkadot_overseer_gen::metered::MeteredSender> = channels_out. #channel_name .clone(); + let message_tx: ::polkadot_overseer_gen::metered::UnboundedMeteredSender> = channels_out. #channel_name .clone(); let message_rx: SubsystemIncomingMessages< #message_wrapper > = ::polkadot_overseer_gen::select( @@ -376,8 +379,7 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result()? + lit.base10_parse::()? } else { unreachable!() } } else { - 64 + 64_usize }; let message_channel_capacity = if let Some(item) = unique.remove(TAG_MESSAGE_CAPACITY) { if let AttrItem::MessageChannelCapacity(lit) = item { - lit.base10_parse::()? + lit.base10_parse::()? } else { unreachable!() } } else { - 1024 + 1024_usize }; let extern_error_ty = unique.remove(TAG_EXT_ERROR_TY) .map(|x| if let AttrItem::ExternErrorType(x) = x { x.clone() } else { unreachable!() } ) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 2ad8629ea06d..ed47cde6308e 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -22,7 +22,7 @@ pub(crate) struct SubSysField { /// Generate generic type name for the `AllSubsystems` type. pub(crate) generic: Ident, /// Type of the subsystem. - pub(crate) ty: Ident, + pub(crate) ty: Path, /// Type to be consumed by the subsystem. pub(crate) consumes: Ident, /// If `no_dispatch` is present, if the message is incomming via @@ -33,10 +33,10 @@ pub(crate) struct SubSysField { pub(crate) blocking: bool, } -fn try_type_to_ident(ty: Type, span: Span) -> Result { +fn try_type_to_path(ty: Type, span: Span) -> Result { match ty { Type::Path(path) => { - path.path.get_ident().cloned().ok_or_else(|| Error::new(span, "Expected an identifier, but got a path.")) + path } _ => Err(Error::new(span, "Type must be a path expression.")), } @@ -115,9 +115,9 @@ pub(crate) struct OverseerInfo { pub(crate) overseer_name: Ident, /// Size of the bounded channel. - pub(crate) message_channel_capacity: u64, + pub(crate) message_channel_capacity: usize, /// Size of the bounded signal channel. - pub(crate) signal_channel_capacity: u64, + pub(crate) signal_channel_capacity: usize, /// Signals to be sent, sparse information that is used intermittendly. pub(crate) extern_signal_ty: Path, @@ -226,13 +226,14 @@ impl OverseerGuts { subsystems.push(SubSysField { name: ident, generic: Ident::new(format!("Sub{}", idx).as_str(), span), - ty: try_type_to_ident(ty, span)?, + ty: try_type_to_path(ty, span)?, consumes: consumes_idents[0].clone(), no_dispatch: variant.no_dispatch, blocking: variant.blocking, }); } else { - let field_ty = try_type_to_ident(ty, ident.span())?; + let field_ty: Path = try_type_to_path(ty, ident.span())?; + field_ty. baggage.push(BaggageField { field_name: ident, generic: !baggage_generics.contains(&field_ty), From 6701179be98cc6d1a3baf42892b62f8f416db807 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 18 May 2021 16:22:12 +0200 Subject: [PATCH 035/161] apply --- Cargo.lock | 1 + node/overseer/Cargo.toml | 1 + node/overseer/src/lib.rs | 1057 ++++------------------------------ node/subsystem/src/errors.rs | 7 +- node/subsystem/src/lib.rs | 2 +- 5 files changed, 107 insertions(+), 961 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9556d473ba76..03cf8356bf10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6251,6 +6251,7 @@ dependencies = [ "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-util", + "polkadot-overseer-gen", "polkadot-primitives", "sc-client-api", "sp-api", diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml index 674c22eb8069..d714224f013a 100644 --- a/node/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -13,6 +13,7 @@ futures-timer = "3.0.2" polkadot-node-primitives = { package = "polkadot-node-primitives", path = "../primitives" } polkadot-node-subsystem-util = { path = "../subsystem-util" } polkadot-primitives = { path = "../../primitives" } +polkadot-overseer-gen = { package = "polkadot-overseer-gen", path = "./overseer-gen" } polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../subsystem" } tracing = "0.1.25" diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 6a674fb29e35..eee158eae1f3 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -92,9 +92,22 @@ pub use polkadot_subsystem::{ Subsystem, SubsystemContext, SubsystemSender, OverseerSignal, FromOverseer, SubsystemError, SubsystemResult, SpawnedSubsystem, ActiveLeavesUpdate, ActivatedLeaf, DummySubsystem, jaeger, }; -use polkadot_node_subsystem_util::{TimeoutExt, metrics::{self, prometheus}, metered, Metronome}; + +mod metrics; +use self::metrics::{Metrics, MetricsInner}; + +use polkadot_node_subsystem_util::{TimeoutExt, metrics::{prometheus, Metrics as MetricsTrait}, metered, Metronome}; use polkadot_node_primitives::SpawnNamed; -use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; +use polkadot_overseer_gen::{ + SubsystemMeterReadouts, + ChannelsOut, + OverseenSubsystem, + SubsystemMeters, + SubsystemIncomingMessages, + overlord, + OverseerSubsystemSender, +}; + // A capacity of bounded channels inside the overseer. const CHANNEL_CAPACITY: usize = 1024; @@ -144,7 +157,7 @@ impl HeadSupportsParachains for Arc where /// Each [`Subsystem`] is supposed to implement some interface that is generic over /// message type that is specific to this [`Subsystem`]. At the moment not all /// subsystems are implemented and the rest can be mocked with the [`DummySubsystem`]. -#[derive(Debug, Clone, AllSubsystemsGen)] +#[derive(Debug, Clone)] pub struct AllSubsystems< CV = (), CB = (), CS = (), SD = (), AD = (), AR = (), BS = (), BD = (), P = (), RA = (), AS = (), NB = (), CA = (), CG = (), CP = (), ApD = (), ApV = (), @@ -538,273 +551,6 @@ fn make_packet(signals_received: usize, message: T) -> MessagePacket { message, } } - -// The channels held by every subsystem to communicate with every other subsystem. -#[derive(Debug, Clone)] -struct ChannelsOut { - candidate_validation: metered::MeteredSender>, - candidate_backing: metered::MeteredSender>, - candidate_selection: metered::MeteredSender>, - statement_distribution: metered::MeteredSender>, - availability_distribution: metered::MeteredSender>, - availability_recovery: metered::MeteredSender>, - bitfield_signing: metered::MeteredSender>, - bitfield_distribution: metered::MeteredSender>, - provisioner: metered::MeteredSender>, - runtime_api: metered::MeteredSender>, - availability_store: metered::MeteredSender>, - network_bridge: metered::MeteredSender>, - chain_api: metered::MeteredSender>, - collation_generation: metered::MeteredSender>, - collator_protocol: metered::MeteredSender>, - approval_distribution: metered::MeteredSender>, - approval_voting: metered::MeteredSender>, - gossip_support: metered::MeteredSender>, - - candidate_validation_unbounded: metered::UnboundedMeteredSender>, - candidate_backing_unbounded: metered::UnboundedMeteredSender>, - candidate_selection_unbounded: metered::UnboundedMeteredSender>, - statement_distribution_unbounded: metered::UnboundedMeteredSender>, - availability_distribution_unbounded: metered::UnboundedMeteredSender>, - availability_recovery_unbounded: metered::UnboundedMeteredSender>, - bitfield_signing_unbounded: metered::UnboundedMeteredSender>, - bitfield_distribution_unbounded: metered::UnboundedMeteredSender>, - provisioner_unbounded: metered::UnboundedMeteredSender>, - runtime_api_unbounded: metered::UnboundedMeteredSender>, - availability_store_unbounded: metered::UnboundedMeteredSender>, - network_bridge_unbounded: metered::UnboundedMeteredSender>, - chain_api_unbounded: metered::UnboundedMeteredSender>, - collation_generation_unbounded: metered::UnboundedMeteredSender>, - collator_protocol_unbounded: metered::UnboundedMeteredSender>, - approval_distribution_unbounded: metered::UnboundedMeteredSender>, - approval_voting_unbounded: metered::UnboundedMeteredSender>, - gossip_support_unbounded: metered::UnboundedMeteredSender>, -} - -impl ChannelsOut { - async fn send_and_log_error( - &mut self, - signals_received: usize, - message: AllMessages, - ) { - let res = match message { - AllMessages::CandidateValidation(msg) => { - self.candidate_validation.send(make_packet(signals_received, msg)).await - }, - AllMessages::CandidateBacking(msg) => { - self.candidate_backing.send(make_packet(signals_received, msg)).await - }, - AllMessages::CandidateSelection(msg) => { - self.candidate_selection.send(make_packet(signals_received, msg)).await - }, - AllMessages::StatementDistribution(msg) => { - self.statement_distribution.send(make_packet(signals_received, msg)).await - }, - AllMessages::AvailabilityDistribution(msg) => { - self.availability_distribution.send(make_packet(signals_received, msg)).await - }, - AllMessages::AvailabilityRecovery(msg) => { - self.availability_recovery.send(make_packet(signals_received, msg)).await - }, - AllMessages::BitfieldDistribution(msg) => { - self.bitfield_distribution.send(make_packet(signals_received, msg)).await - }, - AllMessages::BitfieldSigning(msg) => { - self.bitfield_signing.send(make_packet(signals_received, msg)).await - }, - AllMessages::Provisioner(msg) => { - self.provisioner.send(make_packet(signals_received, msg)).await - }, - AllMessages::RuntimeApi(msg) => { - self.runtime_api.send(make_packet(signals_received, msg)).await - }, - AllMessages::AvailabilityStore(msg) => { - self.availability_store.send(make_packet(signals_received, msg)).await - }, - AllMessages::NetworkBridge(msg) => { - self.network_bridge.send(make_packet(signals_received, msg)).await - }, - AllMessages::ChainApi(msg) => { - self.chain_api.send(make_packet(signals_received, msg)).await - }, - AllMessages::CollationGeneration(msg) => { - self.collation_generation.send(make_packet(signals_received, msg)).await - }, - AllMessages::CollatorProtocol(msg) => { - self.collator_protocol.send(make_packet(signals_received, msg)).await - }, - AllMessages::ApprovalDistribution(msg) => { - self.approval_distribution.send(make_packet(signals_received, msg)).await - }, - AllMessages::ApprovalVoting(msg) => { - self.approval_voting.send(make_packet(signals_received, msg)).await - }, - AllMessages::GossipSupport(msg) => { - self.gossip_support.send(make_packet(signals_received, msg)).await - }, - }; - - if res.is_err() { - tracing::debug!( - target: LOG_TARGET, - "Failed to send a message to another subsystem", - ); - } - } - - - fn send_unbounded_and_log_error( - &self, - signals_received: usize, - message: AllMessages, - ) { - let res = match message { - AllMessages::CandidateValidation(msg) => { - self.candidate_validation_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::CandidateBacking(msg) => { - self.candidate_backing_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::CandidateSelection(msg) => { - self.candidate_selection_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::StatementDistribution(msg) => { - self.statement_distribution_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::AvailabilityDistribution(msg) => { - self.availability_distribution_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::AvailabilityRecovery(msg) => { - self.availability_recovery_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::BitfieldDistribution(msg) => { - self.bitfield_distribution_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::BitfieldSigning(msg) => { - self.bitfield_signing_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::Provisioner(msg) => { - self.provisioner_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::RuntimeApi(msg) => { - self.runtime_api_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::AvailabilityStore(msg) => { - self.availability_store_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::NetworkBridge(msg) => { - self.network_bridge_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::ChainApi(msg) => { - self.chain_api_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::CollationGeneration(msg) => { - self.collation_generation_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::CollatorProtocol(msg) => { - self.collator_protocol_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::ApprovalDistribution(msg) => { - self.approval_distribution_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::ApprovalVoting(msg) => { - self.approval_voting_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - AllMessages::GossipSupport(msg) => { - self.gossip_support_unbounded - .unbounded_send(make_packet(signals_received, msg)) - .map_err(|e| e.into_send_error()) - }, - }; - - if res.is_err() { - tracing::debug!( - target: LOG_TARGET, - "Failed to send a message to another subsystem", - ); - } - } -} - -type SubsystemIncomingMessages = stream::Select< - metered::MeteredReceiver>, - metered::UnboundedMeteredReceiver>, ->; - -#[derive(Debug, Default, Clone)] -struct SignalsReceived(Arc); - -impl SignalsReceived { - fn load(&self) -> usize { - self.0.load(atomic::Ordering::SeqCst) - } - - fn inc(&self) { - self.0.fetch_add(1, atomic::Ordering::SeqCst); - } -} - -/// A sender from subsystems to other subsystems. -#[derive(Debug, Clone)] -pub struct OverseerSubsystemSender { - channels: ChannelsOut, - signals_received: SignalsReceived, -} - -#[async_trait::async_trait] -impl SubsystemSender for OverseerSubsystemSender { - async fn send_message(&mut self, msg: AllMessages) { - self.channels.send_and_log_error(self.signals_received.load(), msg).await; - } - - async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send - { - // This can definitely be optimized if necessary. - for msg in msgs { - self.send_message(msg).await; - } - } - - fn send_unbounded_message(&mut self, msg: AllMessages) { - self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); - } -} - /// A context type that is given to the [`Subsystem`] upon spawning. /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s /// or to spawn it's [`SubsystemJob`]s. @@ -961,125 +707,63 @@ impl SubsystemContext for OverseerSubsystemContext { } } -/// A subsystem that we oversee. -/// -/// Ties together the [`Subsystem`] itself and it's running instance -/// (which may be missing if the [`Subsystem`] is not running at the moment -/// for whatever reason). -/// -/// [`Subsystem`]: trait.Subsystem.html -struct OverseenSubsystem { - instance: Option>, -} -impl OverseenSubsystem { - /// Send a message to the wrapped subsystem. - /// - /// If the inner `instance` is `None`, nothing is happening. - async fn send_message(&mut self, msg: M) -> SubsystemResult<()> { - const MESSAGE_TIMEOUT: Duration = Duration::from_secs(10); - - if let Some(ref mut instance) = self.instance { - match instance.tx_bounded.send(MessagePacket { - signals_received: instance.signals_received, - message: msg.into() - }).timeout(MESSAGE_TIMEOUT).await - { - None => { - tracing::error!(target: LOG_TARGET, "Subsystem {} appears unresponsive.", instance.name); - Err(SubsystemError::SubsystemStalled(instance.name)) - } - Some(res) => res.map_err(Into::into), - } - } else { - Ok(()) - } - } +/// The `Overseer` itself. +#[overlord(gen=AllMessages, event=Event, signal=OverseerSignal, error=SubsystemResult)] +pub struct Overseer { + #[subsystem(no_dispatch, blocking, CandidateValidationMessage)] + candidate_validation_message: CandidateValidation, - /// Send a signal to the wrapped subsystem. - /// - /// If the inner `instance` is `None`, nothing is happening. - async fn send_signal(&mut self, signal: OverseerSignal) -> SubsystemResult<()> { - const SIGNAL_TIMEOUT: Duration = Duration::from_secs(10); - - if let Some(ref mut instance) = self.instance { - match instance.tx_signal.send(signal).timeout(SIGNAL_TIMEOUT).await { - None => { - tracing::error!(target: LOG_TARGET, "Subsystem {} appears unresponsive.", instance.name); - Err(SubsystemError::SubsystemStalled(instance.name)) - } - Some(res) => { - let res = res.map_err(Into::into); - if res.is_ok() { - instance.signals_received += 1; - } - res - } - } - } else { - Ok(()) - } - } -} + #[subsystem(no_dispatch, blocking, CandidateBackingMessage)] + candidate_backing_message: CandidateBacking, -#[derive(Clone)] -struct SubsystemMeters { - bounded: metered::Meter, - unbounded: metered::Meter, - signals: metered::Meter, -} + #[subsystem(no_dispatch, blocking, CandidateSelectionMessage)] + candidate_selection_message: CandidateSelection, -impl SubsystemMeters { - fn read(&self) -> SubsystemMeterReadouts { - SubsystemMeterReadouts { - bounded: self.bounded.read(), - unbounded: self.unbounded.read(), - signals: self.signals.read(), - } - } -} + #[subsystem(no_dispatch, blocking, StatementDistributionMessage)] + statement_distribution_message: StatementDistribution, -struct SubsystemMeterReadouts { - bounded: metered::Readout, - unbounded: metered::Readout, - signals: metered::Readout, -} + #[subsystem(no_dispatch, blocking, AvailabilityDistributionMessage)] + availability_distribution_message: AvailabilityDistribution, -/// The `Overseer` itself. -pub struct Overseer { - /// Handles to all subsystems. - subsystems: AllSubsystems< - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - OverseenSubsystem, - >, - - /// Spawner to spawn tasks to. - s: S, - - /// Here we keep handles to spawned subsystems to be notified when they terminate. - running_subsystems: FuturesUnordered>>, - - /// Gather running subsystems' outbound streams into one. - to_overseer_rx: Fuse>, - - /// Events that are sent to the overseer from the outside world - events_rx: metered::MeteredReceiver, + #[subsystem(no_dispatch, blocking, AvailabilityRecoveryMessage)] + availability_recovery_message: AvailabilityRecovery, + + #[subsystem(no_dispatch, blocking, BitfieldSigningMessage)] + bitfield_signing_message: BitfieldSigning, + + #[subsystem(no_dispatch, blocking, BitfieldDistributionMessage)] + bitfield_distribution_message: BitfieldDistribution, + + #[subsystem(no_dispatch, blocking, ProvisionerMessage)] + provisioner_message: Provisioner, + + #[subsystem(no_dispatch, blocking, RuntimeApiMessage)] + runtime_api_message: RuntimeApi, + + #[subsystem(no_dispatch, blocking, AvailabilityStoreMessage)] + availability_store_message: AvailabilityStore, + + #[subsystem(no_dispatch, blocking, NetworkBridgeMessage)] + network_bridge_message: NetworkBridge, + + #[subsystem(no_dispatch, blocking, ChainApiMessage)] + chain_api_message: ChainApi, + + #[subsystem(no_dispatch, blocking, CollationGenerationMessage)] + collation_generation_message: CollationGeneration, + + #[subsystem(no_dispatch, blocking, CollatorProtocolMessage)] + collator_protocol_message: CollatorProtocol, + + #[subsystem(no_dispatch, blocking, ApprovalDistributionMessage)] + approval_distribution_message: ApprovalDistribution, + + #[subsystem(no_dispatch, blocking, ApprovalVotingMessage)] + approval_voting_message: ApprovalVoting, + + #[subsystem(no_dispatch, blocking, GossipSupportMessage)] + gossip_support_message: GossipSupport, /// External listeners waiting for a hash to be in the active-leave set. activation_external_listeners: HashMap>>>, @@ -1102,181 +786,8 @@ pub struct Overseer { metrics: Metrics, } -/// Overseer Prometheus metrics. -#[derive(Clone)] -struct MetricsInner { - activated_heads_total: prometheus::Counter, - deactivated_heads_total: prometheus::Counter, - messages_relayed_total: prometheus::Counter, - to_subsystem_bounded_sent: prometheus::GaugeVec, - to_subsystem_bounded_received: prometheus::GaugeVec, - to_subsystem_unbounded_sent: prometheus::GaugeVec, - to_subsystem_unbounded_received: prometheus::GaugeVec, - signals_sent: prometheus::GaugeVec, - signals_received: prometheus::GaugeVec, -} - -#[derive(Default, Clone)] -struct Metrics(Option); - -impl Metrics { - fn on_head_activated(&self) { - if let Some(metrics) = &self.0 { - metrics.activated_heads_total.inc(); - } - } - - fn on_head_deactivated(&self) { - if let Some(metrics) = &self.0 { - metrics.deactivated_heads_total.inc(); - } - } - - fn on_message_relayed(&self) { - if let Some(metrics) = &self.0 { - metrics.messages_relayed_total.inc(); - } - } - - fn channel_fill_level_snapshot( - &self, - to_subsystem: AllSubsystemsSame<(&'static str, SubsystemMeterReadouts)>, - ) { - self.0.as_ref().map(|metrics| { - to_subsystem.map_subsystems( - |(name, readouts): (_, SubsystemMeterReadouts)| { - metrics.to_subsystem_bounded_sent.with_label_values(&[name]) - .set(readouts.bounded.sent as u64); - - metrics.to_subsystem_bounded_received.with_label_values(&[name]) - .set(readouts.bounded.received as u64); - - metrics.to_subsystem_unbounded_sent.with_label_values(&[name]) - .set(readouts.unbounded.sent as u64); - - metrics.to_subsystem_unbounded_received.with_label_values(&[name]) - .set(readouts.unbounded.received as u64); - - metrics.signals_sent.with_label_values(&[name]) - .set(readouts.signals.sent as u64); - - metrics.signals_received.with_label_values(&[name]) - .set(readouts.signals.received as u64); - }); - }); - } -} - -impl metrics::Metrics for Metrics { - fn try_register(registry: &prometheus::Registry) -> Result { - let metrics = MetricsInner { - activated_heads_total: prometheus::register( - prometheus::Counter::new( - "parachain_activated_heads_total", - "Number of activated heads." - )?, - registry, - )?, - deactivated_heads_total: prometheus::register( - prometheus::Counter::new( - "parachain_deactivated_heads_total", - "Number of deactivated heads." - )?, - registry, - )?, - messages_relayed_total: prometheus::register( - prometheus::Counter::new( - "parachain_messages_relayed_total", - "Number of messages relayed by Overseer." - )?, - registry, - )?, - to_subsystem_bounded_sent: prometheus::register( - prometheus::GaugeVec::::new( - prometheus::Opts::new( - "parachain_subsystem_bounded_sent", - "Number of elements sent to subsystems' bounded queues", - ), - &[ - "subsystem_name", - ], - )?, - registry, - )?, - to_subsystem_bounded_received: prometheus::register( - prometheus::GaugeVec::::new( - prometheus::Opts::new( - "parachain_subsystem_bounded_received", - "Number of elements received by subsystems' bounded queues", - ), - &[ - "subsystem_name", - ], - )?, - registry, - )?, - to_subsystem_unbounded_sent: prometheus::register( - prometheus::GaugeVec::::new( - prometheus::Opts::new( - "parachain_subsystem_unbounded_sent", - "Number of elements sent to subsystems' unbounded queues", - ), - &[ - "subsystem_name", - ], - )?, - registry, - )?, - to_subsystem_unbounded_received: prometheus::register( - prometheus::GaugeVec::::new( - prometheus::Opts::new( - "parachain_subsystem_unbounded_received", - "Number of elements received by subsystems' unbounded queues", - ), - &[ - "subsystem_name", - ], - )?, - registry, - )?, - signals_sent: prometheus::register( - prometheus::GaugeVec::::new( - prometheus::Opts::new( - "parachain_overseer_signals_sent", - "Number of signals sent by overseer to subsystems", - ), - &[ - "subsystem_name", - ], - )?, - registry, - )?, - signals_received: prometheus::register( - prometheus::GaugeVec::::new( - prometheus::Opts::new( - "parachain_overseer_signals_received", - "Number of signals received by subsystems from overseer", - ), - &[ - "subsystem_name", - ], - )?, - registry, - )?, - }; - Ok(Metrics(Some(metrics))) - } -} - -impl fmt::Debug for Metrics { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("Metrics {{...}}") - } -} - impl Overseer where - S: SpawnNamed, SupportsParachains: HeadSupportsParachains, { /// Create a new instance of the `Overseer` with a fixed set of [`Subsystem`]s. @@ -1396,395 +907,36 @@ where ApV: Subsystem> + Send, GS: Subsystem> + Send, { - let (events_tx, events_rx) = metered::channel(CHANNEL_CAPACITY); - - let handler = OverseerHandler { - events_tx: events_tx.clone(), - }; - - let metrics = ::register(prometheus_registry)?; - - let (to_overseer_tx, to_overseer_rx) = metered::unbounded(); - - let mut running_subsystems = FuturesUnordered::new(); - - let (candidate_validation_bounded_tx, candidate_validation_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (candidate_backing_bounded_tx, candidate_backing_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (candidate_selection_bounded_tx, candidate_selection_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (statement_distribution_bounded_tx, statement_distribution_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (availability_distribution_bounded_tx, availability_distribution_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (availability_recovery_bounded_tx, availability_recovery_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (bitfield_signing_bounded_tx, bitfield_signing_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (bitfield_distribution_bounded_tx, bitfield_distribution_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (provisioner_bounded_tx, provisioner_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (runtime_api_bounded_tx, runtime_api_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (availability_store_bounded_tx, availability_store_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (network_bridge_bounded_tx, network_bridge_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (chain_api_bounded_tx, chain_api_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (collator_protocol_bounded_tx, collator_protocol_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (collation_generation_bounded_tx, collation_generation_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (approval_distribution_bounded_tx, approval_distribution_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (approval_voting_bounded_tx, approval_voting_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - let (gossip_support_bounded_tx, gossip_support_bounded_rx) - = metered::channel(CHANNEL_CAPACITY); - - let (candidate_validation_unbounded_tx, candidate_validation_unbounded_rx) - = metered::unbounded(); - let (candidate_backing_unbounded_tx, candidate_backing_unbounded_rx) - = metered::unbounded(); - let (candidate_selection_unbounded_tx, candidate_selection_unbounded_rx) - = metered::unbounded(); - let (statement_distribution_unbounded_tx, statement_distribution_unbounded_rx) - = metered::unbounded(); - let (availability_distribution_unbounded_tx, availability_distribution_unbounded_rx) - = metered::unbounded(); - let (availability_recovery_unbounded_tx, availability_recovery_unbounded_rx) - = metered::unbounded(); - let (bitfield_signing_unbounded_tx, bitfield_signing_unbounded_rx) - = metered::unbounded(); - let (bitfield_distribution_unbounded_tx, bitfield_distribution_unbounded_rx) - = metered::unbounded(); - let (provisioner_unbounded_tx, provisioner_unbounded_rx) - = metered::unbounded(); - let (runtime_api_unbounded_tx, runtime_api_unbounded_rx) - = metered::unbounded(); - let (availability_store_unbounded_tx, availability_store_unbounded_rx) - = metered::unbounded(); - let (network_bridge_unbounded_tx, network_bridge_unbounded_rx) - = metered::unbounded(); - let (chain_api_unbounded_tx, chain_api_unbounded_rx) - = metered::unbounded(); - let (collator_protocol_unbounded_tx, collator_protocol_unbounded_rx) - = metered::unbounded(); - let (collation_generation_unbounded_tx, collation_generation_unbounded_rx) - = metered::unbounded(); - let (approval_distribution_unbounded_tx, approval_distribution_unbounded_rx) - = metered::unbounded(); - let (approval_voting_unbounded_tx, approval_voting_unbounded_rx) - = metered::unbounded(); - let (gossip_support_unbounded_tx, gossip_support_unbounded_rx) - = metered::unbounded(); - - let channels_out = ChannelsOut { - candidate_validation: candidate_validation_bounded_tx.clone(), - candidate_backing: candidate_backing_bounded_tx.clone(), - candidate_selection: candidate_selection_bounded_tx.clone(), - statement_distribution: statement_distribution_bounded_tx.clone(), - availability_distribution: availability_distribution_bounded_tx.clone(), - availability_recovery: availability_recovery_bounded_tx.clone(), - bitfield_signing: bitfield_signing_bounded_tx.clone(), - bitfield_distribution: bitfield_distribution_bounded_tx.clone(), - provisioner: provisioner_bounded_tx.clone(), - runtime_api: runtime_api_bounded_tx.clone(), - availability_store: availability_store_bounded_tx.clone(), - network_bridge: network_bridge_bounded_tx.clone(), - chain_api: chain_api_bounded_tx.clone(), - collator_protocol: collator_protocol_bounded_tx.clone(), - collation_generation: collation_generation_bounded_tx.clone(), - approval_distribution: approval_distribution_bounded_tx.clone(), - approval_voting: approval_voting_bounded_tx.clone(), - gossip_support: gossip_support_bounded_tx.clone(), - - candidate_validation_unbounded: candidate_validation_unbounded_tx.clone(), - candidate_backing_unbounded: candidate_backing_unbounded_tx.clone(), - candidate_selection_unbounded: candidate_selection_unbounded_tx.clone(), - statement_distribution_unbounded: statement_distribution_unbounded_tx.clone(), - availability_distribution_unbounded: availability_distribution_unbounded_tx.clone(), - availability_recovery_unbounded: availability_recovery_unbounded_tx.clone(), - bitfield_signing_unbounded: bitfield_signing_unbounded_tx.clone(), - bitfield_distribution_unbounded: bitfield_distribution_unbounded_tx.clone(), - provisioner_unbounded: provisioner_unbounded_tx.clone(), - runtime_api_unbounded: runtime_api_unbounded_tx.clone(), - availability_store_unbounded: availability_store_unbounded_tx.clone(), - network_bridge_unbounded: network_bridge_unbounded_tx.clone(), - chain_api_unbounded: chain_api_unbounded_tx.clone(), - collator_protocol_unbounded: collator_protocol_unbounded_tx.clone(), - collation_generation_unbounded: collation_generation_unbounded_tx.clone(), - approval_distribution_unbounded: approval_distribution_unbounded_tx.clone(), - approval_voting_unbounded: approval_voting_unbounded_tx.clone(), - gossip_support_unbounded: gossip_support_unbounded_tx.clone(), - }; - - let candidate_validation_subsystem = spawn( - &mut s, - candidate_validation_bounded_tx, - stream::select(candidate_validation_bounded_rx, candidate_validation_unbounded_rx), - candidate_validation_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.candidate_validation, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let candidate_backing_subsystem = spawn( - &mut s, - candidate_backing_bounded_tx, - stream::select(candidate_backing_bounded_rx, candidate_backing_unbounded_rx), - candidate_backing_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.candidate_backing, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let candidate_selection_subsystem = spawn( - &mut s, - candidate_selection_bounded_tx, - stream::select(candidate_selection_bounded_rx, candidate_selection_unbounded_rx), - candidate_selection_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.candidate_selection, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let statement_distribution_subsystem = spawn( - &mut s, - statement_distribution_bounded_tx, - stream::select(statement_distribution_bounded_rx, statement_distribution_unbounded_rx), - candidate_validation_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.statement_distribution, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let availability_distribution_subsystem = spawn( - &mut s, - availability_distribution_bounded_tx, - stream::select(availability_distribution_bounded_rx, availability_distribution_unbounded_rx), - availability_distribution_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.availability_distribution, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let availability_recovery_subsystem = spawn( - &mut s, - availability_recovery_bounded_tx, - stream::select(availability_recovery_bounded_rx, availability_recovery_unbounded_rx), - availability_recovery_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.availability_recovery, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let bitfield_signing_subsystem = spawn( - &mut s, - bitfield_signing_bounded_tx, - stream::select(bitfield_signing_bounded_rx, bitfield_signing_unbounded_rx), - bitfield_signing_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.bitfield_signing, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let bitfield_distribution_subsystem = spawn( - &mut s, - bitfield_distribution_bounded_tx, - stream::select(bitfield_distribution_bounded_rx, bitfield_distribution_unbounded_rx), - bitfield_distribution_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.bitfield_distribution, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let provisioner_subsystem = spawn( - &mut s, - provisioner_bounded_tx, - stream::select(provisioner_bounded_rx, provisioner_unbounded_rx), - provisioner_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.provisioner, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let runtime_api_subsystem = spawn( - &mut s, - runtime_api_bounded_tx, - stream::select(runtime_api_bounded_rx, runtime_api_unbounded_rx), - runtime_api_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.runtime_api, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let availability_store_subsystem = spawn( - &mut s, - availability_store_bounded_tx, - stream::select(availability_store_bounded_rx, availability_store_unbounded_rx), - availability_store_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.availability_store, - &metrics, - &mut running_subsystems, - TaskKind::Blocking, - )?; - - let network_bridge_subsystem = spawn( - &mut s, - network_bridge_bounded_tx, - stream::select(network_bridge_bounded_rx, network_bridge_unbounded_rx), - network_bridge_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.network_bridge, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let chain_api_subsystem = spawn( - &mut s, - chain_api_bounded_tx, - stream::select(chain_api_bounded_rx, chain_api_unbounded_rx), - chain_api_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.chain_api, - &metrics, - &mut running_subsystems, - TaskKind::Blocking, - )?; - - let collation_generation_subsystem = spawn( - &mut s, - collation_generation_bounded_tx, - stream::select(collation_generation_bounded_rx, collation_generation_unbounded_rx), - collation_generation_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.collation_generation, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let collator_protocol_subsystem = spawn( - &mut s, - collator_protocol_bounded_tx, - stream::select(collator_protocol_bounded_rx, collator_protocol_unbounded_rx), - collator_protocol_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.collator_protocol, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let approval_distribution_subsystem = spawn( - &mut s, - approval_distribution_bounded_tx, - stream::select(approval_distribution_bounded_rx, approval_distribution_unbounded_rx), - approval_distribution_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.approval_distribution, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let approval_voting_subsystem = spawn( - &mut s, - approval_voting_bounded_tx, - stream::select(approval_voting_bounded_rx, approval_voting_unbounded_rx), - approval_voting_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.approval_voting, - &metrics, - &mut running_subsystems, - TaskKind::Blocking, - )?; - - let gossip_support_subsystem = spawn( - &mut s, - gossip_support_bounded_tx, - stream::select(gossip_support_bounded_rx, gossip_support_unbounded_rx), - gossip_support_unbounded_tx.meter().clone(), - channels_out.clone(), - to_overseer_tx.clone(), - all_subsystems.gossip_support, - &metrics, - &mut running_subsystems, - TaskKind::Regular, - )?; - - let leaves = leaves - .into_iter() - .map(|BlockInfo { hash, parent_hash: _, number }| (hash, number)) - .collect(); - - let active_leaves = HashMap::new(); - let activation_external_listeners = HashMap::new(); - - let subsystems = AllSubsystems { - candidate_validation: candidate_validation_subsystem, - candidate_backing: candidate_backing_subsystem, - candidate_selection: candidate_selection_subsystem, - statement_distribution: statement_distribution_subsystem, - availability_distribution: availability_distribution_subsystem, - availability_recovery: availability_recovery_subsystem, - bitfield_signing: bitfield_signing_subsystem, - bitfield_distribution: bitfield_distribution_subsystem, - provisioner: provisioner_subsystem, - runtime_api: runtime_api_subsystem, - availability_store: availability_store_subsystem, - network_bridge: network_bridge_subsystem, - chain_api: chain_api_subsystem, - collation_generation: collation_generation_subsystem, - collator_protocol: collator_protocol_subsystem, - approval_distribution: approval_distribution_subsystem, - approval_voting: approval_voting_subsystem, - gossip_support: gossip_support_subsystem, - }; + let (overseer, handler) = Self::builder() + .candidate_validation(all_subsystems.candidate_validation) + .candidate_backing(all_subsystems.candidate_backing) + .candidate_selection(all_subsystems.candidate_selection) + .statement_distribution(all_subsystems.statement_distribution) + .availability_distribution(all_subsystems.availability_distribution) + .availability_recovery(all_subsystems.availability_recovery) + .bitfield_signing(all_subsystems.bitfield_signing) + .bitfield_distribution(all_subsystems.bitfield_distribution) + .provisioner(all_subsystems.provisioner) + .runtime_api(all_subsystems.runtime_api) + .availability_store(all_subsystems.availability_store) + .network_bridge(all_subsystems.network_bridge) + .chain_api(all_subsystems.chain_api) + .collation_generation(all_subsystems.collation_generation) + .collator_protocol(all_subsystems.collator_protocol) + .approval_distribution(all_subsystems.approval_distribution) + .approval_voting(all_subsystems.approval_voting) + .gossip_support(all_subsystems.gossip_support) + .leaves(leaves.collect()) + .active_leaves(Default::default()) + .span_per_active_leaf(Default::default()) + .activation_external_listeners(Default::default()) + .metrics(::register(prometheus_registry)?) + .active_leaves(leaves.collect()) + .spawner(s) + .build(); + + // spawn the metrics metronome task { struct ExtractNameAndMeters; impl<'a, T: 'a> MapSubsystem<&'a OverseenSubsystem> for ExtractNameAndMeters { @@ -1821,21 +973,8 @@ where s.spawn("metrics_metronome", Box::pin(metronome)); } - let this = Self { - subsystems, - s, - running_subsystems, - to_overseer_rx: to_overseer_rx.fuse(), - events_rx, - activation_external_listeners, - leaves, - active_leaves, - metrics, - span_per_active_leaf: Default::default(), - supports_parachains, - }; - Ok((this, handler)) + Ok((overseer, OverseerHandler { handler })) } // Stop the overseer. @@ -1874,6 +1013,8 @@ where } } + + /// Run the `Overseer`. #[tracing::instrument(skip(self), fields(subsystem = LOG_TARGET))] pub async fn run(mut self) -> SubsystemResult<()> { diff --git a/node/subsystem/src/errors.rs b/node/subsystem/src/errors.rs index 89c21426a53d..e2c0d1af2a06 100644 --- a/node/subsystem/src/errors.rs +++ b/node/subsystem/src/errors.rs @@ -16,6 +16,9 @@ //! Error types for the subsystem requests. + +use crate::JaegerError; + /// A description of an error causing the runtime API request to be unservable. #[derive(Debug, Clone)] pub struct RuntimeApiError(String); @@ -92,10 +95,10 @@ impl std::error::Error for RecoveryError {} #[allow(missing_docs)] pub enum SubsystemError { #[error(transparent)] - NotifyCancellation(#[from] oneshot::Canceled), + NotifyCancellation(#[from] futures::channel::oneshot::Canceled), #[error(transparent)] - QueueError(#[from] mpsc::SendError), + QueueError(#[from] futures::channel::mpsc::SendError), #[error(transparent)] TaskSpawn(#[from] futures::task::SpawnError), diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 3cc4894a493e..fafdc65b3181 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -170,7 +170,7 @@ pub enum SubsystemError { Prometheus(#[from] substrate_prometheus_endpoint::PrometheusError), #[error(transparent)] - Jaeger(#[from] JaegerError), + Jaeger(#[from] jaeger::JaegerError), #[error("Failed to {0}")] Context(String), From 2865a31bf456a85c6c56ccddae51a5814f12892f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 19 May 2021 13:14:13 +0200 Subject: [PATCH 036/161] generator --- node/overseer/overseer-gen/Cargo.toml | 2 +- .../proc-macro/src/impl_dispatch.rs | 4 +- .../proc-macro/src/impl_overseer.rs | 38 +++++++---- .../proc-macro/src/parse_struct.rs | 64 +++++++++++-------- node/overseer/overseer-gen/src/lib.rs | 1 + 5 files changed, 68 insertions(+), 41 deletions(-) diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index 9de6a41a686b..4e91e04b4247 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -14,7 +14,7 @@ overseer-gen-proc-macro = { package = "polkadot-overseer-gen-proc-macro", path = metered = { package = "metered-channel", path = "../../metered-channel" } # trait SpawnNamed sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } - +futures-timer = "3.0.2" [dev-dependencies] trybuild = "1.0.41" diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index d158bdbb4b44..34ffd080a280 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -16,14 +16,14 @@ use proc_macro2::TokenStream; use quote::quote; -use syn::{Ident, Result}; +use syn::{Ident, Path, Result}; use super::*; pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { let message_wrapper = &info.message_wrapper; - let dispatchable = info.subsystems().into_iter().filter(|ssf| !ssf.no_dispatch).map(|ssf| ssf.consumes.clone()).collect::>(); + let dispatchable = info.subsystems().into_iter().filter(|ssf| !ssf.no_dispatch).map(|ssf| ssf.consumes.clone()).collect::>(); let msg = "Generated by #[subsystem_dispatch_gen] proc-macro."; let extern_event_ty= &info.extern_event_ty.clone(); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 70d5bce2d88a..66cb985c3e54 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -7,7 +7,7 @@ pub(crate) fn impl_overseer_struct( info: &OverseerInfo, ) -> Result { let message_wrapper = &info.message_wrapper.clone(); - let overseer_name = dbg!(info.overseer_name.clone()); + let overseer_name = info.overseer_name.clone(); let subsystem_name = &info.subsystem_names(); let builder_generic_ty = &info.builder_generic_types(); @@ -15,7 +15,7 @@ pub(crate) fn impl_overseer_struct( let baggage_name = &info.baggage_names(); let baggage_ty = &info.baggage_types(); - let baggage_generic_ty = &info.baggage_generic_types(); + let baggage_generic_ty = &dbg!(info.baggage_generic_types()); let generics = quote! { < S, #( #baggage_generic_ty, )* > @@ -39,6 +39,8 @@ pub(crate) fn impl_overseer_struct( let log_target = syn::LitStr::new(overseer_name.to_string().to_lowercase().as_str(), overseer_name.span()); let mut ts = quote! { + const STOP_DELAY: ::std::time::Duration = ::std::time::Duration::from_secs(1); + /// Capacity of a bounded message channel between overseer and subsystem /// but also for bounded channels between two subsystems. const CHANNEL_CAPACITY: usize = #message_channel_capacity; @@ -75,8 +77,22 @@ pub(crate) fn impl_overseer_struct( } impl #generics #overseer_name #generics #where_clause { - pub async fn stop(mut self) { - unimplemented!("Stopping is not yet implemented") + pub async fn broadcast_signal(mut self, signal: #signal_ty) { + self.broadcast_signal(signal); + + let mut stop_delay = ::polkadot_overseer_gen::Delay::new(STOP_DELAY).fuse(); + + loop { + select! { + _ = self.running_subsystems.next() => { + if self.running_subsystems.is_empty() { + break; + } + }, + _ = stop_delay => break, + complete => break, + } + } } pub async fn broadcast_signal(&mut self, signal: #signal_ty) -> SubsystemResult<()> { @@ -131,6 +147,7 @@ pub(crate) fn impl_builder( let baggage_generic_ty = &info.baggage_generic_types(); let baggage_name = &info.baggage_names(); + let baggage_ty = &info.baggage_types(); let blocking = &info.subsystems().iter().map(|x| x.blocking).collect::>(); @@ -284,9 +301,7 @@ pub(crate) fn impl_builder( let fut = Box::pin(async move { if let Err(e) = future.await { - ::polkadot_overseer_gen::tracing::error!(subsystem=name, err = ?e, "subsystem exited with error"); } else { - ::polkadot_overseer_gen::tracing::debug!(subsystem=name, "subsystem exited without an error"); } let _ = terminated_tx.send(()); }); @@ -298,7 +313,6 @@ pub(crate) fn impl_builder( } running_subsystems.push(Box::pin(terminated_rx.map(|e| { - ::polkadot_overseer_gen::tracing::warn!(err = ?e, "dropping error"); Ok(()) }))); @@ -404,7 +418,6 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { - ::polkadot_overseer_gen::tracing::error!(target: crate::LOG_TARGET, "Subsystem {} appears unresponsive.", instance.name); Err(SubsystemError::SubsystemStalled(instance.name)) } Some(res) => res.map_err(Into::into), @@ -423,7 +436,6 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { - ::polkadot_overseer_gen::tracing::error!(target: crate::LOG_TARGET, "Subsystem {} appears unresponsive.", instance.name); Err(SubsystemError::SubsystemStalled(instance.name)) } Some(res) => { @@ -502,8 +514,9 @@ pub(crate) fn impl_trait_subsystem(info: &OverseerInfo) -> Result Result(&mut self, msgs: T) where T: IntoIterator + Send, T::IntoIter: Send; + + /// Obtain the sender. + fn sender(&self) -> Self::Sender; } /// A trait that describes the [`Subsystem`]s that can run on the [`Overseer`]. diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index ed47cde6308e..37b58c4d2444 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -24,7 +24,7 @@ pub(crate) struct SubSysField { /// Type of the subsystem. pub(crate) ty: Path, /// Type to be consumed by the subsystem. - pub(crate) consumes: Ident, + pub(crate) consumes: Path, /// If `no_dispatch` is present, if the message is incomming via /// an extern `Event`, it will not be dispatched to all subsystems. pub(crate) no_dispatch: bool, @@ -33,29 +33,33 @@ pub(crate) struct SubSysField { pub(crate) blocking: bool, } -fn try_type_to_path(ty: Type, span: Span) -> Result { +fn try_type_to_path(ty: Type, span: Span) -> Result { match ty { - Type::Path(path) => { - path - } + Type::Path(path) => Ok(path.path), _ => Err(Error::new(span, "Type must be a path expression.")), } } + +fn try_type_to_ident(ty: Type, span: Span) -> Result { + try_type_to_path(ty, span.clone())?.get_ident().cloned().ok_or_else(|| Error::new(span, "Expected an identifier, but got a path.")) +} + + pub(crate) struct SubSystemTag { #[allow(dead_code)] pub(crate) attrs: Vec, #[allow(dead_code)] pub(crate) no_dispatch: bool, pub(crate) blocking: bool, - pub(crate) consumes: Punctuated, + pub(crate) consumes: Punctuated, } impl Parse for SubSystemTag { fn parse(input: syn::parse::ParseStream) -> Result { let attrs = Attribute::parse_outer(input)?; - let input = dbg!(input); + let input = input; let content; let _ = syn::parenthesized!(content in input); let parse_tags = || -> Result> { @@ -80,7 +84,7 @@ impl Parse for SubSystemTag { let no_dispatch = unique.take("no_dispatch").is_some(); let blocking = unique.take("blocking").is_some(); - let consumes = content.parse_terminated(Ident::parse)?; + let consumes = content.parse_terminated(Path::parse)?; Ok(Self { attrs, @@ -96,7 +100,7 @@ impl Parse for SubSystemTag { #[derive(Debug, Clone)] pub(crate) struct BaggageField { pub(crate) field_name: Ident, - pub(crate) field_ty: Ident, + pub(crate) field_ty: Path, pub(crate) generic: bool, } @@ -140,14 +144,14 @@ impl OverseerInfo { #[allow(dead_code)] // FIXME use as the defaults - pub(crate) fn subsystem_types(&self) -> Vec { + pub(crate) fn subsystem_types(&self) -> Vec { self.subsystems.iter().map(|ssf| ssf.ty.clone()).collect::>() } pub(crate) fn baggage_names(&self) -> Vec { self.baggage.iter().map(|bag| bag.field_name.clone()).collect::>() } - pub(crate) fn baggage_types(&self) -> Vec { + pub(crate) fn baggage_types(&self) -> Vec { self.baggage.iter().map(|bag| bag.field_ty.clone()).collect::>() } @@ -157,7 +161,7 @@ impl OverseerInfo { } pub(crate) fn baggage_generic_types(&self) -> Vec { - self.baggage.iter().filter(|bag| bag.generic).map(|bag| dbg!(bag.field_ty.clone())).collect::>() + self.baggage.iter().filter(|bag| bag.generic).filter_map(|bag| bag.field_ty.get_ident().cloned()).collect::>() } pub(crate) fn channel_names(&self, suffix: &'static str) -> Vec { @@ -166,7 +170,7 @@ impl OverseerInfo { .collect::>() } - pub(crate) fn consumes(&self) -> Vec { + pub(crate) fn consumes(&self) -> Vec { self.subsystems.iter() .map(|ssf| ssf.consumes.clone()) .collect::>() @@ -187,13 +191,15 @@ impl OverseerGuts { let mut subsystems = Vec::with_capacity(n); let mut baggage = Vec::with_capacity(n); for (idx, Field { attrs, vis: _, ident, ty, .. }) in fields.named.into_iter().enumerate() { - let mut consumes = attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| { - let span = attr.path.span(); - attr.path.get_ident().filter(|ident| *ident == "subsystem").map(move |_ident| { - let attr_tokens = attr.tokens.clone(); - (attr_tokens, span) - }) - }); + let mut consumes = attrs.iter() + .filter(|attr| attr.style == AttrStyle::Outer) + .filter_map(|attr| { + let span = attr.path.span(); + attr.path.get_ident().filter(|ident| *ident == "subsystem").map(move |_ident| { + let attr_tokens = attr.tokens.clone(); + (attr_tokens, span) + }) + }); let ident = ident.ok_or_else(|| { Error::new(ty.span(), "Missing identifier for member. BUG") })?; @@ -208,16 +214,16 @@ impl OverseerGuts { err }) } - let mut consumes_idents = Vec::with_capacity(attrs.len()); - let attr_tokens = dbg!(attr_tokens.clone()); + let mut consumes_paths = Vec::with_capacity(attrs.len()); + let attr_tokens = attr_tokens.clone(); let variant: SubSystemTag = syn::parse2(attr_tokens.clone())?; if variant.consumes.len() != 1 { return Err(Error::new(attr_tokens.span(), "Exactly one message can be consumed per subsystem.")) } - consumes_idents.extend(variant.consumes.into_iter()); + consumes_paths.extend(variant.consumes.into_iter()); - if consumes_idents.is_empty() { + if consumes_paths.is_empty() { return Err( Error::new(span, "Subsystem must consume at least one message") ) @@ -227,16 +233,20 @@ impl OverseerGuts { name: ident, generic: Ident::new(format!("Sub{}", idx).as_str(), span), ty: try_type_to_path(ty, span)?, - consumes: consumes_idents[0].clone(), + consumes: consumes_paths[0].clone(), no_dispatch: variant.no_dispatch, blocking: variant.blocking, }); } else { let field_ty: Path = try_type_to_path(ty, ident.span())?; - field_ty. + let generic: bool = if let Some(ident) = field_ty.get_ident() { + baggage_generics.contains(ident) + } else { + false + }; baggage.push(BaggageField { field_name: ident, - generic: !baggage_generics.contains(&field_ty), + generic, field_ty, }); } diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index f1b09eacfd8b..a789a5f26351 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -86,6 +86,7 @@ pub use std::time::Duration; use std::sync::atomic::{self, AtomicUsize}; use std::sync::Arc; +pub use futures_timer::Delay; /// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. /// /// Used to launch jobs. From 734d5766ada81858bb485a2384e290a854f32525 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 19 May 2021 13:14:30 +0200 Subject: [PATCH 037/161] avoid redundant definitions / conflicts --- Cargo.lock | 1 + node/overseer/src/lib.rs | 393 +++++++-------------------------------- 2 files changed, 66 insertions(+), 328 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03cf8356bf10..4a6040e02a77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6265,6 +6265,7 @@ version = "0.1.0" dependencies = [ "async-trait", "futures 0.3.14", + "futures-timer 3.0.2", "metered-channel", "polkadot-overseer-gen-proc-macro", "sp-core", diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index eee158eae1f3..4a73922d74c6 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -84,40 +84,32 @@ use polkadot_subsystem::messages::{ CandidateSelectionMessage, ChainApiMessage, StatementDistributionMessage, AvailabilityDistributionMessage, BitfieldSigningMessage, BitfieldDistributionMessage, ProvisionerMessage, RuntimeApiMessage, - AvailabilityStoreMessage, NetworkBridgeMessage, AllMessages, CollationGenerationMessage, + AvailabilityStoreMessage, NetworkBridgeMessage, CollationGenerationMessage, CollatorProtocolMessage, AvailabilityRecoveryMessage, ApprovalDistributionMessage, ApprovalVotingMessage, GossipSupportMessage, }; pub use polkadot_subsystem::{ - Subsystem, SubsystemContext, SubsystemSender, OverseerSignal, FromOverseer, SubsystemError, + OverseerSignal, SubsystemError, SubsystemResult, SpawnedSubsystem, ActiveLeavesUpdate, ActivatedLeaf, DummySubsystem, jaeger, }; mod metrics; -use self::metrics::{Metrics, MetricsInner}; +use self::metrics::Metrics; use polkadot_node_subsystem_util::{TimeoutExt, metrics::{prometheus, Metrics as MetricsTrait}, metered, Metronome}; use polkadot_node_primitives::SpawnNamed; use polkadot_overseer_gen::{ SubsystemMeterReadouts, - ChannelsOut, - OverseenSubsystem, SubsystemMeters, SubsystemIncomingMessages, overlord, - OverseerSubsystemSender, + MessagePacket, + make_packet, + SignalsReceived, }; - -// A capacity of bounded channels inside the overseer. -const CHANNEL_CAPACITY: usize = 1024; -// The capacity of signal channels to subsystems. -const SIGNAL_CHANNEL_CAPACITY: usize = 64; - -// A graceful `Overseer` teardown time delay. -const STOP_DELAY: u64 = 1; // Target for logs. -const LOG_TARGET: &'static str = "parachain::overseer"; +// const LOG_TARGET: &'static str = "parachain::overseer"; trait MapSubsystem { type Output; @@ -430,60 +422,6 @@ enum ExternalRequest { }, } -/// A handler used to communicate with the [`Overseer`]. -/// -/// [`Overseer`]: struct.Overseer.html -#[derive(Clone)] -pub struct OverseerHandler { - events_tx: metered::MeteredSender, -} - -impl OverseerHandler { - /// Inform the `Overseer` that that some block was imported. - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - pub async fn block_imported(&mut self, block: BlockInfo) { - self.send_and_log_error(Event::BlockImported(block)).await - } - - /// Send some message to one of the `Subsystem`s. - #[tracing::instrument(level = "trace", skip(self, msg), fields(subsystem = LOG_TARGET))] - pub async fn send_msg(&mut self, msg: impl Into) { - self.send_and_log_error(Event::MsgToSubsystem(msg.into())).await - } - - /// Inform the `Overseer` that some block was finalized. - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - pub async fn block_finalized(&mut self, block: BlockInfo) { - self.send_and_log_error(Event::BlockFinalized(block)).await - } - - /// Wait for a block with the given hash to be in the active-leaves set. - /// - /// The response channel responds if the hash was activated and is closed if the hash was deactivated. - /// Note that due the fact the overseer doesn't store the whole active-leaves set, only deltas, - /// the response channel may never return if the hash was deactivated before this call. - /// In this case, it's the caller's responsibility to ensure a timeout is set. - #[tracing::instrument(level = "trace", skip(self, response_channel), fields(subsystem = LOG_TARGET))] - pub async fn wait_for_activation(&mut self, hash: Hash, response_channel: oneshot::Sender>) { - self.send_and_log_error(Event::ExternalRequest(ExternalRequest::WaitForActivation { - hash, - response_channel - })).await - } - - /// Tell `Overseer` to shutdown. - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - pub async fn stop(&mut self) { - self.send_and_log_error(Event::Stop).await - } - - async fn send_and_log_error(&mut self, event: Event) { - if self.events_tx.send(event).await.is_err() { - tracing::info!(target: LOG_TARGET, "Failed to send an event to Overseer"); - } - } -} - /// Glues together the [`Overseer`] and `BlockchainEvents` by forwarding /// import and finality notifications into the [`OverseerHandler`]. /// @@ -527,30 +465,6 @@ impl Debug for ToOverseer { } } } - -/// A running instance of some [`Subsystem`]. -/// -/// [`Subsystem`]: trait.Subsystem.html -struct SubsystemInstance { - tx_signal: metered::MeteredSender, - tx_bounded: metered::MeteredSender>, - meters: SubsystemMeters, - signals_received: usize, - name: &'static str, -} - -#[derive(Debug)] -struct MessagePacket { - signals_received: usize, - message: T, -} - -fn make_packet(signals_received: usize, message: T) -> MessagePacket { - MessagePacket { - signals_received, - message, - } -} /// A context type that is given to the [`Subsystem`] upon spawning. /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s /// or to spawn it's [`SubsystemJob`]s. @@ -559,8 +473,8 @@ fn make_packet(signals_received: usize, message: T) -> MessagePacket { /// [`Subsystem`]: trait.Subsystem.html /// [`SubsystemJob`]: trait.SubsystemJob.html #[derive(Debug)] -pub struct OverseerSubsystemContext{ - signals: metered::MeteredReceiver, +pub struct OverseerSubsystemContext{ + signals: metered::MeteredReceiver, messages: SubsystemIncomingMessages, to_subsystems: OverseerSubsystemSender, to_overseer: metered::UnboundedMeteredSender, @@ -569,10 +483,10 @@ pub struct OverseerSubsystemContext{ metrics: Metrics, } -impl OverseerSubsystemContext { +impl OverseerSubsystemContext { /// Create a new `OverseerSubsystemContext`. fn new( - signals: metered::MeteredReceiver, + signals: metered::MeteredReceiver, messages: SubsystemIncomingMessages, to_subsystems: ChannelsOut, to_overseer: metered::UnboundedMeteredSender, @@ -598,7 +512,7 @@ impl OverseerSubsystemContext { /// Intended for tests. #[allow(unused)] fn new_unmetered( - signals: metered::MeteredReceiver, + signals: metered::MeteredReceiver, messages: SubsystemIncomingMessages, to_subsystems: ChannelsOut, to_overseer: metered::UnboundedMeteredSender, @@ -609,18 +523,19 @@ impl OverseerSubsystemContext { } #[async_trait::async_trait] -impl SubsystemContext for OverseerSubsystemContext { +impl SubsystemContext for OverseerSubsystemContext { type Message = M; + type Signal = Signal; type Sender = OverseerSubsystemSender; - async fn try_recv(&mut self) -> Result>, ()> { + async fn try_recv(&mut self) -> Result>, ()> { match poll!(self.recv()) { Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), Poll::Pending => Ok(None), } } - async fn recv(&mut self) -> SubsystemResult> { + async fn recv(&mut self) -> SubsystemResult> { loop { // If we have a message pending an overseer signal, we only poll for signals // in the meantime. @@ -702,7 +617,7 @@ impl SubsystemContext for OverseerSubsystemContext { }).await.map_err(Into::into) } - fn sender(&mut self) -> &mut OverseerSubsystemSender { + fn sender(&mut self) -> &mut Self::Sender { &mut self.to_subsystems } } @@ -888,54 +803,27 @@ where mut s: S, ) -> SubsystemResult<(Self, OverseerHandler)> where - CV: Subsystem> + Send, - CB: Subsystem> + Send, - CS: Subsystem> + Send, - SD: Subsystem> + Send, - AD: Subsystem> + Send, - AR: Subsystem> + Send, - BS: Subsystem> + Send, - BD: Subsystem> + Send, - P: Subsystem> + Send, - RA: Subsystem> + Send, - AS: Subsystem> + Send, - NB: Subsystem> + Send, - CA: Subsystem> + Send, - CG: Subsystem> + Send, - CP: Subsystem> + Send, - ApD: Subsystem> + Send, - ApV: Subsystem> + Send, - GS: Subsystem> + Send, + CV: Subsystem> + Send, + CB: Subsystem> + Send, + CS: Subsystem> + Send, + SD: Subsystem> + Send, + AD: Subsystem> + Send, + AR: Subsystem> + Send, + BS: Subsystem> + Send, + BD: Subsystem> + Send, + P: Subsystem> + Send, + RA: Subsystem> + Send, + AS: Subsystem> + Send, + NB: Subsystem> + Send, + CA: Subsystem> + Send, + CG: Subsystem> + Send, + CP: Subsystem> + Send, + ApD: Subsystem> + Send, + ApV: Subsystem> + Send, + GS: Subsystem> + Send, { - let (overseer, handler) = Self::builder() - .candidate_validation(all_subsystems.candidate_validation) - .candidate_backing(all_subsystems.candidate_backing) - .candidate_selection(all_subsystems.candidate_selection) - .statement_distribution(all_subsystems.statement_distribution) - .availability_distribution(all_subsystems.availability_distribution) - .availability_recovery(all_subsystems.availability_recovery) - .bitfield_signing(all_subsystems.bitfield_signing) - .bitfield_distribution(all_subsystems.bitfield_distribution) - .provisioner(all_subsystems.provisioner) - .runtime_api(all_subsystems.runtime_api) - .availability_store(all_subsystems.availability_store) - .network_bridge(all_subsystems.network_bridge) - .chain_api(all_subsystems.chain_api) - .collation_generation(all_subsystems.collation_generation) - .collator_protocol(all_subsystems.collator_protocol) - .approval_distribution(all_subsystems.approval_distribution) - .approval_voting(all_subsystems.approval_voting) - .gossip_support(all_subsystems.gossip_support) - .leaves(leaves.collect()) - .active_leaves(Default::default()) - .span_per_active_leaf(Default::default()) - .activation_external_listeners(Default::default()) - .metrics(::register(prometheus_registry)?) - .active_leaves(leaves.collect()) - .spawner(s) - .build(); - + let metrics: Metrics = ::register(prometheus_registry)?; // spawn the metrics metronome task { struct ExtractNameAndMeters; @@ -954,7 +842,7 @@ where } } - let subsystem_meters = subsystems.as_ref().map_subsystems(ExtractNameAndMeters); + let subsystem_meters = all_subsystems.as_ref().map_subsystems(ExtractNameAndMeters); let metronome_metrics = metrics.clone(); let metronome = Metronome::new(std::time::Duration::from_millis(950)) .for_each(move |_| { @@ -973,48 +861,41 @@ where s.spawn("metrics_metronome", Box::pin(metronome)); } + let (overseer, handler) = Self::builder() + .candidate_validation(all_subsystems.candidate_validation) + .candidate_backing(all_subsystems.candidate_backing) + .candidate_selection(all_subsystems.candidate_selection) + .statement_distribution(all_subsystems.statement_distribution) + .availability_distribution(all_subsystems.availability_distribution) + .availability_recovery(all_subsystems.availability_recovery) + .bitfield_signing(all_subsystems.bitfield_signing) + .bitfield_distribution(all_subsystems.bitfield_distribution) + .provisioner(all_subsystems.provisioner) + .runtime_api(all_subsystems.runtime_api) + .availability_store(all_subsystems.availability_store) + .network_bridge(all_subsystems.network_bridge) + .chain_api(all_subsystems.chain_api) + .collation_generation(all_subsystems.collation_generation) + .collator_protocol(all_subsystems.collator_protocol) + .approval_distribution(all_subsystems.approval_distribution) + .approval_voting(all_subsystems.approval_voting) + .gossip_support(all_subsystems.gossip_support) + .leaves(leaves.collect()) + .active_leaves(Default::default()) + .span_per_active_leaf(Default::default()) + .activation_external_listeners(Default::default()) + .metrics(metrics) + .spawner(s) + .build(); Ok((overseer, OverseerHandler { handler })) } // Stop the overseer. - async fn stop(mut self) { - let _ = self.subsystems.candidate_validation.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.candidate_backing.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.candidate_selection.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.statement_distribution.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.availability_distribution.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.availability_recovery.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.bitfield_signing.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.bitfield_distribution.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.provisioner.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.runtime_api.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.availability_store.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.network_bridge.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.chain_api.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.collator_protocol.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.collation_generation.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.approval_distribution.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.approval_voting.send_signal(OverseerSignal::Conclude).await; - let _ = self.subsystems.gossip_support.send_signal(OverseerSignal::Conclude).await; - - let mut stop_delay = Delay::new(Duration::from_secs(STOP_DELAY)).fuse(); - - loop { - select! { - _ = self.running_subsystems.next() => { - if self.running_subsystems.is_empty() { - break; - } - }, - _ = stop_delay => break, - complete => break, - } - } + async fn stop2(mut self) { + self.stop(Overseer::Conclude).await } - - /// Run the `Overseer`. #[tracing::instrument(skip(self), fields(subsystem = LOG_TARGET))] pub async fn run(mut self) -> SubsystemResult<()> { @@ -1160,93 +1041,6 @@ where Ok(()) } - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - async fn broadcast_signal(&mut self, signal: OverseerSignal) -> SubsystemResult<()> { - self.subsystems.candidate_validation.send_signal(signal.clone()).await?; - self.subsystems.candidate_backing.send_signal(signal.clone()).await?; - self.subsystems.candidate_selection.send_signal(signal.clone()).await?; - self.subsystems.statement_distribution.send_signal(signal.clone()).await?; - self.subsystems.availability_distribution.send_signal(signal.clone()).await?; - self.subsystems.availability_recovery.send_signal(signal.clone()).await?; - self.subsystems.bitfield_signing.send_signal(signal.clone()).await?; - self.subsystems.bitfield_distribution.send_signal(signal.clone()).await?; - self.subsystems.provisioner.send_signal(signal.clone()).await?; - self.subsystems.runtime_api.send_signal(signal.clone()).await?; - self.subsystems.availability_store.send_signal(signal.clone()).await?; - self.subsystems.network_bridge.send_signal(signal.clone()).await?; - self.subsystems.chain_api.send_signal(signal.clone()).await?; - self.subsystems.collator_protocol.send_signal(signal.clone()).await?; - self.subsystems.collation_generation.send_signal(signal.clone()).await?; - self.subsystems.approval_distribution.send_signal(signal.clone()).await?; - self.subsystems.approval_voting.send_signal(signal.clone()).await?; - self.subsystems.gossip_support.send_signal(signal).await?; - - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - async fn route_message(&mut self, msg: AllMessages) -> SubsystemResult<()> { - self.metrics.on_message_relayed(); - match msg { - AllMessages::CandidateValidation(msg) => { - self.subsystems.candidate_validation.send_message(msg).await?; - }, - AllMessages::CandidateBacking(msg) => { - self.subsystems.candidate_backing.send_message(msg).await?; - }, - AllMessages::CandidateSelection(msg) => { - self.subsystems.candidate_selection.send_message(msg).await?; - }, - AllMessages::StatementDistribution(msg) => { - self.subsystems.statement_distribution.send_message(msg).await?; - }, - AllMessages::AvailabilityDistribution(msg) => { - self.subsystems.availability_distribution.send_message(msg).await?; - }, - AllMessages::AvailabilityRecovery(msg) => { - self.subsystems.availability_recovery.send_message(msg).await?; - }, - AllMessages::BitfieldDistribution(msg) => { - self.subsystems.bitfield_distribution.send_message(msg).await?; - }, - AllMessages::BitfieldSigning(msg) => { - self.subsystems.bitfield_signing.send_message(msg).await?; - }, - AllMessages::Provisioner(msg) => { - self.subsystems.provisioner.send_message(msg).await?; - }, - AllMessages::RuntimeApi(msg) => { - self.subsystems.runtime_api.send_message(msg).await?; - }, - AllMessages::AvailabilityStore(msg) => { - self.subsystems.availability_store.send_message(msg).await?; - }, - AllMessages::NetworkBridge(msg) => { - self.subsystems.network_bridge.send_message(msg).await?; - }, - AllMessages::ChainApi(msg) => { - self.subsystems.chain_api.send_message(msg).await?; - }, - AllMessages::CollationGeneration(msg) => { - self.subsystems.collation_generation.send_message(msg).await?; - }, - AllMessages::CollatorProtocol(msg) => { - self.subsystems.collator_protocol.send_message(msg).await?; - }, - AllMessages::ApprovalDistribution(msg) => { - self.subsystems.approval_distribution.send_message(msg).await?; - }, - AllMessages::ApprovalVoting(msg) => { - self.subsystems.approval_voting.send_message(msg).await?; - }, - AllMessages::GossipSupport(msg) => { - self.subsystems.gossip_support.send_message(msg).await?; - }, - } - - Ok(()) - } - /// Handles a header activation. If the header's state doesn't support the parachains API, /// this returns `None`. #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] @@ -1320,63 +1114,6 @@ enum TaskKind { Blocking, } -fn spawn( - spawner: &mut S, - message_tx: metered::MeteredSender>, - message_rx: SubsystemIncomingMessages, - unbounded_meter: metered::Meter, - to_subsystems: ChannelsOut, - to_overseer_tx: metered::UnboundedMeteredSender, - s: impl Subsystem>, - metrics: &Metrics, - futures: &mut FuturesUnordered>>, - task_kind: TaskKind, -) -> SubsystemResult> { - let (signal_tx, signal_rx) = metered::channel(SIGNAL_CHANNEL_CAPACITY); - let ctx = OverseerSubsystemContext::new( - signal_rx, - message_rx, - to_subsystems, - to_overseer_tx, - metrics.clone(), - ); - let SpawnedSubsystem { future, name } = s.start(ctx); - - let (tx, rx) = oneshot::channel(); - - let fut = Box::pin(async move { - if let Err(e) = future.await { - tracing::error!(subsystem=name, err = ?e, "subsystem exited with error"); - } else { - tracing::debug!(subsystem=name, "subsystem exited without an error"); - } - let _ = tx.send(()); - }); - - match task_kind { - TaskKind::Regular => spawner.spawn(name, fut), - TaskKind::Blocking => spawner.spawn_blocking(name, fut), - } - - futures.push(Box::pin(rx.map(|e| { tracing::warn!(err = ?e, "dropping error"); Ok(()) }))); - - let instance = Some(SubsystemInstance { - meters: SubsystemMeters { - unbounded: unbounded_meter, - bounded: message_tx.meter().clone(), - signals: signal_tx.meter().clone(), - }, - tx_signal: signal_tx, - tx_bounded: message_tx, - signals_received: 0, - name, - }); - - Ok(OverseenSubsystem { - instance, - }) -} - #[cfg(test)] mod tests { use std::sync::atomic; From cc9068696f7f6a81e123c98c45b71175a441e4fb Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 19 May 2021 18:35:31 +0200 Subject: [PATCH 038/161] shuffle --- node/collation-generation/src/lib.rs | 2 +- node/core/approval-voting/src/import.rs | 11 +- node/core/approval-voting/src/lib.rs | 12 +- node/core/approval-voting/src/tests.rs | 1 - node/core/av-store/src/lib.rs | 4 +- node/core/av-store/src/tests.rs | 2 +- node/core/backing/src/lib.rs | 32 +- node/core/bitfield-signing/src/lib.rs | 7 +- node/core/candidate-selection/src/lib.rs | 15 +- node/core/candidate-validation/src/lib.rs | 10 +- node/core/chain-api/src/lib.rs | 2 +- node/core/provisioner/src/lib.rs | 10 +- node/core/runtime-api/src/lib.rs | 2 +- node/network/approval-distribution/src/lib.rs | 18 +- .../src/requester/fetch_task/tests.rs | 2 - .../src/requester/mod.rs | 3 +- node/network/availability-recovery/src/lib.rs | 14 +- node/network/bridge/src/lib.rs | 18 +- node/network/bridge/src/multiplexer.rs | 1 - .../collator-protocol/src/collator_side.rs | 22 +- .../collator-protocol/src/validator_side.rs | 10 +- node/network/gossip-support/src/lib.rs | 6 +- .../network/statement-distribution/src/lib.rs | 38 +- node/overseer/examples/minimal-example.rs | 6 +- .../proc-macro/src/impl_dispatch.rs | 3 +- .../proc-macro/src/impl_message_wrapper.rs | 27 - .../proc-macro/src/impl_overseer.rs | 173 +-- node/overseer/overseer-gen/src/lib.rs | 120 +- node/overseer/src/lib.rs | 1331 ++--------------- node/overseer/src/subsystems.rs | 241 +++ node/overseer/src/tests.rs | 2 +- node/subsystem-test-helpers/src/lib.rs | 3 +- node/subsystem-util/Cargo.toml | 1 + node/subsystem-util/src/lib.rs | 34 +- node/subsystem-util/src/runtime/mod.rs | 6 +- node/subsystem/Cargo.toml | 1 + node/subsystem/src/lib.rs | 161 -- node/subsystem/src/messages.rs | 94 -- 38 files changed, 622 insertions(+), 1823 deletions(-) create mode 100644 node/overseer/src/subsystems.rs diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index d1adcc32dd70..e8f5f0581cf4 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -30,7 +30,7 @@ use polkadot_node_primitives::{ CollationGenerationConfig, AvailableData, PoV, }; use polkadot_node_subsystem::{ - messages::{AllMessages, CollationGenerationMessage, CollatorProtocolMessage}, + messages::{CollationGenerationMessage, CollatorProtocolMessage}, FromOverseer, SpawnedSubsystem, Subsystem, SubsystemContext, SubsystemResult, }; use polkadot_node_subsystem_util::{ diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index c783f49f26c1..f8e96102d5ff 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -94,7 +94,7 @@ impl RollingSessionWindow { // This returns the entire ancestry up to the last finalized block's height or the last item we // have in the DB. This may be somewhat expensive when first recovering from major sync. async fn determine_new_blocks( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, db: &impl DBReader, head: Hash, header: &Header, @@ -196,7 +196,7 @@ async fn determine_new_blocks( struct SessionsUnavailable; async fn load_all_sessions( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, block_hash: Hash, start: SessionIndex, end_inclusive: SessionIndex, @@ -238,7 +238,7 @@ async fn load_all_sessions( // // some backwards drift in session index is acceptable. async fn cache_session_info_for_head( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, session_window: &mut RollingSessionWindow, block_hash: Hash, block_header: &Header, @@ -357,7 +357,7 @@ struct ImportedBlockInfoEnv<'a> { // Computes information about the imported block. Returns `None` if the info couldn't be extracted - // failure to communicate with overseer, async fn imported_block_info( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, env: ImportedBlockInfoEnv<'_>, block_hash: Hash, block_header: &Header, @@ -558,7 +558,7 @@ pub struct BlockImportedCandidates { /// /// It is the responsibility of the caller to schedule wakeups for each block. pub(crate) async fn handle_new_head( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, db_writer: &dyn KeyValueDB, db_config: DatabaseConfig, @@ -816,7 +816,6 @@ mod tests { use polkadot_node_subsystem_test_helpers::make_subsystem_context; use polkadot_node_primitives::approval::{VRFOutput, VRFProof}; use polkadot_primitives::v1::ValidatorIndex; - use polkadot_node_subsystem::messages::AllMessages; use sp_core::testing::TaskExecutor; use sp_runtime::{Digest, DigestItem}; use sp_consensus_babe::{ diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index eb287a75b4cb..9bf4337aed97 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -679,7 +679,7 @@ async fn run( // returns `true` if any of the actions was a `Conclude` command. async fn handle_actions( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, wakeups: &mut Wakeups, db: &dyn KeyValueDB, @@ -878,7 +878,7 @@ fn distribution_messages_for_activation<'a>( // Handle an incoming signal from the overseer. Returns true if execution should conclude. async fn handle_from_overseer( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, metrics: &Metrics, db_writer: &dyn KeyValueDB, @@ -992,7 +992,7 @@ async fn handle_from_overseer( } async fn handle_background_request( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &State, metrics: &Metrics, request: BackgroundRequest, @@ -1022,7 +1022,7 @@ async fn handle_background_request( } async fn handle_approved_ancestor( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, db: &impl DBReader, target: Hash, lower_bound: BlockNumber, @@ -1868,7 +1868,7 @@ fn process_wakeup( // spawned. When the background work is no longer needed, the `AbortHandle` should be dropped // to cancel the background work and any requests it has spawned. async fn launch_approval( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, mut background_tx: mpsc::Sender, session_index: SessionIndex, candidate: &CandidateReceipt, @@ -2029,7 +2029,7 @@ async fn launch_approval( // Issue and import a local approval vote. Should only be invoked after approval checks // have been done. fn issue_approval( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &State, metrics: &Metrics, request: ApprovalVoteRequest, diff --git a/node/core/approval-voting/src/tests.rs b/node/core/approval-voting/src/tests.rs index 95b76d8a5526..e6805479b044 100644 --- a/node/core/approval-voting/src/tests.rs +++ b/node/core/approval-voting/src/tests.rs @@ -21,7 +21,6 @@ use polkadot_node_primitives::approval::{ RELAY_VRF_MODULO_CONTEXT, DelayTranche, }; use polkadot_node_subsystem_test_helpers::make_subsystem_context; -use polkadot_node_subsystem::messages::AllMessages; use sp_core::testing::TaskExecutor; use parking_lot::Mutex; diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index efe0379a4a80..8ed7b8873043 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -576,7 +576,7 @@ where } async fn process_block_activated( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, subsystem: &mut AvailabilityStoreSubsystem, activated: Hash, ) -> Result<(), Error> { @@ -769,7 +769,7 @@ macro_rules! peek_num { } async fn process_block_finalized( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, subsystem: &AvailabilityStoreSubsystem, finalized_hash: Hash, finalized_number: BlockNumber, diff --git a/node/core/av-store/src/tests.rs b/node/core/av-store/src/tests.rs index 78c080f367d1..f720106850b9 100644 --- a/node/core/av-store/src/tests.rs +++ b/node/core/av-store/src/tests.rs @@ -32,7 +32,7 @@ use polkadot_primitives::v1::{ use polkadot_node_primitives::{AvailableData, BlockData, PoV}; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_subsystem::{ - ActiveLeavesUpdate, errors::RuntimeApiError, jaeger, messages::AllMessages, ActivatedLeaf, + ActiveLeavesUpdate, errors::RuntimeApiError, jaeger, ActivatedLeaf, }; use polkadot_node_subsystem_test_helpers as test_helpers; use sp_keyring::Sr25519Keyring; diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index 49d739ff2c3f..90d5da069ead 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -293,7 +293,7 @@ fn table_attested_to_backed( } async fn store_available_data( - sender: &mut JobSender, + sender: &mut JobSender>, id: Option, n_validators: u32, candidate_hash: CandidateHash, @@ -319,7 +319,7 @@ async fn store_available_data( // This returns `Err()` iff there is an internal error. Otherwise, it returns either `Ok(Ok(()))` or `Ok(Err(_))`. #[tracing::instrument(level = "trace", skip(sender, pov, span), fields(subsystem = LOG_TARGET))] async fn make_pov_available( - sender: &mut JobSender, + sender: &mut JobSender>, validator_index: Option, n_validators: usize, pov: Arc, @@ -369,7 +369,7 @@ async fn make_pov_available( } async fn request_pov( - sender: &mut JobSender, + sender: &mut JobSender>, relay_parent: Hash, from_validator: ValidatorIndex, candidate_hash: CandidateHash, @@ -390,7 +390,7 @@ async fn request_pov( } async fn request_candidate_validation( - sender: &mut JobSender, + sender: &mut JobSender>, candidate: CandidateDescriptor, pov: Arc, ) -> Result { @@ -428,7 +428,7 @@ struct BackgroundValidationParams { async fn validate_and_make_available( params: BackgroundValidationParams< - impl SubsystemSender, + impl SubsystemSender , impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Sync, > ) -> Result<(), Error> { @@ -541,7 +541,7 @@ impl CandidateBackingJob { /// Run asynchronously. async fn run_loop( mut self, - mut sender: JobSender, + mut sender: JobSender>, mut rx_to: mpsc::Receiver, span: PerLeafSpan, ) -> Result<(), Error> { @@ -574,7 +574,7 @@ impl CandidateBackingJob { async fn handle_validated_candidate_command( &mut self, root_span: &jaeger::Span, - sender: &mut JobSender, + sender: &mut JobSender>, command: ValidatedCandidateCommand, ) -> Result<(), Error> { let candidate_hash = command.candidate_hash(); @@ -650,9 +650,9 @@ impl CandidateBackingJob { #[tracing::instrument(level = "trace", skip(self, sender, params), fields(subsystem = LOG_TARGET))] async fn background_validate_and_make_available( &mut self, - sender: &mut JobSender, + sender: &mut JobSender<>>, params: BackgroundValidationParams< - impl SubsystemSender, + impl SubsystemSender , impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Send + 'static + Sync >, ) -> Result<(), Error> { @@ -676,7 +676,7 @@ impl CandidateBackingJob { &mut self, parent_span: &jaeger::Span, root_span: &jaeger::Span, - sender: &mut JobSender, + sender: &mut JobSender>, candidate: &CandidateReceipt, pov: Arc, ) -> Result<(), Error> { @@ -727,7 +727,7 @@ impl CandidateBackingJob { async fn sign_import_and_distribute_statement( &mut self, - sender: &mut JobSender, + sender: &mut JobSender>, statement: Statement, root_span: &jaeger::Span, ) -> Result, Error> { @@ -744,7 +744,7 @@ impl CandidateBackingJob { /// Check if there have happened any new misbehaviors and issue necessary messages. #[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))] - async fn issue_new_misbehaviors(&mut self, sender: &mut JobSender) { + async fn issue_new_misbehaviors(&mut self, sender: &mut JobSender>) { // collect the misbehaviors to avoid double mutable self borrow issues let misbehaviors: Vec<_> = self.table.drain_misbehaviors().collect(); for (validator_id, report) in misbehaviors { @@ -761,7 +761,7 @@ impl CandidateBackingJob { #[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))] async fn import_statement( &mut self, - sender: &mut JobSender, + sender: &mut JobSender>, statement: &SignedFullStatement, root_span: &jaeger::Span, ) -> Result, Error> { @@ -831,7 +831,7 @@ impl CandidateBackingJob { async fn process_msg( &mut self, root_span: &jaeger::Span, - sender: &mut JobSender, + sender: &mut JobSender>, msg: CandidateBackingMessage, ) -> Result<(), Error> { match msg { @@ -897,7 +897,7 @@ impl CandidateBackingJob { #[tracing::instrument(level = "trace", skip(self, sender, attesting, span), fields(subsystem = LOG_TARGET))] async fn kick_off_validation_work( &mut self, - sender: &mut JobSender, + sender: &mut JobSender>, attesting: AttestingData, span: Option, ) -> Result<(), Error> { @@ -954,7 +954,7 @@ impl CandidateBackingJob { async fn maybe_validate_and_import( &mut self, root_span: &jaeger::Span, - sender: &mut JobSender, + sender: &mut JobSender>, statement: SignedFullStatement, ) -> Result<(), Error> { if let Some(summary) = self.import_statement(sender, &statement, root_span).await? { diff --git a/node/core/bitfield-signing/src/lib.rs b/node/core/bitfield-signing/src/lib.rs index 39ef5622a5d8..3ddc23c1db49 100644 --- a/node/core/bitfield-signing/src/lib.rs +++ b/node/core/bitfield-signing/src/lib.rs @@ -74,7 +74,7 @@ pub enum Error { async fn get_core_availability( core: &CoreState, validator_idx: ValidatorIndex, - sender: &Mutex<&mut impl SubsystemSender>, + sender: &Mutex<&mut impl SubsystemSender >, span: &jaeger::Span, ) -> Result { if let &CoreState::Occupied(ref core) = core { @@ -112,7 +112,7 @@ async fn get_core_availability( /// delegates to the v1 runtime API async fn get_availability_cores( relay_parent: Hash, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , ) -> Result, Error> { let (tx, rx) = oneshot::channel(); sender @@ -137,7 +137,7 @@ async fn construct_availability_bitfield( relay_parent: Hash, span: &jaeger::Span, validator_idx: ValidatorIndex, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , ) -> Result { // get the set of availability cores from the runtime let availability_cores = { @@ -319,7 +319,6 @@ mod tests { use super::*; use futures::{pin_mut, executor::block_on}; use polkadot_primitives::v1::{CandidateHash, OccupiedCore}; - use polkadot_node_subsystem::messages::AllMessages; fn occupied_core(para_id: u32, candidate_hash: CandidateHash) -> CoreState { CoreState::Occupied(OccupiedCore { diff --git a/node/core/candidate-selection/src/lib.rs b/node/core/candidate-selection/src/lib.rs index 37433ec6345c..6101b6dcefdd 100644 --- a/node/core/candidate-selection/src/lib.rs +++ b/node/core/candidate-selection/src/lib.rs @@ -224,7 +224,7 @@ impl CandidateSelectionJob { async fn run_loop( &mut self, span: &jaeger::Span, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , ) -> Result<(), Error> { let span = span.child("run-loop") .with_stage(jaeger::Stage::CandidateSelection); @@ -266,7 +266,7 @@ impl CandidateSelectionJob { #[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))] async fn handle_collation( &mut self, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , relay_parent: Hash, para_id: ParaId, collator_id: CollatorId, @@ -317,7 +317,7 @@ impl CandidateSelectionJob { #[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))] async fn handle_invalid( &mut self, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , candidate_receipt: CandidateReceipt, ) { let _timer = self.metrics.time_handle_invalid(); @@ -344,7 +344,7 @@ impl CandidateSelectionJob { async fn handle_seconded( &mut self, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , relay_parent: Hash, statement: SignedFullStatement, ) { @@ -386,7 +386,7 @@ async fn get_collation( relay_parent: Hash, para_id: ParaId, collator_id: CollatorId, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , ) -> Result<(CandidateReceipt, PoV), Error> { let (tx, rx) = oneshot::channel(); sender @@ -405,7 +405,7 @@ async fn second_candidate( relay_parent: Hash, candidate_receipt: CandidateReceipt, pov: PoV, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , metrics: &Metrics, ) { sender @@ -421,7 +421,7 @@ async fn second_candidate( async fn forward_invalidity_note( received_from: &CollatorId, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , ) { sender .send_message(CollatorProtocolMessage::ReportCollator(received_from.clone()).into()) @@ -516,7 +516,6 @@ mod tests { use super::*; use futures::lock::Mutex; use polkadot_node_primitives::BlockData; - use polkadot_node_subsystem::messages::AllMessages; use sp_core::crypto::Public; use std::sync::Arc; diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 11537b2e3e96..ec2ffc3dbc52 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -96,7 +96,7 @@ impl Subsystem for CandidateValidationSubsystem where #[tracing::instrument(skip(ctx, metrics), fields(subsystem = LOG_TARGET))] async fn run( - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, metrics: Metrics, cache_path: PathBuf, program_path: PathBuf, @@ -172,7 +172,7 @@ async fn run( } async fn runtime_api_request( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, relay_parent: Hash, request: RuntimeApiRequest, receiver: oneshot::Receiver>, @@ -196,7 +196,7 @@ enum AssumptionCheckOutcome { #[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn check_assumption_validation_data( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, descriptor: &CandidateDescriptor, assumption: OccupiedCoreAssumption, ) -> SubsystemResult { @@ -247,7 +247,7 @@ async fn check_assumption_validation_data( #[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn find_assumed_validation_data( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, descriptor: &CandidateDescriptor, ) -> SubsystemResult { // The candidate descriptor has a `persisted_validation_data_hash` which corresponds to @@ -283,7 +283,7 @@ async fn find_assumed_validation_data( fields(subsystem = LOG_TARGET), )] async fn spawn_validate_from_chain_state( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, validation_host: &mut ValidationHost, descriptor: CandidateDescriptor, pov: Arc, diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index 782fcb19ed06..eb0aa9d19e24 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -79,7 +79,7 @@ impl Subsystem for ChainApiSubsystem where #[tracing::instrument(skip(ctx, subsystem), fields(subsystem = LOG_TARGET))] async fn run( - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, subsystem: ChainApiSubsystem, ) -> SubsystemResult<()> where diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index 5320fb931dd7..8eeb54982e21 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -181,7 +181,7 @@ impl ProvisioningJob { async fn run_loop( mut self, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , span: PerLeafSpan, ) -> Result<(), Error> { use ProvisionerMessage::{ @@ -223,7 +223,7 @@ impl ProvisioningJob { async fn send_inherent_data( &mut self, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , return_senders: Vec>, ) { if let Err(err) = send_inherent_data( @@ -283,7 +283,7 @@ async fn send_inherent_data( bitfields: &[SignedAvailabilityBitfield], candidates: &[CandidateReceipt], return_senders: Vec>, - from_job: &mut impl SubsystemSender, + from_job: &mut impl SubsystemSender , ) -> Result<(), Error> { let availability_cores = request_availability_cores(relay_parent, from_job) .await @@ -359,7 +359,7 @@ async fn select_candidates( bitfields: &[SignedAvailabilityBitfield], candidates: &[CandidateReceipt], relay_parent: Hash, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , ) -> Result, Error> { let block_number = get_block_number_under_construction(relay_parent, sender).await?; @@ -478,7 +478,7 @@ async fn select_candidates( #[tracing::instrument(level = "trace", skip(sender), fields(subsystem = LOG_TARGET))] async fn get_block_number_under_construction( relay_parent: Hash, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , ) -> Result { let (tx, rx) = oneshot::channel(); sender diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index a2cd09a8f380..25f65bdec521 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -264,7 +264,7 @@ impl RuntimeApiSubsystem where #[tracing::instrument(skip(ctx, subsystem), fields(subsystem = LOG_TARGET))] async fn run( - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, mut subsystem: RuntimeApiSubsystem, ) -> SubsystemResult<()> where Client: ProvideRuntimeApi + Send + Sync + 'static, diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 7c69ac136389..caf9e365b56a 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -186,7 +186,7 @@ enum PendingMessage { impl State { async fn handle_network_msg( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, event: NetworkBridgeEvent, ) { @@ -247,7 +247,7 @@ impl State { async fn handle_new_blocks( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, metas: Vec, ) { @@ -349,7 +349,7 @@ impl State { async fn process_incoming_peer_message( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, peer_id: PeerId, msg: protocol_v1::ApprovalDistributionMessage, @@ -437,7 +437,7 @@ impl State { async fn handle_peer_view_change( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, peer_id: PeerId, view: View, @@ -506,7 +506,7 @@ impl State { async fn import_and_circulate_assignment( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, source: MessageSource, assignment: IndirectAssignmentCert, @@ -722,7 +722,7 @@ impl State { async fn import_and_circulate_approval( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, source: MessageSource, vote: IndirectSignedApprovalVote, @@ -960,7 +960,7 @@ impl State { } async fn unify_with_peer( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, entries: &mut HashMap, peer_id: PeerId, @@ -1010,7 +1010,7 @@ impl State { async fn send_gossip_messages_to_peer( entries: &HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, peer_id: PeerId, blocks: Vec<(BlockDepth, Hash)>, ) { @@ -1112,7 +1112,7 @@ impl State { /// Modify the reputation of a peer based on its behavior. #[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn modify_reputation( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, peer_id: PeerId, rep: Rep, ) { diff --git a/node/network/availability-distribution/src/requester/fetch_task/tests.rs b/node/network/availability-distribution/src/requester/fetch_task/tests.rs index db8790435b2c..240cf8c5e9a6 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/tests.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/tests.rs @@ -30,7 +30,6 @@ use polkadot_primitives::v1::{CandidateHash, ValidatorIndex}; use polkadot_node_primitives::{BlockData, PoV}; use polkadot_node_network_protocol::request_response::v1; use polkadot_node_network_protocol::request_response::Recipient; -use polkadot_subsystem::messages::AllMessages; use crate::metrics::Metrics; use crate::tests::mock::get_valid_chunk_data; @@ -300,4 +299,3 @@ fn get_test_running_task() -> (RunningTask, mpsc::Receiver) { rx ) } - diff --git a/node/network/availability-distribution/src/requester/mod.rs b/node/network/availability-distribution/src/requester/mod.rs index b91297987b27..306cd1b96e76 100644 --- a/node/network/availability-distribution/src/requester/mod.rs +++ b/node/network/availability-distribution/src/requester/mod.rs @@ -33,7 +33,7 @@ use futures::{ use polkadot_node_subsystem_util::runtime::{RuntimeInfo, get_occupied_cores}; use polkadot_primitives::v1::{CandidateHash, Hash, OccupiedCore}; use polkadot_subsystem::{ - messages::AllMessages, ActiveLeavesUpdate, SubsystemContext, ActivatedLeaf, + ActiveLeavesUpdate, SubsystemContext, ActivatedLeaf, }; use super::{LOG_TARGET, Metrics}; @@ -231,4 +231,3 @@ impl Stream for Requester { } } } - diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index f8eab8689a95..79c1238b7bff 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -132,7 +132,7 @@ impl RequestFromBackersPhase { async fn run( &mut self, params: &InteractionParams, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , ) -> Result { tracing::trace!( target: LOG_TARGET, @@ -217,7 +217,7 @@ impl RequestChunksPhase { async fn launch_parallel_requests( &mut self, params: &InteractionParams, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , ) { let max_requests = std::cmp::min(N_PARALLEL, params.threshold); while self.requesting_chunks.len() < max_requests { @@ -329,7 +329,7 @@ impl RequestChunksPhase { async fn run( &mut self, params: &InteractionParams, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , ) -> Result { // First query the store for any chunks we've got. { @@ -611,7 +611,7 @@ async fn handle_signal( #[tracing::instrument(level = "trace", skip(ctx, state), fields(subsystem = LOG_TARGET))] async fn launch_interaction( state: &mut State, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, session_index: SessionIndex, session_info: SessionInfo, receipt: CandidateReceipt, @@ -666,7 +666,7 @@ async fn launch_interaction( #[tracing::instrument(level = "trace", skip(ctx, state), fields(subsystem = LOG_TARGET))] async fn handle_recover( state: &mut State, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, receipt: CandidateReceipt, session_index: SessionIndex, backing_group: Option, @@ -729,7 +729,7 @@ async fn handle_recover( /// Queries a chunk from av-store. #[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn query_full_data( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, candidate_hash: CandidateHash, ) -> error::Result> { let (tx, rx) = oneshot::channel(); @@ -753,7 +753,7 @@ impl AvailabilityRecoverySubsystem { async fn run( self, - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, ) -> SubsystemResult<()> { let mut state = State::default(); diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index fcaf7f0f7eca..b02ec9723e3f 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -538,7 +538,7 @@ where } async fn handle_network_messages( - mut sender: impl SubsystemSender, + mut sender: impl SubsystemSender , mut network_service: impl Network, mut network_stream: BoxStream<'static, NetworkEvent>, mut authority_discovery_service: AD, @@ -805,7 +805,7 @@ async fn handle_network_messages( #[tracing::instrument(skip(bridge, ctx, network_stream), fields(subsystem = LOG_TARGET))] async fn run_network( bridge: NetworkBridge, - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, network_stream: BoxStream<'static, NetworkEvent>, ) -> SubsystemResult<()> where @@ -902,7 +902,7 @@ fn construct_view(live_heads: impl DoubleEndedIterator, finalized_n #[tracing::instrument(level = "trace", skip(net, ctx, shared, metrics), fields(subsystem = LOG_TARGET))] async fn update_our_view( net: &mut impl Network, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, live_heads: &[ActivatedLeaf], shared: &Shared, finalized_number: BlockNumber, @@ -1054,21 +1054,21 @@ async fn send_collation_message( async fn dispatch_validation_event_to_all( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) { dispatch_validation_events_to_all(std::iter::once(event), ctx).await } async fn dispatch_collation_event_to_all( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) { dispatch_collation_events_to_all(std::iter::once(event), ctx).await } fn dispatch_validation_event_to_all_unbounded( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) { for msg in AllMessages::dispatch_iter(event) { ctx.send_unbounded_message(msg); @@ -1077,7 +1077,7 @@ fn dispatch_validation_event_to_all_unbounded( fn dispatch_collation_event_to_all_unbounded( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) { if let Some(msg) = event.focus().ok().map(CollatorProtocolMessage::NetworkBridgeUpdateV1) { ctx.send_unbounded_message(msg.into()); @@ -1087,7 +1087,7 @@ fn dispatch_collation_event_to_all_unbounded( #[tracing::instrument(level = "trace", skip(events, ctx), fields(subsystem = LOG_TARGET))] async fn dispatch_validation_events_to_all( events: I, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) where I: IntoIterator>, @@ -1099,7 +1099,7 @@ async fn dispatch_validation_events_to_all( #[tracing::instrument(level = "trace", skip(events, ctx), fields(subsystem = LOG_TARGET))] async fn dispatch_collation_events_to_all( events: I, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) where I: IntoIterator>, diff --git a/node/network/bridge/src/multiplexer.rs b/node/network/bridge/src/multiplexer.rs index ad65309d3eea..e7d958dd8676 100644 --- a/node/network/bridge/src/multiplexer.rs +++ b/node/network/bridge/src/multiplexer.rs @@ -29,7 +29,6 @@ use sc_network::PeerId; use polkadot_node_network_protocol::request_response::{ request::IncomingRequest, v1, Protocol, RequestResponseConfig, }; -use polkadot_subsystem::messages::AllMessages; /// Multiplex incoming network requests. /// diff --git a/node/network/collator-protocol/src/collator_side.rs b/node/network/collator-protocol/src/collator_side.rs index 7a3a22dff8d9..acfcda217957 100644 --- a/node/network/collator-protocol/src/collator_side.rs +++ b/node/network/collator-protocol/src/collator_side.rs @@ -264,7 +264,7 @@ impl State { /// elsewhere in the node. #[tracing::instrument(level = "trace", skip(ctx, runtime, state, pov), fields(subsystem = LOG_TARGET))] async fn distribute_collation( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, runtime: &mut RuntimeInfo, state: &mut State, id: ParaId, @@ -370,7 +370,7 @@ async fn distribute_collation( /// and the total number of cores. #[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn determine_core( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, para_id: ParaId, relay_parent: Hash, ) -> Result> { @@ -400,7 +400,7 @@ struct GroupValidators { /// Returns [`ValidatorId`]'s of current and next group as determined based on the `relay_parent`. #[tracing::instrument(level = "trace", skip(ctx, runtime), fields(subsystem = LOG_TARGET))] async fn determine_our_validators( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, runtime: &mut RuntimeInfo, core_index: CoreIndex, cores: usize, @@ -437,7 +437,7 @@ async fn determine_our_validators( /// Issue a `Declare` collation message to the given `peer`. #[tracing::instrument(level = "trace", skip(ctx, state), fields(subsystem = LOG_TARGET))] async fn declare( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, peer: PeerId, ) { @@ -463,7 +463,7 @@ async fn declare( /// revoke the previous connection request. #[tracing::instrument(level = "trace", skip(ctx, state), fields(subsystem = LOG_TARGET))] async fn connect_to_validators( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, group: GroupValidators, ) { @@ -482,7 +482,7 @@ async fn connect_to_validators( /// set as validator for our para at the given `relay_parent`. #[tracing::instrument(level = "trace", skip(ctx, state), fields(subsystem = LOG_TARGET))] async fn advertise_collation( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, relay_parent: Hash, peer: PeerId, @@ -543,7 +543,7 @@ async fn advertise_collation( /// The main incoming message dispatching switch. #[tracing::instrument(level = "trace", skip(ctx, runtime, state), fields(subsystem = LOG_TARGET))] async fn process_msg( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, runtime: &mut RuntimeInfo, state: &mut State, msg: CollatorProtocolMessage, @@ -684,7 +684,7 @@ async fn send_collation( /// A networking messages switch. #[tracing::instrument(level = "trace", skip(ctx, runtime, state), fields(subsystem = LOG_TARGET))] async fn handle_incoming_peer_message( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, runtime: &mut RuntimeInfo, state: &mut State, origin: PeerId, @@ -756,7 +756,7 @@ async fn handle_incoming_peer_message( /// Our view has changed. #[tracing::instrument(level = "trace", skip(ctx, state), fields(subsystem = LOG_TARGET))] async fn handle_peer_view_change( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, peer_id: PeerId, view: View, @@ -775,7 +775,7 @@ async fn handle_peer_view_change( /// Bridge messages switch. #[tracing::instrument(level = "trace", skip(ctx, runtime, state), fields(subsystem = LOG_TARGET))] async fn handle_network_msg( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, runtime: &mut RuntimeInfo, state: &mut State, bridge_message: NetworkBridgeEvent, @@ -883,7 +883,7 @@ async fn handle_our_view_change( /// The collator protocol collator side main loop. #[tracing::instrument(skip(ctx, collator_pair, metrics), fields(subsystem = LOG_TARGET))] pub(crate) async fn run( - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, local_peer_id: PeerId, collator_pair: CollatorPair, metrics: Metrics, diff --git a/node/network/collator-protocol/src/validator_side.rs b/node/network/collator-protocol/src/validator_side.rs index 2a161c65d832..7584303c22d0 100644 --- a/node/network/collator-protocol/src/validator_side.rs +++ b/node/network/collator-protocol/src/validator_side.rs @@ -312,7 +312,7 @@ struct ActiveParas { impl ActiveParas { async fn assign_incoming( &mut self, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , keystore: &SyncCryptoStorePtr, new_relay_parents: impl IntoIterator, ) { @@ -477,7 +477,7 @@ fn collator_peer_id( ) } -async fn disconnect_peer(ctx: &mut impl SubsystemContext, peer_id: PeerId) { +async fn disconnect_peer(ctx: &mut impl SubsystemContext, peer_id: PeerId) { ctx.send_message( NetworkBridgeMessage::DisconnectPeer(peer_id, PeerSet::Collation).into() ).await @@ -539,7 +539,7 @@ where /// Notify a collator that its collation got seconded. #[tracing::instrument(level = "trace", skip(ctx, peer_data), fields(subsystem = LOG_TARGET))] async fn notify_collation_seconded( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, peer_data: &HashMap, id: CollatorId, relay_parent: Hash, @@ -813,7 +813,7 @@ async fn remove_relay_parent( /// Our view has changed. #[tracing::instrument(level = "trace", skip(ctx, state, keystore), fields(subsystem = LOG_TARGET))] async fn handle_our_view_change( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, keystore: &SyncCryptoStorePtr, view: OurView, @@ -1060,7 +1060,7 @@ pub(crate) async fn run( // earliest possible point. This does not yet clean up any metadata, as that will be done upon // receipt of the `PeerDisconnected` event. async fn disconnect_inactive_peers( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, eviction_policy: &crate::CollatorEvictionPolicy, peers: &HashMap, ) { diff --git a/node/network/gossip-support/src/lib.rs b/node/network/gossip-support/src/lib.rs index adfc6344e17a..4b44c2aa3b79 100644 --- a/node/network/gossip-support/src/lib.rs +++ b/node/network/gossip-support/src/lib.rs @@ -98,7 +98,7 @@ impl GossipSupport { } async fn determine_relevant_authorities( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, relay_parent: Hash, ) -> Result, util::Error> { let authorities = util::request_authorities(relay_parent, ctx.sender()).await.await??; @@ -122,7 +122,7 @@ async fn ensure_i_am_an_authority( /// A helper function for making a `ConnectToValidators` request. pub async fn connect_to_authorities( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, validator_ids: Vec, peer_set: PeerSet, ) -> oneshot::Sender<()> { @@ -145,7 +145,7 @@ impl State { /// and issue a connection request. async fn handle_active_leaves( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, keystore: &SyncCryptoStorePtr, leaves: impl Iterator, ) -> Result<(), util::Error> { diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index ccef05229f9f..fc1af1d39ea7 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -589,7 +589,7 @@ enum Message { impl Message { async fn receive( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, from_requester: &mut mpsc::Receiver, from_responder: &mut mpsc::Receiver, ) -> Message { @@ -848,7 +848,7 @@ fn check_statement_signature( async fn circulate_statement_and_dependents( peers: &mut HashMap, active_heads: &mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, relay_parent: Hash, statement: SignedFullStatement, priority_peers: Vec, @@ -903,7 +903,7 @@ async fn circulate_statement_and_dependents( fn statement_message(relay_parent: Hash, statement: SignedFullStatement) -> protocol_v1::ValidationProtocol -{ +{ let msg = if is_statement_large(&statement) { protocol_v1::StatementDistributionMessage::LargeStatement( StatementMetadata { @@ -948,7 +948,7 @@ fn is_statement_large(statement: &SignedFullStatement) -> bool { #[tracing::instrument(level = "trace", skip(peers, ctx), fields(subsystem = LOG_TARGET))] async fn circulate_statement<'a>( peers: &mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, relay_parent: Hash, stored: StoredStatement<'a>, mut priority_peers: Vec, @@ -1026,7 +1026,7 @@ async fn circulate_statement<'a>( async fn send_statements_about( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, relay_parent: Hash, candidate_hash: CandidateHash, active_head: &ActiveHeadData, @@ -1064,7 +1064,7 @@ async fn send_statements_about( async fn send_statements( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, relay_parent: Hash, active_head: &ActiveHeadData, metrics: &Metrics, @@ -1096,7 +1096,7 @@ async fn send_statements( } async fn report_peer( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, peer: PeerId, rep: Rep, ) { @@ -1116,7 +1116,7 @@ async fn retrieve_statement_from_message<'a>( peer: PeerId, message: protocol_v1::StatementDistributionMessage, active_head: &'a mut ActiveHeadData, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, req_sender: &mpsc::Sender, metrics: &Metrics, ) -> Option { @@ -1198,7 +1198,7 @@ async fn retrieve_statement_from_message<'a>( ).await { vacant.insert(new_status); } - } + } protocol_v1::StatementDistributionMessage::Statement(_, s) => { // No fetch in progress, safe to return any statement immediately (we don't bother // about normal network jitter which might cause `Valid` statements to arrive early @@ -1218,7 +1218,7 @@ async fn launch_request( meta: StatementMetadata, peer: PeerId, req_sender: mpsc::Sender, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, ) -> Option { @@ -1256,7 +1256,7 @@ async fn handle_incoming_message_and_circulate<'a>( peer: PeerId, peers: &mut HashMap, active_heads: &'a mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, @@ -1304,7 +1304,7 @@ async fn handle_incoming_message<'a>( peer: PeerId, peer_data: &mut PeerData, active_heads: &'a mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, @@ -1454,7 +1454,7 @@ async fn handle_incoming_message<'a>( async fn update_peer_view_and_send_unlocked( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, active_heads: &HashMap, new_view: View, metrics: &Metrics, @@ -1489,7 +1489,7 @@ async fn handle_network_update( peers: &mut HashMap, authorities: &mut HashMap, active_heads: &mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, req_sender: &mpsc::Sender, update: NetworkBridgeEvent, metrics: &Metrics, @@ -1563,7 +1563,7 @@ impl StatementDistribution { #[tracing::instrument(skip(self, ctx), fields(subsystem = LOG_TARGET))] async fn run( self, - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, ) -> std::result::Result<(), Fatal> { let mut peers: HashMap = HashMap::new(); let mut authorities: HashMap = HashMap::new(); @@ -1594,7 +1594,7 @@ impl StatementDistribution { match result { Ok(true) => break, Ok(false) => {} - Err(Error(Fault::Fatal(f))) => return Err(f), + Err(Error(Fault::Fatal(f))) => return Err(f), Err(Error(Fault::Err(error))) => tracing::debug!(target: LOG_TARGET, ?error) } @@ -1670,7 +1670,7 @@ impl StatementDistribution { async fn handle_requester_message( &self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, peers: &mut HashMap, active_heads: &mut HashMap, req_sender: &mpsc::Sender, @@ -1783,7 +1783,7 @@ impl StatementDistribution { async fn handle_subsystem_message( &self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, runtime: &mut RuntimeInfo, peers: &mut HashMap, authorities: &mut HashMap, @@ -3627,7 +3627,7 @@ mod tests { NetworkBridgeEvent::PeerViewChange(peer_a.clone(), view![hash_a]) ) }).await; - + // receive a seconded statement from peer A. let statement = { let signing_context = SigningContext { diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index 830c65e89d53..00e1ac7cada9 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -32,7 +32,7 @@ use polkadot_overseer::{Overseer, HeadSupportsParachains, AllSubsystems}; use polkadot_subsystem::{Subsystem, SubsystemContext, SpawnedSubsystem, FromOverseer}; use polkadot_subsystem::messages::{ - CandidateValidationMessage, CandidateBackingMessage, AllMessages, + CandidateValidationMessage, CandidateBackingMessage, }; struct AlwaysSupportsParachains; @@ -43,7 +43,7 @@ impl HeadSupportsParachains for AlwaysSupportsParachains { struct Subsystem1; impl Subsystem1 { - async fn run(mut ctx: impl SubsystemContext) { + async fn run(mut ctx: impl SubsystemContext) { loop { match ctx.try_recv().await { Ok(Some(msg)) => { @@ -94,7 +94,7 @@ impl Subsystem for Subsystem1 struct Subsystem2; impl Subsystem2 { - async fn run(mut ctx: impl SubsystemContext) { + async fn run(mut ctx: impl SubsystemContext) { ctx.spawn( "subsystem-2-job", Box::pin(async { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index 34ffd080a280..011dba1ee14c 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -25,12 +25,11 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { let dispatchable = info.subsystems().into_iter().filter(|ssf| !ssf.no_dispatch).map(|ssf| ssf.consumes.clone()).collect::>(); - let msg = "Generated by #[subsystem_dispatch_gen] proc-macro."; let extern_event_ty= &info.extern_event_ty.clone(); let ts = quote! { impl #message_wrapper { - #[doc = #msg] + /// Generated dispatch iterator generator. pub fn dispatch_iter(event: #extern_event_ty) -> impl Iterator + Send { let mut iter = None.into_iter(); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 7911e48dae46..2915c698a8ae 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -28,33 +28,6 @@ pub(crate) fn impl_message_wrapper_enum( } } )* - - #[derive(Debug, Clone)] - pub struct OverseerSubsystemSender { - channels: ChannelsOut, - signals_received: SignalsReceived, - } - - #[::polkadot_overseer_gen::async_trait] - impl SubsystemSender for OverseerSubsystemSender { - async fn send_message(&mut self, msg: #message_wrapper) { - self.channels.send_and_log_error(self.signals_received.load(), msg).await; - } - - async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send - { - // This can definitely be optimized if necessary. - for msg in msgs { - self.send_message(msg).await; - } - } - - fn send_unbounded_message(&mut self, msg: #message_wrapper) { - self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); - } - } - }; Ok(ts) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 66cb985c3e54..2bf27cb7d68c 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -77,24 +77,6 @@ pub(crate) fn impl_overseer_struct( } impl #generics #overseer_name #generics #where_clause { - pub async fn broadcast_signal(mut self, signal: #signal_ty) { - self.broadcast_signal(signal); - - let mut stop_delay = ::polkadot_overseer_gen::Delay::new(STOP_DELAY).fuse(); - - loop { - select! { - _ = self.running_subsystems.next() => { - if self.running_subsystems.is_empty() { - break; - } - }, - _ = stop_delay => break, - complete => break, - } - } - } - pub async fn broadcast_signal(&mut self, signal: #signal_ty) -> SubsystemResult<()> { #( self. #subsystem_name .send_signal(signal.clone()).await; @@ -116,11 +98,7 @@ pub(crate) fn impl_overseer_struct( }; ts.extend(impl_builder(info)?); - ts.extend(impl_subsystem_instance(info)?); ts.extend(impl_overseen_subsystem(info)?); - ts.extend(impl_trait_subsystem(info)?); - ts.extend(impl_trait_subsystem_sender(info)?); - Ok(ts) } @@ -201,7 +179,7 @@ pub(crate) fn impl_builder( #subsystem_name : ::std::option::Option< #builder_generic_ty >, )* #( - #baggage_name : ::std::option::Option< #baggage_name >, + #baggage_name : ::std::option::Option< #baggage_ty >, )* spawner: ::std::option::Option< S >, _phantom_ctx: ::std::marker::PhantomData< Ctx >, @@ -274,7 +252,7 @@ pub(crate) fn impl_builder( // FIXME generate a builder pattern that ensures this let #subsystem_name = self. #subsystem_name .expect("All subsystem must exist with the builder pattern."); - let #subsystem_name: OverseenSubsystem< #message_wrapper > = { + let #subsystem_name: OverseenSubsystem< #consumes > = { let unbounded_meter = channels_out. #channel_name .meter().clone(); @@ -286,7 +264,7 @@ pub(crate) fn impl_builder( channels_out. #channel_name_unbounded .clone(), ); - let (signal_tx, signal_rx) = ::polkadot_overseer_gen::metered::channel(SIGNAL_CHANNEL_CAPACITY); + let (signal_tx, signal_rx) = ::polkadot_overseer_gen::metered::channel::(SIGNAL_CHANNEL_CAPACITY); let ctx = create_subsystem_ctx( signal_rx, @@ -317,7 +295,7 @@ pub(crate) fn impl_builder( }))); let instance = Some( - SubsystemInstance::< #message_wrapper > { + SubsystemInstance::< #consumes > { meters: SubsystemMeters { unbounded: unbounded_meter, bounded: message_tx.meter().clone(), @@ -330,7 +308,7 @@ pub(crate) fn impl_builder( } ); - OverseenSubsystem::< #message_wrapper > { + OverseenSubsystem::< #consumes > { instance, } }; @@ -365,31 +343,9 @@ pub(crate) fn impl_builder( - -pub(crate) fn impl_subsystem_instance(info: &OverseerInfo) -> Result { - let signal = &info.extern_signal_ty; - - let ts = quote::quote! { - /// A running instance of some [`Subsystem`]. - /// - /// [`Subsystem`]: trait.Subsystem.html - /// - /// `M` here is the inner message type, and _not_ the generated `enum AllMessages`. - pub struct SubsystemInstance { - tx_signal: ::polkadot_overseer_gen::metered::MeteredSender< #signal >, - tx_bounded: ::polkadot_overseer_gen::metered::MeteredSender>, - meters: SubsystemMeters, - signals_received: usize, - name: &'static str, - } - }; - - Ok(ts) -} - - pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { let signal = &info.extern_signal_ty; + let message_wrapper = &info.message_wrapper; let ts = quote::quote! { @@ -400,15 +356,15 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { - pub instance: std::option::Option>, + pub struct OverseenSubsystem { + pub instance: std::option::Option>, } - impl OverseenSubsystem { + impl OverseenSubsystem { /// Send a message to the wrapped subsystem. /// /// If the inner `instance` is `None`, nothing is happening. - pub async fn send_message(&mut self, msg: M) -> SubsystemResult<()> { + pub async fn send_message(&mut self, msg: #message_wrapper) -> SubsystemResult<()> { const MESSAGE_TIMEOUT: Duration = Duration::from_secs(10); if let Some(ref mut instance) = self.instance { @@ -454,112 +410,3 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result Result { - let message_wrapper = &info.message_wrapper; - - let ts = quote! { - #[::polkadot_overseer_gen::async_trait] - pub trait SubsystemSender: Send + Clone + 'static { - /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: #message_wrapper); - - /// Send multiple direct messages to other `Subsystem`s, routed based on message type. - async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send; - - /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message - /// type. - /// - /// This function should be used only when there is some other bounding factor on the messages - /// sent with it. Otherwise, it risks a memory leak. - fn send_unbounded_message(&mut self, msg: #message_wrapper); - } - }; - Ok(ts) -} - - - -pub(crate) fn impl_trait_subsystem(info: &OverseerInfo) -> Result { - let message_wrapper = &info.message_wrapper; - let error = &info.extern_error_ty; - - let ts = quote! { - /// A message type that a subsystem receives from an overseer. - /// It wraps signals from an overseer and messages that are circulating - /// between subsystems. - /// - /// It is generic over over the message type `M` that a particular `Subsystem` may use. - #[derive(Debug)] - pub enum FromOverseer { - /// Signal from the `Overseer`. - Signal(Signal), - - /// Some other `Subsystem`'s message. - Communication { - /// Contained message - msg: M, - }, - } - - /// A context type that is given to the [`Subsystem`] upon spawning. - /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s - /// or spawn jobs. - /// - /// [`Overseer`]: struct.Overseer.html - /// [`SubsystemJob`]: trait.SubsystemJob.html - #[::polkadot_overseer_gen::async_trait] - pub trait SubsystemContext: Send + 'static { - /// The message type of this context. Subsystems launched with this context will expect - /// to receive messages of this type. - type Message: Send + 'static; - type Signal: Send + 'static; - type Sender: Send + Clone + 'static; - - /// Try to asynchronously receive a message. - /// - /// This has to be used with caution, if you loop over this without - /// using `pending!()` macro you will end up with a busy loop! - async fn try_recv(&mut self) -> Result>, ()>; - - /// Receive a message. - async fn recv(&mut self) -> SubsystemResult>; - - /// Spawn a child task on the executor. - async fn spawn(&mut self, name: &'static str, s: ::std::pin::Pin + Send>>) -> SubsystemResult<()>; - - /// Spawn a blocking child task on the executor's dedicated thread pool. - async fn spawn_blocking( - &mut self, - name: &'static str, - s: ::std::pin::Pin + Send>>, - ) -> SubsystemResult<()>; - - /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: #message_wrapper); - - /// Send multiple direct messages to other `Subsystem`s, routed based on message type. - async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send; - - /// Obtain the sender. - fn sender(&self) -> Self::Sender; - } - - /// A trait that describes the [`Subsystem`]s that can run on the [`Overseer`]. - /// - /// It is generic over the message type circulating in the system. - /// The idea that we want some type contaning persistent state that - /// can spawn actually running subsystems when asked to. - /// - /// [`Overseer`]: struct.Overseer.html - /// [`Subsystem`]: trait.Subsystem.html - pub trait Subsystem { - /// Start this `Subsystem` and return `SpawnedSubsystem`. - fn start(self, ctx: Ctx) -> SpawnedSubsystem < #error >; - } - }; - Ok(ts) -} diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index a789a5f26351..59b5ccc5cfdc 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -114,9 +114,11 @@ pub enum ToOverseer { /// A helper trait to map a subsystem to smth. else. -pub(crate) trait MapSubsystem { +pub trait MapSubsystem { + /// The output type of the mapping. type Output; + /// Consumes a `T` per subsystem, and maps it to `Self::Output`. fn map_subsystem(&self, sub: T) -> Self::Output; } @@ -184,11 +186,10 @@ pub trait AnnotateErrorOrigin: 'static + Send + Sync + std::error::Error { fn with_origin(self, origin: &'static str) -> Self; } - /// An asynchronous subsystem task.. /// /// In essence it's just a newtype wrapping a `BoxFuture`. -pub struct SpawnedSubsystem +pub struct SpawnedSubsystem where E: std::error::Error + 'static @@ -201,7 +202,6 @@ E: std::error::Error pub future: BoxFuture<'static, Result<(), E>>, } - /// An error type that describes faults that may happen /// /// These are: @@ -277,5 +277,117 @@ pub struct SubsystemMeterReadouts { pub signals: metered::Readout, } +/// A running instance of some [`Subsystem`]. +/// +/// [`Subsystem`]: trait.Subsystem.html +/// +/// `M` here is the inner message type, and _not_ the generated `enum AllMessages`. +pub struct SubsystemInstance { + tx_signal: crate::metered::MeteredSender, + tx_bounded: crate::metered::MeteredSender>, + meters: SubsystemMeters, + signals_received: usize, + name: &'static str, +} + +/// A message type that a subsystem receives from an overseer. +/// It wraps signals from an overseer and messages that are circulating +/// between subsystems. +/// +/// It is generic over over the message type `M` that a particular `Subsystem` may use. +#[derive(Debug)] +pub enum FromOverseer { + /// Signal from the `Overseer`. + Signal(Signal), + + /// Some other `Subsystem`'s message. + Communication { + /// Contained message + msg: Message, + }, +} + +/// A context type that is given to the [`Subsystem`] upon spawning. +/// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s +/// or spawn jobs. +/// +/// [`Overseer`]: struct.Overseer.html +/// [`SubsystemJob`]: trait.SubsystemJob.html +#[async_trait::async_trait] +pub trait SubsystemContext: Send + 'static { + /// The message type of this context. Subsystems launched with this context will expect + /// to receive messages of this type. + type Message: Send + 'static; + /// And the same for signals. + type Signal: Send + 'static; + /// The sender type as provided by `sender()` and underlying. + type Sender: SubsystemSender; + + /// Try to asynchronously receive a message. + /// + /// This has to be used with caution, if you loop over this without + /// using `pending!()` macro you will end up with a busy loop! + async fn try_recv(&mut self) -> Result>, ()>; + + /// Receive a message. + async fn recv(&mut self) -> SubsystemResult>; + + /// Spawn a child task on the executor. + async fn spawn(&mut self, name: &'static str, s: ::std::pin::Pin + Send>>) -> SubsystemResult<()>; + + /// Spawn a blocking child task on the executor's dedicated thread pool. + async fn spawn_blocking( + &mut self, + name: &'static str, + s: ::std::pin::Pin + Send>>, + ) -> SubsystemResult<()>; + + /// Send a direct message to some other `Subsystem`, routed based on message type. + async fn send_message(&mut self, msg: Self::Message); + + /// Send multiple direct messages to other `Subsystem`s, routed based on message type. + async fn send_messages(&mut self, msgs: T) + where T: IntoIterator + Send, T::IntoIter: Send; + + /// Obtain the sender. + fn sender(&self) -> Self::Sender; +} + +/// A trait that describes the [`Subsystem`]s that can run on the [`Overseer`]. +/// +/// It is generic over the message type circulating in the system. +/// The idea that we want some type contaning persistent state that +/// can spawn actually running subsystems when asked to. +/// +/// [`Overseer`]: struct.Overseer.html +/// [`Subsystem`]: trait.Subsystem.html +pub trait Subsystem +where + Ctx: SubsystemContext, + E: std::error::Error + Send + Sync + 'static + From, +{ + /// Start this `Subsystem` and return `SpawnedSubsystem`. + fn start(self, ctx: Ctx) -> SpawnedSubsystem < E >; +} + + +/// TODO FIXME +#[async_trait::async_trait] +pub trait SubsystemSender: Send + Clone + 'static { + /// Send a direct message to some other `Subsystem`, routed based on message type. + async fn send_message(&mut self, msg: Message); + + /// Send multiple direct messages to other `Subsystem`s, routed based on message type. + async fn send_messages(&mut self, msgs: T) + where T: IntoIterator + Send, T::IntoIter: Send; + + /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message + /// type. + /// + /// This function should be used only when there is some other bounding factor on the messages + /// sent with it. Otherwise, it risks a memory leak. + fn send_unbounded_message(&mut self, msg: Message); +} + #[cfg(test)] mod tests; diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 4a73922d74c6..71c26f386c8c 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -22,7 +22,7 @@ //! check out that guide, documentation in this crate will be mostly discussing //! technical stuff. //! -//! An `Overseer` is something that allows spawning/stopping and overseing +//! An `Overseer` is something that allows spawning/stopping and overseeing //! asynchronous tasks as well as establishing a well-defined and easy to use //! protocol that the tasks can use to communicate with each other. It is desired //! that this protocol is the only way tasks communicate with each other, however @@ -89,16 +89,22 @@ use polkadot_subsystem::messages::{ ApprovalVotingMessage, GossipSupportMessage, }; pub use polkadot_subsystem::{ - OverseerSignal, SubsystemError, - SubsystemResult, SpawnedSubsystem, ActiveLeavesUpdate, ActivatedLeaf, DummySubsystem, jaeger, + OverseerSignal, + SubsystemResult, ActiveLeavesUpdate, ActivatedLeaf, DummySubsystem, jaeger, }; +/// TODO legacy, to be deleted, left for easier integration +mod subsystems; +use self::subsystems::{AllSubsystems, AllSubsystemsSame}; + mod metrics; use self::metrics::Metrics; use polkadot_node_subsystem_util::{TimeoutExt, metrics::{prometheus, Metrics as MetricsTrait}, metered, Metronome}; -use polkadot_node_primitives::SpawnNamed; use polkadot_overseer_gen::{ + SpawnNamed, + SpawnedSubsystem, + SubsystemError, SubsystemMeterReadouts, SubsystemMeters, SubsystemIncomingMessages, @@ -140,209 +146,60 @@ impl HeadSupportsParachains for Arc where self.runtime_api().has_api::>(&id).unwrap_or(false) } } - -/// This struct is passed as an argument to create a new instance of an [`Overseer`]. +/// A handler used to communicate with the [`Overseer`]. /// -/// As any entity that satisfies the interface may act as a [`Subsystem`] this allows -/// mocking in the test code: -/// -/// Each [`Subsystem`] is supposed to implement some interface that is generic over -/// message type that is specific to this [`Subsystem`]. At the moment not all -/// subsystems are implemented and the rest can be mocked with the [`DummySubsystem`]. -#[derive(Debug, Clone)] -pub struct AllSubsystems< - CV = (), CB = (), CS = (), SD = (), AD = (), AR = (), BS = (), BD = (), P = (), - RA = (), AS = (), NB = (), CA = (), CG = (), CP = (), ApD = (), ApV = (), - GS = (), -> { - /// A candidate validation subsystem. - pub candidate_validation: CV, - /// A candidate backing subsystem. - pub candidate_backing: CB, - /// A candidate selection subsystem. - pub candidate_selection: CS, - /// A statement distribution subsystem. - pub statement_distribution: SD, - /// An availability distribution subsystem. - pub availability_distribution: AD, - /// An availability recovery subsystem. - pub availability_recovery: AR, - /// A bitfield signing subsystem. - pub bitfield_signing: BS, - /// A bitfield distribution subsystem. - pub bitfield_distribution: BD, - /// A provisioner subsystem. - pub provisioner: P, - /// A runtime API subsystem. - pub runtime_api: RA, - /// An availability store subsystem. - pub availability_store: AS, - /// A network bridge subsystem. - pub network_bridge: NB, - /// A Chain API subsystem. - pub chain_api: CA, - /// A Collation Generation subsystem. - pub collation_generation: CG, - /// A Collator Protocol subsystem. - pub collator_protocol: CP, - /// An Approval Distribution subsystem. - pub approval_distribution: ApD, - /// An Approval Voting subsystem. - pub approval_voting: ApV, - /// A Connection Request Issuer subsystem. - pub gossip_support: GS, +/// [`Overseer`]: struct.Overseer.html +#[derive(Clone)] +pub struct OverseerHandler2 { + events_tx: OverseerHandler, } -impl - AllSubsystems -{ - /// Create a new instance of [`AllSubsystems`]. - /// - /// Each subsystem is set to [`DummySystem`]. - /// - ///# Note - /// - /// Because of a bug in rustc it is required that when calling this function, - /// you provide a "random" type for the first generic parameter: - /// - /// ``` - /// polkadot_overseer::AllSubsystems::<()>::dummy(); - /// ``` - pub fn dummy() -> AllSubsystems< - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - DummySubsystem, - > { - AllSubsystems { - candidate_validation: DummySubsystem, - candidate_backing: DummySubsystem, - candidate_selection: DummySubsystem, - statement_distribution: DummySubsystem, - availability_distribution: DummySubsystem, - availability_recovery: DummySubsystem, - bitfield_signing: DummySubsystem, - bitfield_distribution: DummySubsystem, - provisioner: DummySubsystem, - runtime_api: DummySubsystem, - availability_store: DummySubsystem, - network_bridge: DummySubsystem, - chain_api: DummySubsystem, - collation_generation: DummySubsystem, - collator_protocol: DummySubsystem, - approval_distribution: DummySubsystem, - approval_voting: DummySubsystem, - gossip_support: DummySubsystem, - } - } - - fn as_ref(&self) -> AllSubsystems<&'_ CV, &'_ CB, &'_ CS, &'_ SD, &'_ AD, &'_ AR, &'_ BS, &'_ BD, &'_ P, &'_ RA, &'_ AS, &'_ NB, &'_ CA, &'_ CG, &'_ CP, &'_ ApD, &'_ ApV, &'_ GS> { - AllSubsystems { - candidate_validation: &self.candidate_validation, - candidate_backing: &self.candidate_backing, - candidate_selection: &self.candidate_selection, - statement_distribution: &self.statement_distribution, - availability_distribution: &self.availability_distribution, - availability_recovery: &self.availability_recovery, - bitfield_signing: &self.bitfield_signing, - bitfield_distribution: &self.bitfield_distribution, - provisioner: &self.provisioner, - runtime_api: &self.runtime_api, - availability_store: &self.availability_store, - network_bridge: &self.network_bridge, - chain_api: &self.chain_api, - collation_generation: &self.collation_generation, - collator_protocol: &self.collator_protocol, - approval_distribution: &self.approval_distribution, - approval_voting: &self.approval_voting, - gossip_support: &self.gossip_support, - } - } - - fn map_subsystems(self, m: M) - -> AllSubsystems< - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - > - where - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem

, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - { - AllSubsystems { - candidate_validation: m.map_subsystem(self.candidate_validation), - candidate_backing: m.map_subsystem(self.candidate_backing), - candidate_selection: m.map_subsystem(self.candidate_selection), - statement_distribution: m.map_subsystem(self.statement_distribution), - availability_distribution: m.map_subsystem(self.availability_distribution), - availability_recovery: m.map_subsystem(self.availability_recovery), - bitfield_signing: m.map_subsystem(self.bitfield_signing), - bitfield_distribution: m.map_subsystem(self.bitfield_distribution), - provisioner: m.map_subsystem(self.provisioner), - runtime_api: m.map_subsystem(self.runtime_api), - availability_store: m.map_subsystem(self.availability_store), - network_bridge: m.map_subsystem(self.network_bridge), - chain_api: m.map_subsystem(self.chain_api), - collation_generation: m.map_subsystem(self.collation_generation), - collator_protocol: m.map_subsystem(self.collator_protocol), - approval_distribution: m.map_subsystem(self.approval_distribution), - approval_voting: m.map_subsystem(self.approval_voting), - gossip_support: m.map_subsystem(self.gossip_support), - } - } +impl OverseerHandler2 { + /// Inform the `Overseer` that that some block was imported. + #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] + pub async fn block_imported(&mut self, block: BlockInfo) { + self.send_and_log_error(Event::BlockImported(block)).await + } + + /// Send some message to one of the `Subsystem`s. + #[tracing::instrument(level = "trace", skip(self, msg), fields(subsystem = LOG_TARGET))] + pub async fn send_msg(&mut self, msg: impl Into) { + self.send_and_log_error(Event::MsgToSubsystem(msg.into())).await + } + + /// Inform the `Overseer` that some block was finalized. + #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] + pub async fn block_finalized(&mut self, block: BlockInfo) { + self.send_and_log_error(Event::BlockFinalized(block)).await + } + + /// Wait for a block with the given hash to be in the active-leaves set. + /// + /// The response channel responds if the hash was activated and is closed if the hash was deactivated. + /// Note that due the fact the overseer doesn't store the whole active-leaves set, only deltas, + /// the response channel may never return if the hash was deactivated before this call. + /// In this case, it's the caller's responsibility to ensure a timeout is set. + #[tracing::instrument(level = "trace", skip(self, response_channel), fields(subsystem = LOG_TARGET))] + pub async fn wait_for_activation(&mut self, hash: Hash, response_channel: oneshot::Sender>) { + self.send_and_log_error(Event::ExternalRequest(ExternalRequest::WaitForActivation { + hash, + response_channel + })).await + } + + /// Tell `Overseer` to shutdown. + #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] + pub async fn stop(&mut self) { + self.send_and_log_error(Event::Stop).await + } + + async fn send_and_log_error(&mut self, event: Event) { + if self.events_tx.send(event).await.is_err() { + tracing::info!(target: LOG_TARGET, "Failed to send an event to Overseer"); + } + } } -type AllSubsystemsSame = AllSubsystems< - T, T, T, T, T, - T, T, T, T, T, - T, T, T, T, T, - T, T, T, ->; /// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. /// @@ -429,7 +286,7 @@ enum ExternalRequest { /// [`OverseerHandler`]: struct.OverseerHandler.html pub async fn forward_events>( client: Arc

, - mut handler: OverseerHandler, + mut handler: OverseerHandler2, ) { let mut finality = client.finality_notification_stream(); let mut imports = client.import_notification_stream(); @@ -483,11 +340,11 @@ pub struct OverseerSubsystemContext{ metrics: Metrics, } -impl OverseerSubsystemContext { +impl OverseerSubsystemContext { /// Create a new `OverseerSubsystemContext`. fn new( signals: metered::MeteredReceiver, - messages: SubsystemIncomingMessages, + messages: SubsystemIncomingMessages, to_subsystems: ChannelsOut, to_overseer: metered::UnboundedMeteredSender, metrics: Metrics, @@ -513,7 +370,7 @@ impl OverseerSubsystemContext { #[allow(unused)] fn new_unmetered( signals: metered::MeteredReceiver, - messages: SubsystemIncomingMessages, + messages: SubsystemIncomingMessages, to_subsystems: ChannelsOut, to_overseer: metered::UnboundedMeteredSender, ) -> Self { @@ -548,7 +405,7 @@ impl SubsystemContext for OverseerSub // wait for next signal. let signal = self.signals.next().await .ok_or(SubsystemError::Context( - "No more messages in rx queue to process" + "Signal channel is terminated and empty." .to_owned() ))?; @@ -567,7 +424,7 @@ impl SubsystemContext for OverseerSub signal = await_signal => { let signal = signal .ok_or(SubsystemError::Context( - "No more messages in rx queue to process" + "Signal channel is terminated and empty." .to_owned() ))?; @@ -576,7 +433,7 @@ impl SubsystemContext for OverseerSub msg = await_message => { let packet = msg .ok_or(SubsystemError::Context( - "No more messages in rx queue to process" + "Message channel is terminated and empty." .to_owned() ))?; @@ -622,35 +479,34 @@ impl SubsystemContext for OverseerSub } } - /// The `Overseer` itself. -#[overlord(gen=AllMessages, event=Event, signal=OverseerSignal, error=SubsystemResult)] +#[overlord(gen=AllMessages, event=Event, signal=OverseerSignal, error=SubsystemError)] pub struct Overseer { - #[subsystem(no_dispatch, blocking, CandidateValidationMessage)] + #[subsystem(no_dispatch, CandidateValidationMessage)] candidate_validation_message: CandidateValidation, - #[subsystem(no_dispatch, blocking, CandidateBackingMessage)] + #[subsystem(no_dispatch, CandidateBackingMessage)] candidate_backing_message: CandidateBacking, - #[subsystem(no_dispatch, blocking, CandidateSelectionMessage)] + #[subsystem(no_dispatch, CandidateSelectionMessage)] candidate_selection_message: CandidateSelection, - #[subsystem(no_dispatch, blocking, StatementDistributionMessage)] + #[subsystem(StatementDistributionMessage)] statement_distribution_message: StatementDistribution, - #[subsystem(no_dispatch, blocking, AvailabilityDistributionMessage)] + #[subsystem(no_dispatch, AvailabilityDistributionMessage)] availability_distribution_message: AvailabilityDistribution, - #[subsystem(no_dispatch, blocking, AvailabilityRecoveryMessage)] + #[subsystem(no_dispatch, AvailabilityRecoveryMessage)] availability_recovery_message: AvailabilityRecovery, - #[subsystem(no_dispatch, blocking, BitfieldSigningMessage)] + #[subsystem(blocking, BitfieldSigningMessage)] bitfield_signing_message: BitfieldSigning, - #[subsystem(no_dispatch, blocking, BitfieldDistributionMessage)] + #[subsystem(no_dispatch, BitfieldDistributionMessage)] bitfield_distribution_message: BitfieldDistribution, - #[subsystem(no_dispatch, blocking, ProvisionerMessage)] + #[subsystem(no_dispatch, ProvisionerMessage)] provisioner_message: Provisioner, #[subsystem(no_dispatch, blocking, RuntimeApiMessage)] @@ -659,25 +515,25 @@ pub struct Overseer { #[subsystem(no_dispatch, blocking, AvailabilityStoreMessage)] availability_store_message: AvailabilityStore, - #[subsystem(no_dispatch, blocking, NetworkBridgeMessage)] + #[subsystem(no_dispatch, NetworkBridgeMessage)] network_bridge_message: NetworkBridge, #[subsystem(no_dispatch, blocking, ChainApiMessage)] chain_api_message: ChainApi, - #[subsystem(no_dispatch, blocking, CollationGenerationMessage)] + #[subsystem(no_dispatch, CollationGenerationMessage)] collation_generation_message: CollationGeneration, - #[subsystem(no_dispatch, blocking, CollatorProtocolMessage)] + #[subsystem(no_dispatch, CollatorProtocolMessage)] collator_protocol_message: CollatorProtocol, - #[subsystem(no_dispatch, blocking, ApprovalDistributionMessage)] + #[subsystem(ApprovalDistributionMessage)] approval_distribution_message: ApprovalDistribution, - #[subsystem(no_dispatch, blocking, ApprovalVotingMessage)] + #[subsystem(no_dispatch, ApprovalVotingMessage)] approval_voting_message: ApprovalVoting, - #[subsystem(no_dispatch, blocking, GossipSupportMessage)] + #[subsystem(no_dispatch, GossipSupportMessage)] gossip_support_message: GossipSupport, /// External listeners waiting for a hash to be in the active-leave set. @@ -880,7 +736,7 @@ where .approval_distribution(all_subsystems.approval_distribution) .approval_voting(all_subsystems.approval_voting) .gossip_support(all_subsystems.gossip_support) - .leaves(leaves.collect()) + .leaves(Vec::from_iter(leaves)) .active_leaves(Default::default()) .span_per_active_leaf(Default::default()) .activation_external_listeners(Default::default()) @@ -892,8 +748,8 @@ where } // Stop the overseer. - async fn stop2(mut self) { - self.stop(Overseer::Conclude).await + async fn stop(mut self) { + self.wait_for_terminate(OverseerSignal::Conclude).await } /// Run the `Overseer`. @@ -1109,1012 +965,33 @@ where } } -enum TaskKind { - Regular, - Blocking, -} - -#[cfg(test)] -mod tests { - use std::sync::atomic; - use std::collections::HashMap; - use futures::{executor, pin_mut, select, FutureExt, pending}; - - use polkadot_primitives::v1::{CollatorPair, CandidateHash}; - use polkadot_subsystem::{messages::RuntimeApiRequest, messages::NetworkBridgeEvent, jaeger}; - use polkadot_node_primitives::{CollationResult, CollationGenerationConfig, PoV, BlockData}; - use polkadot_node_network_protocol::{PeerId, UnifiedReputationChange}; - use polkadot_node_subsystem_util::metered; - - use sp_core::crypto::Pair as _; - use assert_matches::assert_matches; - - use super::*; - - struct TestSubsystem1(metered::MeteredSender); - - impl Subsystem for TestSubsystem1 - where C: SubsystemContext - { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let mut sender = self.0; - SpawnedSubsystem { - name: "test-subsystem-1", - future: Box::pin(async move { - let mut i = 0; - loop { - match ctx.recv().await { - Ok(FromOverseer::Communication { .. }) => { - let _ = sender.send(i).await; - i += 1; - continue; - } - Ok(FromOverseer::Signal(OverseerSignal::Conclude)) => return Ok(()), - Err(_) => return Ok(()), - _ => (), - } - } - }), - } - } - } - - struct TestSubsystem2(metered::MeteredSender); - - impl Subsystem for TestSubsystem2 - where C: SubsystemContext - { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let sender = self.0.clone(); - SpawnedSubsystem { - name: "test-subsystem-2", - future: Box::pin(async move { - let _sender = sender; - let mut c: usize = 0; - loop { - if c < 10 { - let (tx, _) = oneshot::channel(); - ctx.send_message( - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - Default::default(), - PoV { - block_data: BlockData(Vec::new()), - }.into(), - tx, - ) - ) - ).await; - c += 1; - continue; - } - match ctx.try_recv().await { - Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => { - break; - } - Ok(Some(_)) => { - continue; - } - Err(_) => return Ok(()), - _ => (), - } - pending!(); - } - - Ok(()) - }), - } - } - } - - struct ReturnOnStart; - - impl Subsystem for ReturnOnStart - where C: SubsystemContext - { - fn start(self, mut _ctx: C) -> SpawnedSubsystem { - SpawnedSubsystem { - name: "test-subsystem-4", - future: Box::pin(async move { - // Do nothing and exit. - Ok(()) - }), - } - } - } - - struct MockSupportsParachains; - - impl HeadSupportsParachains for MockSupportsParachains { - fn head_supports_parachains(&self, _head: &Hash) -> bool { - true - } - } - - // Checks that a minimal configuration of two jobs can run and exchange messages. - #[test] - fn overseer_works() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let (s1_tx, s1_rx) = metered::channel::(64); - let (s2_tx, s2_rx) = metered::channel::(64); - - let mut s1_rx = s1_rx.fuse(); - let mut s2_rx = s2_rx.fuse(); - - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_validation(TestSubsystem1(s1_tx)) - .replace_candidate_backing(TestSubsystem2(s2_tx)); - - let (overseer, mut handler) = Overseer::new( - vec![], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - let overseer_fut = overseer.run().fuse(); - - pin_mut!(overseer_fut); - - let mut s1_results = Vec::new(); - let mut s2_results = Vec::new(); - - loop { - select! { - _ = overseer_fut => break, - s1_next = s1_rx.next() => { - match s1_next { - Some(msg) => { - s1_results.push(msg); - if s1_results.len() == 10 { - handler.stop().await; - } - } - None => break, - } - }, - s2_next = s2_rx.next() => { - match s2_next { - Some(_) => s2_results.push(s2_next), - None => break, - } - }, - complete => break, - } - } - - assert_eq!(s1_results, (0..10).collect::>()); - }); - } - - // Checks activated/deactivated metrics are updated properly. - #[test] - fn overseer_metrics_work() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let first_block_hash = [1; 32].into(); - let second_block_hash = [2; 32].into(); - let third_block_hash = [3; 32].into(); - - let first_block = BlockInfo { - hash: first_block_hash, - parent_hash: [0; 32].into(), - number: 1, - }; - let second_block = BlockInfo { - hash: second_block_hash, - parent_hash: first_block_hash, - number: 2, - }; - let third_block = BlockInfo { - hash: third_block_hash, - parent_hash: second_block_hash, - number: 3, - }; - - let all_subsystems = AllSubsystems::<()>::dummy(); - let registry = prometheus::Registry::new(); - let (overseer, mut handler) = Overseer::new( - vec![first_block], - all_subsystems, - Some(®istry), - MockSupportsParachains, - spawner, - ).unwrap(); - let overseer_fut = overseer.run().fuse(); - - pin_mut!(overseer_fut); - - handler.block_imported(second_block).await; - handler.block_imported(third_block).await; - handler.send_msg(AllMessages::CandidateValidation(test_candidate_validation_msg())).await; - handler.stop().await; - - select! { - res = overseer_fut => { - assert!(res.is_ok()); - let metrics = extract_metrics(®istry); - assert_eq!(metrics["activated"], 3); - assert_eq!(metrics["deactivated"], 2); - assert_eq!(metrics["relayed"], 1); - }, - complete => (), - } - }); - } - - fn extract_metrics(registry: &prometheus::Registry) -> HashMap<&'static str, u64> { - let gather = registry.gather(); - assert_eq!(gather[0].get_name(), "parachain_activated_heads_total"); - assert_eq!(gather[1].get_name(), "parachain_deactivated_heads_total"); - assert_eq!(gather[2].get_name(), "parachain_messages_relayed_total"); - let activated = gather[0].get_metric()[0].get_counter().get_value() as u64; - let deactivated = gather[1].get_metric()[0].get_counter().get_value() as u64; - let relayed = gather[2].get_metric()[0].get_counter().get_value() as u64; - let mut result = HashMap::new(); - result.insert("activated", activated); - result.insert("deactivated", deactivated); - result.insert("relayed", relayed); - result - } - - // Spawn a subsystem that immediately exits. - // - // Should immediately conclude the overseer itself. - #[test] - fn overseer_ends_on_subsystem_exit() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_backing(ReturnOnStart); - let (overseer, _handle) = Overseer::new( - vec![], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - - overseer.run().await.unwrap(); - }) - } - - struct TestSubsystem5(metered::MeteredSender); - - impl Subsystem for TestSubsystem5 - where C: SubsystemContext - { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let mut sender = self.0.clone(); - - SpawnedSubsystem { - name: "test-subsystem-5", - future: Box::pin(async move { - loop { - match ctx.try_recv().await { - Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => break, - Ok(Some(FromOverseer::Signal(s))) => { - sender.send(s).await.unwrap(); - continue; - }, - Ok(Some(_)) => continue, - Err(_) => break, - _ => (), - } - pending!(); - } - - Ok(()) - }), - } - } - } - - struct TestSubsystem6(metered::MeteredSender); - - impl Subsystem for TestSubsystem6 - where C: SubsystemContext - { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let mut sender = self.0.clone(); - - SpawnedSubsystem { - name: "test-subsystem-6", - future: Box::pin(async move { - loop { - match ctx.try_recv().await { - Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => break, - Ok(Some(FromOverseer::Signal(s))) => { - sender.send(s).await.unwrap(); - continue; - }, - Ok(Some(_)) => continue, - Err(_) => break, - _ => (), - } - pending!(); - } - - Ok(()) - }), - } - } - } - - // Tests that starting with a defined set of leaves and receiving - // notifications on imported blocks triggers expected `StartWork` and `StopWork` heartbeats. - #[test] - fn overseer_start_stop_works() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let first_block_hash = [1; 32].into(); - let second_block_hash = [2; 32].into(); - let third_block_hash = [3; 32].into(); - - let first_block = BlockInfo { - hash: first_block_hash, - parent_hash: [0; 32].into(), - number: 1, - }; - let second_block = BlockInfo { - hash: second_block_hash, - parent_hash: first_block_hash, - number: 2, - }; - let third_block = BlockInfo { - hash: third_block_hash, - parent_hash: second_block_hash, - number: 3, - }; - - let (tx_5, mut rx_5) = metered::channel(64); - let (tx_6, mut rx_6) = metered::channel(64); - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_validation(TestSubsystem5(tx_5)) - .replace_candidate_backing(TestSubsystem6(tx_6)); - let (overseer, mut handler) = Overseer::new( - vec![first_block], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - - let overseer_fut = overseer.run().fuse(); - pin_mut!(overseer_fut); - - let mut ss5_results = Vec::new(); - let mut ss6_results = Vec::new(); - - handler.block_imported(second_block).await; - handler.block_imported(third_block).await; - - let expected_heartbeats = vec![ - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: first_block_hash, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - })), - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: [ActivatedLeaf { - hash: second_block_hash, - number: 2, - span: Arc::new(jaeger::Span::Disabled), - }].as_ref().into(), - deactivated: [first_block_hash].as_ref().into(), - }), - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: [ActivatedLeaf { - hash: third_block_hash, - number: 3, - span: Arc::new(jaeger::Span::Disabled), - }].as_ref().into(), - deactivated: [second_block_hash].as_ref().into(), - }), - ]; - - loop { - select! { - res = overseer_fut => { - assert!(res.is_ok()); - break; - }, - res = rx_5.next() => { - if let Some(res) = res { - ss5_results.push(res); - } - } - res = rx_6.next() => { - if let Some(res) = res { - ss6_results.push(res); - } - } - complete => break, - } - - if ss5_results.len() == expected_heartbeats.len() && - ss6_results.len() == expected_heartbeats.len() { - handler.stop().await; - } - } - - assert_eq!(ss5_results, expected_heartbeats); - assert_eq!(ss6_results, expected_heartbeats); - }); - } - - // Tests that starting with a defined set of leaves and receiving - // notifications on imported blocks triggers expected `StartWork` and `StopWork` heartbeats. - #[test] - fn overseer_finalize_works() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let first_block_hash = [1; 32].into(); - let second_block_hash = [2; 32].into(); - let third_block_hash = [3; 32].into(); - - let first_block = BlockInfo { - hash: first_block_hash, - parent_hash: [0; 32].into(), - number: 1, - }; - let second_block = BlockInfo { - hash: second_block_hash, - parent_hash: [42; 32].into(), - number: 2, - }; - let third_block = BlockInfo { - hash: third_block_hash, - parent_hash: second_block_hash, - number: 3, - }; - - let (tx_5, mut rx_5) = metered::channel(64); - let (tx_6, mut rx_6) = metered::channel(64); - - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_validation(TestSubsystem5(tx_5)) - .replace_candidate_backing(TestSubsystem6(tx_6)); - - // start with two forks of different height. - let (overseer, mut handler) = Overseer::new( - vec![first_block, second_block], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - - let overseer_fut = overseer.run().fuse(); - pin_mut!(overseer_fut); - - let mut ss5_results = Vec::new(); - let mut ss6_results = Vec::new(); - - // this should stop work on both forks we started with earlier. - handler.block_finalized(third_block).await; - - let expected_heartbeats = vec![ - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: [ - ActivatedLeaf { - hash: first_block_hash, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - }, - ActivatedLeaf { - hash: second_block_hash, - number: 2, - span: Arc::new(jaeger::Span::Disabled), - }, - ].as_ref().into(), - ..Default::default() - }), - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - deactivated: [first_block_hash, second_block_hash].as_ref().into(), - ..Default::default() - }), - OverseerSignal::BlockFinalized(third_block_hash, 3), - ]; - - loop { - select! { - res = overseer_fut => { - assert!(res.is_ok()); - break; - }, - res = rx_5.next() => { - if let Some(res) = res { - ss5_results.push(res); - } - } - res = rx_6.next() => { - if let Some(res) = res { - ss6_results.push(res); - } - } - complete => break, - } - - if ss5_results.len() == expected_heartbeats.len() && ss6_results.len() == expected_heartbeats.len() { - handler.stop().await; - } - } - - assert_eq!(ss5_results.len(), expected_heartbeats.len()); - assert_eq!(ss6_results.len(), expected_heartbeats.len()); - - // Notifications on finality for multiple blocks at once - // may be received in different orders. - for expected in expected_heartbeats { - assert!(ss5_results.contains(&expected)); - assert!(ss6_results.contains(&expected)); - } - }); - } - - #[test] - fn do_not_send_empty_leaves_update_on_block_finalization() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let imported_block = BlockInfo { - hash: Hash::random(), - parent_hash: Hash::random(), - number: 1, - }; - - let finalized_block = BlockInfo { - hash: Hash::random(), - parent_hash: Hash::random(), - number: 1, - }; - - let (tx_5, mut rx_5) = metered::channel(64); - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_backing(TestSubsystem6(tx_5)); - - let (overseer, mut handler) = Overseer::new( - Vec::new(), - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - - let overseer_fut = overseer.run().fuse(); - pin_mut!(overseer_fut); - - let mut ss5_results = Vec::new(); - - handler.block_finalized(finalized_block.clone()).await; - handler.block_imported(imported_block.clone()).await; - - let expected_heartbeats = vec![ - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: [ - ActivatedLeaf { - hash: imported_block.hash, - number: imported_block.number, - span: Arc::new(jaeger::Span::Disabled) - } - ].as_ref().into(), - ..Default::default() - }), - OverseerSignal::BlockFinalized(finalized_block.hash, 1), - ]; - - loop { - select! { - res = overseer_fut => { - assert!(res.is_ok()); - break; - }, - res = rx_5.next() => { - if let Some(res) = dbg!(res) { - ss5_results.push(res); - } - } - } - - if ss5_results.len() == expected_heartbeats.len() { - handler.stop().await; - } - } - - assert_eq!(ss5_results.len(), expected_heartbeats.len()); - - for expected in expected_heartbeats { - assert!(ss5_results.contains(&expected)); - } - }); - } - - #[derive(Clone)] - struct CounterSubsystem { - stop_signals_received: Arc, - signals_received: Arc, - msgs_received: Arc, - } +#[derive(Debug, Clone)] +pub struct OverseerSubsystemSender { + channels: ChannelsOut, + signals_received: SignalsReceived, +} - impl CounterSubsystem { - fn new( - stop_signals_received: Arc, - signals_received: Arc, - msgs_received: Arc, - ) -> Self { - Self { - stop_signals_received, - signals_received, - msgs_received, - } - } +#[::polkadot_overseer_gen::async_trait] +impl SubsystemSender for OverseerSubsystemSender { + async fn send_message(&mut self, msg: AllMessages) { + self.channels.send_and_log_error(self.signals_received.load(), msg).await; } - impl Subsystem for CounterSubsystem - where - C: SubsystemContext, - M: Send, + async fn send_messages(&mut self, msgs: AllMessages) + where T: IntoIterator + Send, T::IntoIter: Send { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - SpawnedSubsystem { - name: "counter-subsystem", - future: Box::pin(async move { - loop { - match ctx.try_recv().await { - Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => { - self.stop_signals_received.fetch_add(1, atomic::Ordering::SeqCst); - break; - }, - Ok(Some(FromOverseer::Signal(_))) => { - self.signals_received.fetch_add(1, atomic::Ordering::SeqCst); - continue; - }, - Ok(Some(FromOverseer::Communication { .. })) => { - self.msgs_received.fetch_add(1, atomic::Ordering::SeqCst); - continue; - }, - Err(_) => (), - _ => (), - } - pending!(); - } - - Ok(()) - }), - } + // This can definitely be optimized if necessary. + for msg in msgs { + self.send_message(msg).await; } } - fn test_candidate_validation_msg() -> CandidateValidationMessage { - let (sender, _) = oneshot::channel(); - let pov = Arc::new(PoV { block_data: BlockData(Vec::new()) }); - CandidateValidationMessage::ValidateFromChainState(Default::default(), pov, sender) + fn send_unbounded_message(&mut self, msg: AllMessages) { + self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); } +} - fn test_candidate_backing_msg() -> CandidateBackingMessage { - let (sender, _) = oneshot::channel(); - CandidateBackingMessage::GetBackedCandidates(Default::default(), Vec::new(), sender) - } - - fn test_candidate_selection_msg() -> CandidateSelectionMessage { - CandidateSelectionMessage::default() - } - - fn test_chain_api_msg() -> ChainApiMessage { - let (sender, _) = oneshot::channel(); - ChainApiMessage::FinalizedBlockNumber(sender) - } - - fn test_collator_generation_msg() -> CollationGenerationMessage { - CollationGenerationMessage::Initialize(CollationGenerationConfig { - key: CollatorPair::generate().0, - collator: Box::new(|_, _| TestCollator.boxed()), - para_id: Default::default(), - }) - } - struct TestCollator; - - impl Future for TestCollator { - type Output = Option; - - fn poll(self: Pin<&mut Self>, _cx: &mut futures::task::Context) -> Poll { - panic!("at the Disco") - } - } - - impl Unpin for TestCollator {} - - fn test_collator_protocol_msg() -> CollatorProtocolMessage { - CollatorProtocolMessage::CollateOn(Default::default()) - } - - fn test_network_bridge_event() -> NetworkBridgeEvent { - NetworkBridgeEvent::PeerDisconnected(PeerId::random()) - } - - fn test_statement_distribution_msg() -> StatementDistributionMessage { - StatementDistributionMessage::NetworkBridgeUpdateV1(test_network_bridge_event()) - } - - fn test_availability_recovery_msg() -> AvailabilityRecoveryMessage { - let (sender, _) = oneshot::channel(); - AvailabilityRecoveryMessage::RecoverAvailableData( - Default::default(), - Default::default(), - None, - sender, - ) - } - - fn test_bitfield_distribution_msg() -> BitfieldDistributionMessage { - BitfieldDistributionMessage::NetworkBridgeUpdateV1(test_network_bridge_event()) - } - - fn test_provisioner_msg() -> ProvisionerMessage { - let (sender, _) = oneshot::channel(); - ProvisionerMessage::RequestInherentData(Default::default(), sender) - } - - fn test_runtime_api_msg() -> RuntimeApiMessage { - let (sender, _) = oneshot::channel(); - RuntimeApiMessage::Request(Default::default(), RuntimeApiRequest::Validators(sender)) - } - - fn test_availability_store_msg() -> AvailabilityStoreMessage { - let (sender, _) = oneshot::channel(); - AvailabilityStoreMessage::QueryAvailableData(CandidateHash(Default::default()), sender) - } - - fn test_network_bridge_msg() -> NetworkBridgeMessage { - NetworkBridgeMessage::ReportPeer(PeerId::random(), UnifiedReputationChange::BenefitMinor("")) - } - - fn test_approval_distribution_msg() -> ApprovalDistributionMessage { - ApprovalDistributionMessage::NewBlocks(Default::default()) - } - - fn test_approval_voting_msg() -> ApprovalVotingMessage { - let (sender, _) = oneshot::channel(); - ApprovalVotingMessage::ApprovedAncestor(Default::default(), 0, sender) - } - - // Checks that `stop`, `broadcast_signal` and `broadcast_message` are implemented correctly. - #[test] - fn overseer_all_subsystems_receive_signals_and_messages() { - const NUM_SUBSYSTEMS: usize = 18; - // -3 for BitfieldSigning, GossipSupport and AvailabilityDistribution - const NUM_SUBSYSTEMS_MESSAGED: usize = NUM_SUBSYSTEMS - 3; - - let spawner = sp_core::testing::TaskExecutor::new(); - executor::block_on(async move { - let stop_signals_received = Arc::new(atomic::AtomicUsize::new(0)); - let signals_received = Arc::new(atomic::AtomicUsize::new(0)); - let msgs_received = Arc::new(atomic::AtomicUsize::new(0)); - - let subsystem = CounterSubsystem::new( - stop_signals_received.clone(), - signals_received.clone(), - msgs_received.clone(), - ); - - let all_subsystems = AllSubsystems { - candidate_validation: subsystem.clone(), - candidate_backing: subsystem.clone(), - candidate_selection: subsystem.clone(), - collation_generation: subsystem.clone(), - collator_protocol: subsystem.clone(), - statement_distribution: subsystem.clone(), - availability_distribution: subsystem.clone(), - availability_recovery: subsystem.clone(), - bitfield_signing: subsystem.clone(), - bitfield_distribution: subsystem.clone(), - provisioner: subsystem.clone(), - runtime_api: subsystem.clone(), - availability_store: subsystem.clone(), - network_bridge: subsystem.clone(), - chain_api: subsystem.clone(), - approval_distribution: subsystem.clone(), - approval_voting: subsystem.clone(), - gossip_support: subsystem.clone(), - }; - let (overseer, mut handler) = Overseer::new( - vec![], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - let overseer_fut = overseer.run().fuse(); - - pin_mut!(overseer_fut); - - // send a signal to each subsystem - handler.block_imported(BlockInfo { - hash: Default::default(), - parent_hash: Default::default(), - number: Default::default(), - }).await; - - // send a msg to each subsystem - // except for BitfieldSigning and GossipSupport as the messages are not instantiable - handler.send_msg(AllMessages::CandidateValidation(test_candidate_validation_msg())).await; - handler.send_msg(AllMessages::CandidateBacking(test_candidate_backing_msg())).await; - handler.send_msg(AllMessages::CandidateSelection(test_candidate_selection_msg())).await; - handler.send_msg(AllMessages::CollationGeneration(test_collator_generation_msg())).await; - handler.send_msg(AllMessages::CollatorProtocol(test_collator_protocol_msg())).await; - handler.send_msg(AllMessages::StatementDistribution(test_statement_distribution_msg())).await; - handler.send_msg(AllMessages::AvailabilityRecovery(test_availability_recovery_msg())).await; - // handler.send_msg(AllMessages::BitfieldSigning(test_bitfield_signing_msg())).await; - // handler.send_msg(AllMessages::GossipSupport(test_bitfield_signing_msg())).await; - handler.send_msg(AllMessages::BitfieldDistribution(test_bitfield_distribution_msg())).await; - handler.send_msg(AllMessages::Provisioner(test_provisioner_msg())).await; - handler.send_msg(AllMessages::RuntimeApi(test_runtime_api_msg())).await; - handler.send_msg(AllMessages::AvailabilityStore(test_availability_store_msg())).await; - handler.send_msg(AllMessages::NetworkBridge(test_network_bridge_msg())).await; - handler.send_msg(AllMessages::ChainApi(test_chain_api_msg())).await; - handler.send_msg(AllMessages::ApprovalDistribution(test_approval_distribution_msg())).await; - handler.send_msg(AllMessages::ApprovalVoting(test_approval_voting_msg())).await; - - // Wait until all subsystems have received. Otherwise the messages might race against - // the conclude signal. - loop { - match (&mut overseer_fut).timeout(Duration::from_millis(100)).await { - None => { - let r = msgs_received.load(atomic::Ordering::SeqCst); - if r < NUM_SUBSYSTEMS_MESSAGED { - Delay::new(Duration::from_millis(100)).await; - } else if r > NUM_SUBSYSTEMS_MESSAGED { - panic!("too many messages received??"); - } else { - break - } - } - Some(_) => panic!("exited too early"), - } - } - - // send a stop signal to each subsystems - handler.stop().await; - - let res = overseer_fut.await; - assert_eq!(stop_signals_received.load(atomic::Ordering::SeqCst), NUM_SUBSYSTEMS); - assert_eq!(signals_received.load(atomic::Ordering::SeqCst), NUM_SUBSYSTEMS); - assert_eq!(msgs_received.load(atomic::Ordering::SeqCst), NUM_SUBSYSTEMS_MESSAGED); - - assert!(res.is_ok()); - }); - } - - #[test] - fn context_holds_onto_message_until_enough_signals_received() { - let (candidate_validation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (candidate_backing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (candidate_selection_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (statement_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (availability_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (availability_recovery_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (bitfield_signing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (bitfield_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (provisioner_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (runtime_api_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (availability_store_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (network_bridge_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (chain_api_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (collator_protocol_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (collation_generation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (approval_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (approval_voting_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (gossip_support_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - - let (candidate_validation_unbounded_tx, _) = metered::unbounded(); - let (candidate_backing_unbounded_tx, _) = metered::unbounded(); - let (candidate_selection_unbounded_tx, _) = metered::unbounded(); - let (statement_distribution_unbounded_tx, _) = metered::unbounded(); - let (availability_distribution_unbounded_tx, _) = metered::unbounded(); - let (availability_recovery_unbounded_tx, _) = metered::unbounded(); - let (bitfield_signing_unbounded_tx, _) = metered::unbounded(); - let (bitfield_distribution_unbounded_tx, _) = metered::unbounded(); - let (provisioner_unbounded_tx, _) = metered::unbounded(); - let (runtime_api_unbounded_tx, _) = metered::unbounded(); - let (availability_store_unbounded_tx, _) = metered::unbounded(); - let (network_bridge_unbounded_tx, _) = metered::unbounded(); - let (chain_api_unbounded_tx, _) = metered::unbounded(); - let (collator_protocol_unbounded_tx, _) = metered::unbounded(); - let (collation_generation_unbounded_tx, _) = metered::unbounded(); - let (approval_distribution_unbounded_tx, _) = metered::unbounded(); - let (approval_voting_unbounded_tx, _) = metered::unbounded(); - let (gossip_support_unbounded_tx, _) = metered::unbounded(); - - let channels_out = ChannelsOut { - candidate_validation: candidate_validation_bounded_tx.clone(), - candidate_backing: candidate_backing_bounded_tx.clone(), - candidate_selection: candidate_selection_bounded_tx.clone(), - statement_distribution: statement_distribution_bounded_tx.clone(), - availability_distribution: availability_distribution_bounded_tx.clone(), - availability_recovery: availability_recovery_bounded_tx.clone(), - bitfield_signing: bitfield_signing_bounded_tx.clone(), - bitfield_distribution: bitfield_distribution_bounded_tx.clone(), - provisioner: provisioner_bounded_tx.clone(), - runtime_api: runtime_api_bounded_tx.clone(), - availability_store: availability_store_bounded_tx.clone(), - network_bridge: network_bridge_bounded_tx.clone(), - chain_api: chain_api_bounded_tx.clone(), - collator_protocol: collator_protocol_bounded_tx.clone(), - collation_generation: collation_generation_bounded_tx.clone(), - approval_distribution: approval_distribution_bounded_tx.clone(), - approval_voting: approval_voting_bounded_tx.clone(), - gossip_support: gossip_support_bounded_tx.clone(), - - candidate_validation_unbounded: candidate_validation_unbounded_tx.clone(), - candidate_backing_unbounded: candidate_backing_unbounded_tx.clone(), - candidate_selection_unbounded: candidate_selection_unbounded_tx.clone(), - statement_distribution_unbounded: statement_distribution_unbounded_tx.clone(), - availability_distribution_unbounded: availability_distribution_unbounded_tx.clone(), - availability_recovery_unbounded: availability_recovery_unbounded_tx.clone(), - bitfield_signing_unbounded: bitfield_signing_unbounded_tx.clone(), - bitfield_distribution_unbounded: bitfield_distribution_unbounded_tx.clone(), - provisioner_unbounded: provisioner_unbounded_tx.clone(), - runtime_api_unbounded: runtime_api_unbounded_tx.clone(), - availability_store_unbounded: availability_store_unbounded_tx.clone(), - network_bridge_unbounded: network_bridge_unbounded_tx.clone(), - chain_api_unbounded: chain_api_unbounded_tx.clone(), - collator_protocol_unbounded: collator_protocol_unbounded_tx.clone(), - collation_generation_unbounded: collation_generation_unbounded_tx.clone(), - approval_distribution_unbounded: approval_distribution_unbounded_tx.clone(), - approval_voting_unbounded: approval_voting_unbounded_tx.clone(), - gossip_support_unbounded: gossip_support_unbounded_tx.clone(), - }; - - let (mut signal_tx, signal_rx) = metered::channel(CHANNEL_CAPACITY); - let (mut bounded_tx, bounded_rx) = metered::channel(CHANNEL_CAPACITY); - let (unbounded_tx, unbounded_rx) = metered::unbounded(); - let (to_overseer_tx, _to_overseer_rx) = metered::unbounded(); - - let mut ctx = OverseerSubsystemContext::<()>::new_unmetered( - signal_rx, - stream::select(bounded_rx, unbounded_rx), - channels_out, - to_overseer_tx, - ); - - assert_eq!(ctx.signals_received.load(), 0); - - let test_fut = async move { - signal_tx.send(OverseerSignal::Conclude).await.unwrap(); - assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Signal(OverseerSignal::Conclude)); - - assert_eq!(ctx.signals_received.load(), 1); - bounded_tx.send(MessagePacket { - signals_received: 2, - message: (), - }).await.unwrap(); - unbounded_tx.unbounded_send(MessagePacket { - signals_received: 2, - message: (), - }).unwrap(); - - match poll!(ctx.recv()) { - Poll::Pending => {} - Poll::Ready(_) => panic!("ready too early"), - }; - - assert!(ctx.pending_incoming.is_some()); - - signal_tx.send(OverseerSignal::Conclude).await.unwrap(); - assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Signal(OverseerSignal::Conclude)); - assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Communication { msg: () }); - assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Communication { msg: () }); - assert!(ctx.pending_incoming.is_none()); - }; - futures::executor::block_on(test_fut); - } -} +#[cfg(test)] +mod tests; diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs new file mode 100644 index 000000000000..86d0ca8054b0 --- /dev/null +++ b/node/overseer/src/subsystems.rs @@ -0,0 +1,241 @@ + +use ::polkadot_overseer_gen::MapSubsystem; +use crate::SubsystemContext; +use crate::Subsystem; + +/// A dummy subsystem that implements [`Subsystem`] for all +/// types of messages. Used for tests or as a placeholder. +pub struct DummySubsystem; + +impl Subsystem for DummySubsystem +where + C::Message: std::fmt::Debug +{ + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let future = Box::pin(async move { + loop { + match ctx.recv().await { + Err(_) => return Ok(()), + Ok(FromOverseer::Signal(OverseerSignal::Conclude)) => return Ok(()), + Ok(overseer_msg) => { + tracing::debug!( + target: "dummy-subsystem", + "Discarding a message sent from overseer {:?}", + overseer_msg + ); + continue; + } + } + } + }); + + SpawnedSubsystem { + name: "dummy-subsystem", + future, + } + } +} + + +/// This struct is passed as an argument to create a new instance of an [`Overseer`]. +/// +/// As any entity that satisfies the interface may act as a [`Subsystem`] this allows +/// mocking in the test code: +/// +/// Each [`Subsystem`] is supposed to implement some interface that is generic over +/// message type that is specific to this [`Subsystem`]. At the moment not all +/// subsystems are implemented and the rest can be mocked with the [`DummySubsystem`]. +#[derive(Debug, Clone)] +pub struct AllSubsystems< + CV = (), CB = (), CS = (), SD = (), AD = (), AR = (), BS = (), BD = (), P = (), + RA = (), AS = (), NB = (), CA = (), CG = (), CP = (), ApD = (), ApV = (), + GS = (), +> { + /// A candidate validation subsystem. + pub candidate_validation: CV, + /// A candidate backing subsystem. + pub candidate_backing: CB, + /// A candidate selection subsystem. + pub candidate_selection: CS, + /// A statement distribution subsystem. + pub statement_distribution: SD, + /// An availability distribution subsystem. + pub availability_distribution: AD, + /// An availability recovery subsystem. + pub availability_recovery: AR, + /// A bitfield signing subsystem. + pub bitfield_signing: BS, + /// A bitfield distribution subsystem. + pub bitfield_distribution: BD, + /// A provisioner subsystem. + pub provisioner: P, + /// A runtime API subsystem. + pub runtime_api: RA, + /// An availability store subsystem. + pub availability_store: AS, + /// A network bridge subsystem. + pub network_bridge: NB, + /// A Chain API subsystem. + pub chain_api: CA, + /// A Collation Generation subsystem. + pub collation_generation: CG, + /// A Collator Protocol subsystem. + pub collator_protocol: CP, + /// An Approval Distribution subsystem. + pub approval_distribution: ApD, + /// An Approval Voting subsystem. + pub approval_voting: ApV, + /// A Connection Request Issuer subsystem. + pub gossip_support: GS, +} + +impl + AllSubsystems +{ + /// Create a new instance of [`AllSubsystems`]. + /// + /// Each subsystem is set to [`DummySystem`]. + /// + ///# Note + /// + /// Because of a bug in rustc it is required that when calling this function, + /// you provide a "random" type for the first generic parameter: + /// + /// ``` + /// polkadot_overseer::AllSubsystems::<()>::dummy(); + /// ``` + pub fn dummy() -> AllSubsystems< + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + DummySubsystem, + > { + AllSubsystems { + candidate_validation: DummySubsystem, + candidate_backing: DummySubsystem, + candidate_selection: DummySubsystem, + statement_distribution: DummySubsystem, + availability_distribution: DummySubsystem, + availability_recovery: DummySubsystem, + bitfield_signing: DummySubsystem, + bitfield_distribution: DummySubsystem, + provisioner: DummySubsystem, + runtime_api: DummySubsystem, + availability_store: DummySubsystem, + network_bridge: DummySubsystem, + chain_api: DummySubsystem, + collation_generation: DummySubsystem, + collator_protocol: DummySubsystem, + approval_distribution: DummySubsystem, + approval_voting: DummySubsystem, + gossip_support: DummySubsystem, + } + } + + fn as_ref(&self) -> AllSubsystems<&'_ CV, &'_ CB, &'_ CS, &'_ SD, &'_ AD, &'_ AR, &'_ BS, &'_ BD, &'_ P, &'_ RA, &'_ AS, &'_ NB, &'_ CA, &'_ CG, &'_ CP, &'_ ApD, &'_ ApV, &'_ GS> { + AllSubsystems { + candidate_validation: &self.candidate_validation, + candidate_backing: &self.candidate_backing, + candidate_selection: &self.candidate_selection, + statement_distribution: &self.statement_distribution, + availability_distribution: &self.availability_distribution, + availability_recovery: &self.availability_recovery, + bitfield_signing: &self.bitfield_signing, + bitfield_distribution: &self.bitfield_distribution, + provisioner: &self.provisioner, + runtime_api: &self.runtime_api, + availability_store: &self.availability_store, + network_bridge: &self.network_bridge, + chain_api: &self.chain_api, + collation_generation: &self.collation_generation, + collator_protocol: &self.collator_protocol, + approval_distribution: &self.approval_distribution, + approval_voting: &self.approval_voting, + gossip_support: &self.gossip_support, + } + } + + fn map_subsystems(self, m: M) + -> AllSubsystems< + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + > + where + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem

, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + M: MapSubsystem, + { + AllSubsystems { + candidate_validation: m.map_subsystem(self.candidate_validation), + candidate_backing: m.map_subsystem(self.candidate_backing), + candidate_selection: m.map_subsystem(self.candidate_selection), + statement_distribution: m.map_subsystem(self.statement_distribution), + availability_distribution: m.map_subsystem(self.availability_distribution), + availability_recovery: m.map_subsystem(self.availability_recovery), + bitfield_signing: m.map_subsystem(self.bitfield_signing), + bitfield_distribution: m.map_subsystem(self.bitfield_distribution), + provisioner: m.map_subsystem(self.provisioner), + runtime_api: m.map_subsystem(self.runtime_api), + availability_store: m.map_subsystem(self.availability_store), + network_bridge: m.map_subsystem(self.network_bridge), + chain_api: m.map_subsystem(self.chain_api), + collation_generation: m.map_subsystem(self.collation_generation), + collator_protocol: m.map_subsystem(self.collator_protocol), + approval_distribution: m.map_subsystem(self.approval_distribution), + approval_voting: m.map_subsystem(self.approval_voting), + gossip_support: m.map_subsystem(self.gossip_support), + } + } +} + +pub type AllSubsystemsSame = AllSubsystems< + T, T, T, T, T, + T, T, T, T, T, + T, T, T, T, T, + T, T, T, +>; diff --git a/node/overseer/src/tests.rs b/node/overseer/src/tests.rs index 0fb65958c078..cc427f2bbf78 100644 --- a/node/overseer/src/tests.rs +++ b/node/overseer/src/tests.rs @@ -46,7 +46,7 @@ struct TestSubsystem2(metered::MeteredSender); impl Subsystem for TestSubsystem2 where C: SubsystemContext { - fn start(self, mut ctx: C) -> SpawnedSubsystem { + fn start(self, mut ctx: C) -> SpawnedSubsystem { let sender = self.0.clone(); SpawnedSubsystem { name: "test-subsystem-2", diff --git a/node/subsystem-test-helpers/src/lib.rs b/node/subsystem-test-helpers/src/lib.rs index 44d2d922888d..e12c91a828e6 100644 --- a/node/subsystem-test-helpers/src/lib.rs +++ b/node/subsystem-test-helpers/src/lib.rs @@ -18,7 +18,6 @@ #![warn(missing_docs)] -use polkadot_node_subsystem::messages::AllMessages; use polkadot_node_subsystem::{ FromOverseer, SubsystemContext, SubsystemError, SubsystemResult, Subsystem, SpawnedSubsystem, OverseerSignal, SubsystemSender, @@ -172,7 +171,7 @@ pub fn sender_receiver() -> (TestSubsystemSender, mpsc::UnboundedReceiver for TestSubsystemSender { async fn send_message(&mut self, msg: AllMessages) { self.tx .send(msg) diff --git a/node/subsystem-util/Cargo.toml b/node/subsystem-util/Cargo.toml index cd84dc98b46f..4436181ff90e 100644 --- a/node/subsystem-util/Cargo.toml +++ b/node/subsystem-util/Cargo.toml @@ -23,6 +23,7 @@ polkadot-node-subsystem = { path = "../subsystem" } polkadot-node-jaeger = { path = "../jaeger" } polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-primitives = { path = "../../primitives" } +polkadot-overseer-gen = { path = "../overseer/overseer-gen" } metered-channel = { path = "../metered-channel"} sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 0a7f85196d23..8da385f67c4b 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -26,10 +26,20 @@ use polkadot_node_subsystem::{ errors::RuntimeApiError, - messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest, RuntimeApiSender, BoundToRelayParent}, - FromOverseer, SpawnedSubsystem, Subsystem, SubsystemContext, SubsystemError, SubsystemSender, + messages::{RuntimeApiMessage, RuntimeApiRequest, RuntimeApiSender, BoundToRelayParent}, + SpawnedSubsystem, SubsystemError, ActiveLeavesUpdate, OverseerSignal, }; + +use polkadot_overseer_gen::{ + AllMessages, + FromOverseer, + SpawnedSubsystem, + Subsystem, + SubsystemContext, + SubsystemSender, +}; + use polkadot_node_jaeger as jaeger; use futures::{channel::{mpsc, oneshot}, prelude::*, select, stream::Stream}; use futures_timer::Delay; @@ -61,8 +71,8 @@ pub use error_handling::{Fault, unwrap_non_fatal}; /// These reexports are required so that external crates can use the `delegated_subsystem` macro properly. pub mod reexports { - pub use sp_core::traits::SpawnNamed; - pub use polkadot_node_subsystem::{ + pub use polkadot_overseer::{ + SpawnNamed, SpawnedSubsystem, Subsystem, SubsystemContext, @@ -147,7 +157,7 @@ macro_rules! specialize_requests { $( $param_name: $param_ty, )* - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , ) -> RuntimeApiReceiver<$return_ty> { request_from_runtime(parent, sender, |tx| RuntimeApiRequest::$request_variant( $( $param_name, )* tx @@ -259,7 +269,7 @@ impl Validator { pub async fn new( parent: Hash, keystore: SyncCryptoStorePtr, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender , ) -> Result { // Note: request_validators and request_session_index_for_child do not and cannot // run concurrently: they both have a mutable handle to the same sender. @@ -394,7 +404,7 @@ pub struct JobSender { from_job: mpsc::Sender, } -impl JobSender { +impl> JobSender { /// Get access to the underlying subsystem sender. pub fn subsystem_sender(&mut self) -> &mut S { &mut self.sender @@ -429,7 +439,7 @@ impl JobSender { } #[async_trait::async_trait] -impl SubsystemSender for JobSender { +impl > SubsystemSender for JobSender { async fn send_message(&mut self, msg: AllMessages) { self.sender.send_message(msg).await } @@ -480,7 +490,7 @@ pub trait JobTrait: Unpin + Sized { /// Run a job for the given relay `parent`. /// /// The job should be ended when `receiver` returns `None`. - fn run( + fn run>( parent: Hash, span: Arc, run_args: Self::RunArgs, @@ -536,7 +546,9 @@ impl Jobs { metrics: Job::Metrics, sender: Sender, ) - where Job: JobTrait, Sender: SubsystemSender, + where + Job: JobTrait, + Sender: SubsystemSender, { let (to_job_tx, to_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY); let (from_job_tx, from_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY); @@ -867,7 +879,7 @@ mod tests { use thiserror::Error; use polkadot_node_jaeger as jaeger; use polkadot_node_subsystem::{ - messages::{AllMessages, CandidateSelectionMessage}, ActiveLeavesUpdate, FromOverseer, OverseerSignal, + messages::{CandidateSelectionMessage}, ActiveLeavesUpdate, SpawnedSubsystem, ActivatedLeaf, }; use assert_matches::assert_matches; diff --git a/node/subsystem-util/src/runtime/mod.rs b/node/subsystem-util/src/runtime/mod.rs index f07db0763027..5dda2e961f23 100644 --- a/node/subsystem-util/src/runtime/mod.rs +++ b/node/subsystem-util/src/runtime/mod.rs @@ -24,7 +24,7 @@ use sp_core::crypto::Public; use sp_keystore::{CryptoStore, SyncCryptoStorePtr}; use polkadot_primitives::v1::{CoreState, EncodeAs, GroupIndex, GroupRotationInfo, Hash, OccupiedCore, SessionIndex, SessionInfo, Signed, SigningContext, UncheckedSigned, ValidatorId, ValidatorIndex}; -use polkadot_node_subsystem::SubsystemContext; +use polkadot_overseer_gen::SubsystemContext; use crate::{ request_session_index_for_child, request_session_info, @@ -225,7 +225,7 @@ pub fn check_signature( session_info: &SessionInfo, relay_parent: Hash, signed: UncheckedSigned, -) -> std::result::Result, UncheckedSigned> +) -> std::result::Result, UncheckedSigned> where Payload: EncodeAs + Clone, RealPayload: Encode + Clone, @@ -243,7 +243,7 @@ where /// Request availability cores from the runtime. pub async fn get_availability_cores(ctx: &mut Context, relay_parent: Hash) - -> Result> + -> Result> where Context: SubsystemContext, { diff --git a/node/subsystem/Cargo.toml b/node/subsystem/Cargo.toml index 590af20343e8..f2d0ebd8778a 100644 --- a/node/subsystem/Cargo.toml +++ b/node/subsystem/Cargo.toml @@ -22,6 +22,7 @@ polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-primitives = { path = "../../primitives" } polkadot-statement-table = { path = "../../statement-table" } polkadot-node-jaeger = { path = "../jaeger" } +polkadot-overseer-gen = { path = "../overseer/overseer-gen" } polkadot-procmacro-subsystem-dispatch-gen = { path = "dispatch-gen" } sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } smallvec = "1.6.1" diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index fafdc65b3181..6509d37428d6 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -38,8 +38,6 @@ pub mod messages; pub use polkadot_node_jaeger as jaeger; pub use jaeger::*; -use self::messages::AllMessages; - /// How many slots are stack-reserved for active leaves updates /// /// If there are fewer than this number of slots, then we've wasted some stack space. @@ -126,24 +124,6 @@ pub enum OverseerSignal { Conclude, } -/// A message type that a subsystem receives from an overseer. -/// It wraps signals from an overseer and messages that are circulating -/// between subsystems. -/// -/// It is generic over over the message type `M` that a particular `Subsystem` may use. -#[derive(Debug)] -pub enum FromOverseer { - /// Signal from the `Overseer`. - Signal(OverseerSignal), - - /// Some other `Subsystem`'s message. - Communication { - /// Contained message - msg: M, - }, -} - - /// An error type that describes faults that may happen /// /// These are: @@ -195,148 +175,7 @@ impl SubsystemError { } } -/// An asynchronous subsystem task.. -/// -/// In essence it's just a newtype wrapping a `BoxFuture`. -pub struct SpawnedSubsystem { - /// Name of the subsystem being spawned. - pub name: &'static str, - /// The task of the subsystem being spawned. - pub future: BoxFuture<'static, SubsystemResult<()>>, -} - /// A `Result` type that wraps [`SubsystemError`]. /// /// [`SubsystemError`]: struct.SubsystemError.html pub type SubsystemResult = Result; - -/// A sender used by subsystems to communicate with other subsystems. -/// -/// Each clone of this type may add more capacity to the bounded buffer, so clones should -/// be used sparingly. -#[async_trait] -pub trait SubsystemSender: Send + Clone + 'static { - /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: AllMessages); - - /// Send multiple direct messages to other `Subsystem`s, routed based on message type. - async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send; - - /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message - /// type. - /// - /// This function should be used only when there is some other bounding factor on the messages - /// sent with it. Otherwise, it risks a memory leak. - fn send_unbounded_message(&mut self, msg: AllMessages); -} - -/// A context type that is given to the [`Subsystem`] upon spawning. -/// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s -/// or spawn jobs. -/// -/// [`Overseer`]: struct.Overseer.html -/// [`SubsystemJob`]: trait.SubsystemJob.html -#[async_trait] -pub trait SubsystemContext: Send + Sized + 'static { - /// The message type of this context. Subsystems launched with this context will expect - /// to receive messages of this type. - type Message: Send; - - /// The message sender type of this context. Clones of the sender should be used sparingly. - type Sender: SubsystemSender; - - /// Try to asynchronously receive a message. - /// - /// This has to be used with caution, if you loop over this without - /// using `pending!()` macro you will end up with a busy loop! - async fn try_recv(&mut self) -> Result>, ()>; - - /// Receive a message. - async fn recv(&mut self) -> SubsystemResult>; - - /// Spawn a child task on the executor. - async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) -> SubsystemResult<()>; - - /// Spawn a blocking child task on the executor's dedicated thread pool. - async fn spawn_blocking( - &mut self, - name: &'static str, - s: Pin + Send>>, - ) -> SubsystemResult<()>; - - /// Get a mutable reference to the sender. - fn sender(&mut self) -> &mut Self::Sender; - - /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: AllMessages) { - self.sender().send_message(msg).await - } - - /// Send multiple direct messages to other `Subsystem`s, routed based on message type. - async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send - { - self.sender().send_messages(msgs).await - } - - - /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message - /// type. - /// - /// This function should be used only when there is some other bounding factor on the messages - /// sent with it. Otherwise, it risks a memory leak. - /// - /// Generally, for this method to be used, these conditions should be met: - /// * There is a communication cycle between subsystems - /// * One of the parts of the cycle has a clear bound on the number of messages produced. - fn send_unbounded_message(&mut self, msg: AllMessages) { - self.sender().send_unbounded_message(msg) - } -} - -/// A trait that describes the [`Subsystem`]s that can run on the [`Overseer`]. -/// -/// It is generic over the message type circulating in the system. -/// The idea that we want some type contaning persistent state that -/// can spawn actually running subsystems when asked to. -/// -/// [`Overseer`]: struct.Overseer.html -/// [`Subsystem`]: trait.Subsystem.html -pub trait Subsystem { - /// Start this `Subsystem` and return `SpawnedSubsystem`. - fn start(self, ctx: C) -> SpawnedSubsystem; -} - -/// A dummy subsystem that implements [`Subsystem`] for all -/// types of messages. Used for tests or as a placeholder. -pub struct DummySubsystem; - -impl Subsystem for DummySubsystem -where - C::Message: std::fmt::Debug -{ - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let future = Box::pin(async move { - loop { - match ctx.recv().await { - Err(_) => return Ok(()), - Ok(FromOverseer::Signal(OverseerSignal::Conclude)) => return Ok(()), - Ok(overseer_msg) => { - tracing::debug!( - target: "dummy-subsystem", - "Discarding a message sent from overseer {:?}", - overseer_msg - ); - continue; - } - } - } - }); - - SpawnedSubsystem { - name: "dummy-subsystem", - future, - } - } -} diff --git a/node/subsystem/src/messages.rs b/node/subsystem/src/messages.rs index a0d274ec6fd4..95830ab5c539 100644 --- a/node/subsystem/src/messages.rs +++ b/node/subsystem/src/messages.rs @@ -673,97 +673,3 @@ pub enum ApprovalDistributionMessage { #[derive(Debug)] pub enum GossipSupportMessage { } - -/// A message type tying together all message types that are used across Subsystems. -#[subsystem_dispatch_gen(NetworkBridgeEvent)] -#[derive(Debug, derive_more::From)] -pub enum AllMessages { - /// Message for the validation subsystem. - #[skip] - CandidateValidation(CandidateValidationMessage), - /// Message for the candidate backing subsystem. - #[skip] - CandidateBacking(CandidateBackingMessage), - /// Message for the candidate selection subsystem. - #[skip] - CandidateSelection(CandidateSelectionMessage), - /// Message for the Chain API subsystem. - #[skip] - ChainApi(ChainApiMessage), - /// Message for the Collator Protocol subsystem. - #[skip] - CollatorProtocol(CollatorProtocolMessage), - /// Message for the statement distribution subsystem. - StatementDistribution(StatementDistributionMessage), - /// Message for the availability distribution subsystem. - #[skip] - AvailabilityDistribution(AvailabilityDistributionMessage), - /// Message for the availability recovery subsystem. - #[skip] - AvailabilityRecovery(AvailabilityRecoveryMessage), - /// Message for the bitfield distribution subsystem. - BitfieldDistribution(BitfieldDistributionMessage), - /// Message for the bitfield signing subsystem. - #[skip] - BitfieldSigning(BitfieldSigningMessage), - /// Message for the Provisioner subsystem. - #[skip] - Provisioner(ProvisionerMessage), - /// Message for the Runtime API subsystem. - #[skip] - RuntimeApi(RuntimeApiMessage), - /// Message for the availability store subsystem. - #[skip] - AvailabilityStore(AvailabilityStoreMessage), - /// Message for the network bridge subsystem. - #[skip] - NetworkBridge(NetworkBridgeMessage), - /// Message for the Collation Generation subsystem. - #[skip] - CollationGeneration(CollationGenerationMessage), - /// Message for the Approval Voting subsystem. - #[skip] - ApprovalVoting(ApprovalVotingMessage), - /// Message for the Approval Distribution subsystem. - ApprovalDistribution(ApprovalDistributionMessage), - /// Message for the Gossip Support subsystem. - #[skip] - GossipSupport(GossipSupportMessage), -} - -impl From> for AvailabilityDistributionMessage { - fn from(req: IncomingRequest) -> Self { - Self::PoVFetchingRequest(req) - } -} -impl From> for AvailabilityDistributionMessage { - fn from(req: IncomingRequest) -> Self { - Self::ChunkFetchingRequest(req) - } -} -impl From> for CollatorProtocolMessage { - fn from(req: IncomingRequest) -> Self { - Self::CollationFetchingRequest(req) - } -} - -impl From> for AllMessages { - fn from(req: IncomingRequest) -> Self { - From::::from(From::from(req)) - } -} -impl From> for AllMessages { - fn from(req: IncomingRequest) -> Self { - From::::from(From::from(req)) - } -} -impl From> for AllMessages { - fn from(req: IncomingRequest) -> Self { - From::::from(From::from(req)) - } -} -impl From> for AllMessages { - fn from(req: IncomingRequest) -> Self { - From::::from(From::from(req)) - } -} From 86b4e08e1047ccb5ae5c9b3b147961162867a7d2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 20 May 2021 18:55:46 +0200 Subject: [PATCH 039/161] more generics, better split --- node/overseer/overseer-gen/Cargo.toml | 2 + node/overseer/overseer-gen/examples/dummy.rs | 14 ++- .../proc-macro/src/impl_channels_out.rs | 20 ++-- .../proc-macro/src/impl_message_wrapper.rs | 7 +- .../proc-macro/src/impl_overseer.rs | 62 +++++++---- node/overseer/overseer-gen/src/lib.rs | 82 ++++++++++++-- node/overseer/src/lib.rs | 6 +- node/subsystem-util/src/lib.rs | 103 ++++++------------ 8 files changed, 182 insertions(+), 114 deletions(-) diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index 4e91e04b4247..a15ff42d7ffa 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -15,6 +15,8 @@ metered = { package = "metered-channel", path = "../../metered-channel" } # trait SpawnNamed sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } futures-timer = "3.0.2" +tokio-util = { version = "0.6.7", features = [ "time" ] } +pin-project = "1.0" [dev-dependencies] trybuild = "1.0.41" diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index e05a83ef9ab3..fa43e28950e2 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -1,6 +1,8 @@ //! A dummy to be used with cargo expand use polkadot_overseer_gen::*; +use polkadot_subsystem::messages::NetworkBridgeEvent; + #[derive(Default)] struct AwesomeSubSys; @@ -12,7 +14,13 @@ struct SigSigSig; #[derive(Debug, Clone)] struct EvX; -#[derive(Debug)] +impl EvX { + pub fn focus<'a, T>(&'a self) -> Result, ()> { + unimplemented!("dispatch") + } +} + +#[derive(Debug, Clone, Copy)] struct Yikes; impl std::fmt::Display for Yikes { @@ -29,10 +37,10 @@ impl From for Yikes { } } -#[derive(Clone)] +#[derive(Debug, Clone)] struct MsgStrukt(u8); -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy)] struct Plinko; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 31032b354bae..fdc542ffae06 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -20,11 +20,17 @@ pub(crate) fn impl_channels_out_struct( #[derive(Debug, Clone)] pub struct ChannelsOut { #( - pub #channel_name: ::polkadot_overseer_gen::metered::MeteredSender>, + pub #channel_name: + ::polkadot_overseer_gen::metered::MeteredSender< + MessagePacket< #consumes > + >, )* #( - pub #channel_name_unbounded: ::polkadot_overseer_gen::metered::UnboundedMeteredSender>, + pub #channel_name_unbounded: + ::polkadot_overseer_gen::metered::UnboundedMeteredSender< + MessagePacket< #consumes > + >, )* } @@ -37,11 +43,11 @@ pub(crate) fn impl_channels_out_struct( ) { let res = match message { #( - #message_wrapper :: #consumes ( message ) => { + #message_wrapper :: #consumes ( inner ) => { self. #channel_name .send( - ::polkadot_overseer_gen::make_packet(signals_received, message) + ::polkadot_overseer_gen::make_packet(signals_received, inner) ).await - }, + } )* }; @@ -61,9 +67,9 @@ pub(crate) fn impl_channels_out_struct( ) { let res = match message { #( - #message_wrapper :: #consumes (message) => { + #message_wrapper :: #consumes (inner) => { self. #channel_name_unbounded .send( - make_packet(signals_received, message) + make_packet(signals_received, inner) ) }, )* diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 2915c698a8ae..a6860a8fb899 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -11,9 +11,8 @@ pub(crate) fn impl_message_wrapper_enum( let message_wrapper = &info.message_wrapper; - let msg = "Generated message type wrapper"; let ts = quote! { - #[doc = #msg] + /// Generated message type wrapper #[derive(Debug, Clone)] enum #message_wrapper { #( @@ -23,8 +22,8 @@ pub(crate) fn impl_message_wrapper_enum( #( impl ::std::convert::From<#consumes> for #message_wrapper { - fn from(src: #consumes) -> Self { - #message_wrapper :: #consumes ( src ) + fn from(inner: #consumes) -> Self { + #message_wrapper :: #consumes ( inner ) } } )* diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 2bf27cb7d68c..50dbba3a4cac 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -55,7 +55,7 @@ pub(crate) fn impl_overseer_struct( pub struct #overseer_name #generics { // Subsystem instances. #( - #subsystem_name: OverseenSubsystem< SubsystemInstance < #consumes > >, + #subsystem_name: OverseenSubsystem< #consumes >, )* // Non-subsystem members. @@ -67,10 +67,14 @@ pub(crate) fn impl_overseer_struct( spawner: S, /// The set of running subsystems. - running_subsystems: ::polkadot_overseer_gen::FuturesUnordered>>, + running_subsystems: ::polkadot_overseer_gen::FuturesUnordered< + BoxFuture<'static, SubsystemResult<()>> + >, /// Gather running subsystems' outbound streams into one. - to_overseer_rx: ::polkadot_overseer_gen::Fuse>, + to_overseer_rx: ::polkadot_overseer_gen::Fuse< + metered::UnboundedMeteredReceiver< ToOverseer > + >, /// Events that are sent to the overseer from the outside world. events_rx: ::polkadot_overseer_gen::metered::MeteredReceiver< #event_ty >, @@ -86,10 +90,10 @@ pub(crate) fn impl_overseer_struct( Ok(()) } - pub async fn route_message(&mut self, msg: #message_wrapper) -> SubsystemResult<()> { - match msg { + pub async fn route_message(&mut self, message: #message_wrapper) -> SubsystemResult<()> { + match message { #( - #message_wrapper :: #consumes (msg) => self. #subsystem_name .send_message(msg).await?, + #message_wrapper :: #consumes ( inner ) => self. #subsystem_name .send_message( inner ).await?, )* } Ok(()) @@ -147,12 +151,13 @@ pub(crate) fn impl_builder( < Ctx, #( #builder_generic_ty, )* > }; + let error_ty = &info.extern_error_ty; let builder_where_clause = quote! { where Ctx: SubsystemContext, S: ::polkadot_overseer_gen::SpawnNamed, - #( #builder_generic_ty : Subsystem, )* + #( #builder_generic_ty : Subsystem, )* }; let consumes = &info.consumes(); @@ -171,7 +176,7 @@ pub(crate) fn impl_builder( } - pub type #handler = ::polkadot_overseer_gen::metered::UnboundedMeteredSender< #event >; + pub type #handler = ::polkadot_overseer_gen::metered::MeteredSender< #event >; #[derive(Debug, Clone)] struct #builder #builder_generics { @@ -218,20 +223,29 @@ pub(crate) fn impl_builder( ) -> Ctx, { - let (events_tx, events_rx) = ::polkadot_overseer_gen::metered::channel(SIGNAL_CHANNEL_CAPACITY); + let (events_tx, events_rx) = ::polkadot_overseer_gen::metered::channel::< + #event + >(SIGNAL_CHANNEL_CAPACITY); let handler: #handler = events_tx.clone(); - let (to_overseer_tx, to_overseer_rx) = ::polkadot_overseer_gen::metered::unbounded(); + let (to_overseer_tx, to_overseer_rx) = ::polkadot_overseer_gen::metered::unbounded::< + ToOverseer + >(); let channels_out = { #( - let (#channel_name_tx, #channel_name_rx) = ::polkadot_overseer_gen::metered::channel::< - MessagePacket< #consumes >>(CHANNEL_CAPACITY); + let (#channel_name_tx, #channel_name_rx) = + ::polkadot_overseer_gen::metered::channel::< + MessagePacket< #consumes > + >(CHANNEL_CAPACITY); )* #( - let (#channel_name_unbounded_tx, #channel_name_unbounded_rx) = ::polkadot_overseer_gen::metered::unbounded::>(); + let (#channel_name_unbounded_tx, #channel_name_unbounded_rx) = + ::polkadot_overseer_gen::metered::unbounded::< + MessagePacket< #consumes > + >(); )* ChannelsOut { @@ -246,7 +260,9 @@ pub(crate) fn impl_builder( let spawner = self.spawner.expect("Spawner is set. qed"); - let mut running_subsystems = ::polkadot_overseer_gen::FuturesUnordered::>>::new(); + let mut running_subsystems = ::polkadot_overseer_gen::FuturesUnordered::< + BoxFuture<'static, SubsystemResult<()>> + >::new(); #( // FIXME generate a builder pattern that ensures this @@ -256,7 +272,8 @@ pub(crate) fn impl_builder( let unbounded_meter = channels_out. #channel_name .meter().clone(); - let message_tx: ::polkadot_overseer_gen::metered::UnboundedMeteredSender> = channels_out. #channel_name .clone(); + let message_tx: ::polkadot_overseer_gen::metered::MeteredSender< MessagePacket< #consumes > > + = channels_out. #channel_name .clone(); let message_rx: SubsystemIncomingMessages< #message_wrapper > = ::polkadot_overseer_gen::select( @@ -264,7 +281,7 @@ pub(crate) fn impl_builder( channels_out. #channel_name_unbounded .clone(), ); - let (signal_tx, signal_rx) = ::polkadot_overseer_gen::metered::channel::(SIGNAL_CHANNEL_CAPACITY); + let (signal_tx, signal_rx) = ::polkadot_overseer_gen::metered::channel::< #signal >(SIGNAL_CHANNEL_CAPACITY); let ctx = create_subsystem_ctx( signal_rx, @@ -346,6 +363,7 @@ pub(crate) fn impl_builder( pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { let signal = &info.extern_signal_ty; let message_wrapper = &info.message_wrapper; + let consumes = &info.consumes(); let ts = quote::quote! { @@ -356,21 +374,23 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { - pub instance: std::option::Option>, + pub struct OverseenSubsystem { + pub instance: std::option::Option< + SubsystemInstance + >, } - impl OverseenSubsystem { + impl OverseenSubsystem { /// Send a message to the wrapped subsystem. /// /// If the inner `instance` is `None`, nothing is happening. - pub async fn send_message(&mut self, msg: #message_wrapper) -> SubsystemResult<()> { + pub async fn send_message(&mut self, message: M) -> SubsystemResult<()> { const MESSAGE_TIMEOUT: Duration = Duration::from_secs(10); if let Some(ref mut instance) = self.instance { match instance.tx_bounded.send(MessagePacket { signals_received: instance.signals_received, - message: msg.into() + message: message.into(), }).timeout(MESSAGE_TIMEOUT).await { None => { diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 59b5ccc5cfdc..da48001fdc66 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -86,6 +86,7 @@ pub use std::time::Duration; use std::sync::atomic::{self, AtomicUsize}; use std::sync::Arc; +// pub use tokio_util::time; pub use futures_timer::Delay; /// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. /// @@ -283,11 +284,18 @@ pub struct SubsystemMeterReadouts { /// /// `M` here is the inner message type, and _not_ the generated `enum AllMessages`. pub struct SubsystemInstance { - tx_signal: crate::metered::MeteredSender, - tx_bounded: crate::metered::MeteredSender>, - meters: SubsystemMeters, - signals_received: usize, - name: &'static str, + /// Send sink for `Signal`s to be sent to a subsystem. + pub tx_signal: crate::metered::MeteredSender, + /// Send sink for `Message`s to be sent to a subsystem. + pub tx_bounded: crate::metered::MeteredSender>, + /// All meters of the particular subsystem instance. + pub meters: SubsystemMeters, + /// The number of signals already received. + /// Required to assure messages and signals + /// are processed correctly. + pub signals_received: usize, + /// Name of the subsystem instance. + pub name: &'static str, } /// A message type that a subsystem receives from an overseer. @@ -373,21 +381,77 @@ where /// TODO FIXME #[async_trait::async_trait] -pub trait SubsystemSender: Send + Clone + 'static { +pub trait SubsystemSender: Send + Clone + 'static { + // Inner message type. + // type Message: Send + Clone + 'static; + /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: Message); + async fn send_message(&mut self, msg: AllMessages); /// Send multiple direct messages to other `Subsystem`s, routed based on message type. async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send; + where T: IntoIterator + Send, T::IntoIter: Send; /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message /// type. /// /// This function should be used only when there is some other bounding factor on the messages /// sent with it. Otherwise, it risks a memory leak. - fn send_unbounded_message(&mut self, msg: Message); + fn send_unbounded_message(&mut self, msg: AllMessages); +} + + + + + +use futures::task::{Poll, Context}; +use std::pin::Pin; + +/// A future that wraps another future with a `Delay` allowing for time-limited futures. +#[pin_project::pin_project] +pub struct Timeout { + #[pin] + future: F, + #[pin] + delay: Delay, +} + +/// Extends `Future` to allow time-limited futures. +pub trait TimeoutExt: Future { + /// Adds a timeout of `duration` to the given `Future`. + /// Returns a new `Future`. + fn timeout(self, duration: Duration) -> Timeout + where + Self: Sized, + { + Timeout { + future: self, + delay: Delay::new(duration), + } + } } +impl TimeoutExt for F {} + +impl Future for Timeout { + type Output = Option; + + fn poll(self: Pin<&mut Self>, ctx: &mut Context) -> Poll { + let this = self.project(); + + if this.delay.poll(ctx).is_ready() { + return Poll::Ready(None); + } + + if let Poll::Ready(output) = this.future.poll(ctx) { + return Poll::Ready(Some(output)); + } + + Poll::Pending + } +} + + + #[cfg(test)] mod tests; diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 71c26f386c8c..e941819726b8 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -340,11 +340,11 @@ pub struct OverseerSubsystemContext{ metrics: Metrics, } -impl OverseerSubsystemContext { +impl OverseerSubsystemContext { /// Create a new `OverseerSubsystemContext`. fn new( signals: metered::MeteredReceiver, - messages: SubsystemIncomingMessages, + messages: SubsystemIncomingMessages, to_subsystems: ChannelsOut, to_overseer: metered::UnboundedMeteredSender, metrics: Metrics, @@ -370,7 +370,7 @@ impl OverseerSubsystemContext { #[allow(unused)] fn new_unmetered( signals: metered::MeteredReceiver, - messages: SubsystemIncomingMessages, + messages: SubsystemIncomingMessages, to_subsystems: ChannelsOut, to_overseer: metered::UnboundedMeteredSender, ) -> Self { diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 8da385f67c4b..575a9e42a337 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -27,17 +27,18 @@ use polkadot_node_subsystem::{ errors::RuntimeApiError, messages::{RuntimeApiMessage, RuntimeApiRequest, RuntimeApiSender, BoundToRelayParent}, - SpawnedSubsystem, SubsystemError, ActiveLeavesUpdate, OverseerSignal, }; use polkadot_overseer_gen::{ - AllMessages, FromOverseer, SpawnedSubsystem, Subsystem, SubsystemContext, + SubsystemError, SubsystemSender, + TimeoutExt, + Timeout, }; use polkadot_node_jaeger as jaeger; @@ -71,7 +72,7 @@ pub use error_handling::{Fault, unwrap_non_fatal}; /// These reexports are required so that external crates can use the `delegated_subsystem` macro properly. pub mod reexports { - pub use polkadot_overseer::{ + pub use polkadot_overseer_gen::{ SpawnNamed, SpawnedSubsystem, Subsystem, @@ -120,14 +121,15 @@ pub enum Error { pub type RuntimeApiReceiver = oneshot::Receiver>; /// Request some data from the `RuntimeApi`. -pub async fn request_from_runtime( +pub async fn request_from_runtime( parent: Hash, sender: &mut Sender, request_builder: RequestBuilder, ) -> RuntimeApiReceiver where RequestBuilder: FnOnce(RuntimeApiSender) -> RuntimeApiRequest, - Sender: SubsystemSender, + Sender: SubsystemSender, + AllMessages: From, { let (tx, rx) = oneshot::channel(); @@ -152,14 +154,14 @@ macro_rules! specialize_requests { #[doc = "Request `"] #[doc = $doc_name] #[doc = "` from the runtime"] - pub async fn $func_name( + pub async fn $func_name > ( parent: Hash, $( $param_name: $param_ty, )* sender: &mut impl SubsystemSender , ) -> RuntimeApiReceiver<$return_ty> { - request_from_runtime(parent, sender, |tx| RuntimeApiRequest::$request_variant( + request_from_runtime::<_, _, _, AllMessages>(parent, sender, |tx| RuntimeApiRequest::$request_variant( $( $param_name, )* tx )).await } @@ -266,11 +268,14 @@ pub struct Validator { impl Validator { /// Get a struct representing this node's validator if this node is in fact a validator in the context of the given block. - pub async fn new( + pub async fn new( parent: Hash, keystore: SyncCryptoStorePtr, sender: &mut impl SubsystemSender , - ) -> Result { + ) -> Result + where + AllMessages: From, + { // Note: request_validators and request_session_index_for_child do not and cannot // run concurrently: they both have a mutable handle to the same sender. // However, each of them returns a oneshot::Receiver, and those are resolved concurrently. @@ -399,12 +404,13 @@ pub enum FromJobCommand { /// A sender for messages from jobs, as well as commands to the overseer. #[derive(Clone)] -pub struct JobSender { +pub struct JobSender, AllMessages> { sender: S, from_job: mpsc::Sender, + _phantom: std::marker::PhantomData, } -impl> JobSender { +impl, AllMessages> JobSender { /// Get access to the underlying subsystem sender. pub fn subsystem_sender(&mut self) -> &mut S { &mut self.sender @@ -439,7 +445,10 @@ impl> JobSender { } #[async_trait::async_trait] -impl > SubsystemSender for JobSender { +impl, AllMessages> SubsystemSender for JobSender +where + AllMessages: Clone + Send + 'static +{ async fn send_message(&mut self, msg: AllMessages) { self.sender.send_message(msg).await } @@ -484,19 +493,22 @@ pub trait JobTrait: Unpin + Sized { /// The `delegate_subsystem!` macro should take care of this. type Metrics: 'static + metrics::Metrics + Send; + /// The generated wrapping message type. + type Message: 'static + Send; + /// Name of the job, i.e. `CandidateBackingJob` const NAME: &'static str; /// Run a job for the given relay `parent`. /// /// The job should be ended when `receiver` returns `None`. - fn run>( + fn run>( parent: Hash, span: Arc, run_args: Self::RunArgs, metrics: Self::Metrics, receiver: mpsc::Receiver, - sender: JobSender, + sender: JobSender, ) -> Pin> + Send>>; } @@ -538,7 +550,7 @@ impl Jobs { } /// Spawn a new job for this `parent_hash`, with whatever args are appropriate. - fn spawn_job( + fn spawn_job( &mut self, parent_hash: Hash, span: Arc, @@ -675,11 +687,12 @@ impl JobSubsystem { pub async fn run(self, mut ctx: Context) where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, - Context: SubsystemContext, - Job: 'static + JobTrait + Send, + Context: SubsystemContext, + Job: 'static + JobTrait::Message> + Send, Job::RunArgs: Clone + Sync, Job::ToJob: From<::Message> + Sync, Job::Metrics: Sync, + ::Message: From, { let JobSubsystem { params: JobSubsystemParams { @@ -690,7 +703,7 @@ impl JobSubsystem { .. } = self; - let mut jobs = Jobs::new(spawner); + let mut jobs = Jobs::::new(spawner); loop { select! { @@ -702,7 +715,7 @@ impl JobSubsystem { }))) => { for activated in activated { let sender: Context::Sender = ctx.sender().clone(); - jobs.spawn_job::( + jobs.spawn_job::( activated.hash, activated.span, run_args.clone(), @@ -752,7 +765,7 @@ impl JobSubsystem { } } -impl Subsystem for JobSubsystem +impl Subsystem for JobSubsystem where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext, @@ -774,50 +787,6 @@ where } } -/// A future that wraps another future with a `Delay` allowing for time-limited futures. -#[pin_project] -pub struct Timeout { - #[pin] - future: F, - #[pin] - delay: Delay, -} - -/// Extends `Future` to allow time-limited futures. -pub trait TimeoutExt: Future { - /// Adds a timeout of `duration` to the given `Future`. - /// Returns a new `Future`. - fn timeout(self, duration: Duration) -> Timeout - where - Self: Sized, - { - Timeout { - future: self, - delay: Delay::new(duration), - } - } -} - -impl TimeoutExt for F {} - -impl Future for Timeout { - type Output = Option; - - fn poll(self: Pin<&mut Self>, ctx: &mut Context) -> Poll { - let this = self.project(); - - if this.delay.poll(ctx).is_ready() { - return Poll::Ready(None); - } - - if let Poll::Ready(output) = this.future.poll(ctx) { - return Poll::Ready(Some(output)); - } - - Poll::Pending - } -} - #[derive(Copy, Clone)] enum MetronomeState { @@ -879,8 +848,8 @@ mod tests { use thiserror::Error; use polkadot_node_jaeger as jaeger; use polkadot_node_subsystem::{ - messages::{CandidateSelectionMessage}, ActiveLeavesUpdate, - SpawnedSubsystem, ActivatedLeaf, + messages::CandidateSelectionMessage, ActiveLeavesUpdate, + ActivatedLeaf, }; use assert_matches::assert_matches; use futures::{channel::mpsc, executor, StreamExt, future, Future, FutureExt, SinkExt}; @@ -926,7 +895,7 @@ mod tests { run_args: Self::RunArgs, _metrics: Self::Metrics, receiver: mpsc::Receiver, - mut sender: JobSender, + mut sender: JobSender, ) -> Pin> + Send>> { async move { let job = FakeCandidateSelectionJob { receiver }; From f4defb29f85293e4f9f720fd78bfa290c15b2282 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 20 May 2021 19:11:15 +0200 Subject: [PATCH 040/161] cargo spellcheck --- README.md | 2 +- bridges/primitives/chain-kusama/src/lib.rs | 4 ++-- bridges/primitives/chain-polkadot/src/lib.rs | 4 ++-- bridges/primitives/chain-rococo/src/lib.rs | 4 ++-- bridges/primitives/chain-westend/src/lib.rs | 4 ++-- bridges/primitives/runtime/src/lib.rs | 4 ++-- node/core/pvf/src/artifacts.rs | 2 +- node/core/pvf/src/host.rs | 10 +++++----- .../network/availability-distribution/src/error.rs | 2 +- .../src/pov_requester/mod.rs | 2 +- .../src/requester/fetch_task/mod.rs | 4 ++-- .../availability-distribution/src/requester/mod.rs | 2 +- .../availability-distribution/src/tests/state.rs | 2 +- node/network/bitfield-distribution/src/lib.rs | 4 ++-- node/network/bridge/src/lib.rs | 2 +- node/network/bridge/src/network.rs | 2 +- .../network/collator-protocol/src/collator_side.rs | 4 ++-- node/network/protocol/src/lib.rs | 8 ++++---- node/network/protocol/src/peer_set.rs | 2 +- .../protocol/src/request_response/request.rs | 2 +- node/network/statement-distribution/src/error.rs | 2 +- .../statement-distribution/src/requester.rs | 4 ++-- .../statement-distribution/src/responder.rs | 4 ++-- node/overseer/overseer-gen/src/lib.rs | 8 ++++---- node/primitives/src/lib.rs | 4 ++-- node/service/src/chain_spec.rs | 8 ++++---- node/subsystem-util/src/error_handling.rs | 14 +++++++------- node/subsystem-util/src/runtime/mod.rs | 2 +- node/subsystem/src/errors.rs | 2 +- node/subsystem/src/lib.rs | 2 +- node/subsystem/src/messages.rs | 2 +- .../subsystem/src/messages/network_bridge_event.rs | 2 +- node/test/service/src/chain_spec.rs | 2 +- node/test/service/src/lib.rs | 8 ++++---- primitives/src/v0.rs | 4 ++-- primitives/src/v1/mod.rs | 10 +++++----- runtime/common/src/auctions.rs | 2 +- runtime/common/src/claims.rs | 2 +- runtime/common/src/crowdloan.rs | 2 +- 39 files changed, 77 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index 584307726f36..5281d40803a5 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ service that can be used to run a Polkadot node. This is disabled by default, and can be started by running `systemctl start polkadot` on demand (use `systemctl enable polkadot` to make it auto-start after reboot). By default, it will run as the `polkadot` user. Command-line flags passed to the binary can -be customised by editing `/etc/default/polkadot`. This file will not be +be customized by editing `/etc/default/polkadot`. This file will not be overwritten on updating polkadot. You may also just run the node directly from the command-line. diff --git a/bridges/primitives/chain-kusama/src/lib.rs b/bridges/primitives/chain-kusama/src/lib.rs index 7163d15ef137..b221aff049d4 100644 --- a/bridges/primitives/chain-kusama/src/lib.rs +++ b/bridges/primitives/chain-kusama/src/lib.rs @@ -79,7 +79,7 @@ sp_api::decl_runtime_apis! { /// /// Returns `None` if message is too expensive to be sent to Kusama from this chain. /// - /// Please keep in mind that this method returns lowest message fee required for message + /// Please keep in mind that this method returns the lowest message fee required for message /// to be accepted to the lane. It may be good idea to pay a bit over this price to account /// future exchange rate changes and guarantee that relayer would deliver your message /// to the target chain. @@ -109,7 +109,7 @@ sp_api::decl_runtime_apis! { pub trait FromKusamaInboundLaneApi { /// Returns nonce of the latest message, received by given lane. fn latest_received_nonce(lane: LaneId) -> MessageNonce; - /// Nonce of latest message that has been confirmed to the bridged chain. + /// Nonce of the latest message that has been confirmed to the bridged chain. fn latest_confirmed_nonce(lane: LaneId) -> MessageNonce; /// State of the unrewarded relayers set at given lane. fn unrewarded_relayers_state(lane: LaneId) -> UnrewardedRelayersState; diff --git a/bridges/primitives/chain-polkadot/src/lib.rs b/bridges/primitives/chain-polkadot/src/lib.rs index 8e0d30cdb607..8398b3d52733 100644 --- a/bridges/primitives/chain-polkadot/src/lib.rs +++ b/bridges/primitives/chain-polkadot/src/lib.rs @@ -79,7 +79,7 @@ sp_api::decl_runtime_apis! { /// /// Returns `None` if message is too expensive to be sent to Polkadot from this chain. /// - /// Please keep in mind that this method returns lowest message fee required for message + /// Please keep in mind that this method returns the lowest message fee required for message /// to be accepted to the lane. It may be good idea to pay a bit over this price to account /// future exchange rate changes and guarantee that relayer would deliver your message /// to the target chain. @@ -109,7 +109,7 @@ sp_api::decl_runtime_apis! { pub trait FromPolkadotInboundLaneApi { /// Returns nonce of the latest message, received by given lane. fn latest_received_nonce(lane: LaneId) -> MessageNonce; - /// Nonce of latest message that has been confirmed to the bridged chain. + /// Nonce of the latest message that has been confirmed to the bridged chain. fn latest_confirmed_nonce(lane: LaneId) -> MessageNonce; /// State of the unrewarded relayers set at given lane. fn unrewarded_relayers_state(lane: LaneId) -> UnrewardedRelayersState; diff --git a/bridges/primitives/chain-rococo/src/lib.rs b/bridges/primitives/chain-rococo/src/lib.rs index 66cbb475bda2..f5953d3c8e1c 100644 --- a/bridges/primitives/chain-rococo/src/lib.rs +++ b/bridges/primitives/chain-rococo/src/lib.rs @@ -127,7 +127,7 @@ sp_api::decl_runtime_apis! { /// /// Returns `None` if message is too expensive to be sent to Rococo from this chain. /// - /// Please keep in mind that this method returns lowest message fee required for message + /// Please keep in mind that this method returns the lowest message fee required for message /// to be accepted to the lane. It may be good idea to pay a bit over this price to account /// future exchange rate changes and guarantee that relayer would deliver your message /// to the target chain. @@ -157,7 +157,7 @@ sp_api::decl_runtime_apis! { pub trait FromRococoInboundLaneApi { /// Returns nonce of the latest message, received by given lane. fn latest_received_nonce(lane: LaneId) -> MessageNonce; - /// Nonce of latest message that has been confirmed to the bridged chain. + /// Nonce of the latest message that has been confirmed to the bridged chain. fn latest_confirmed_nonce(lane: LaneId) -> MessageNonce; /// State of the unrewarded relayers set at given lane. fn unrewarded_relayers_state(lane: LaneId) -> UnrewardedRelayersState; diff --git a/bridges/primitives/chain-westend/src/lib.rs b/bridges/primitives/chain-westend/src/lib.rs index db97364ef419..08ca9c28c8ce 100644 --- a/bridges/primitives/chain-westend/src/lib.rs +++ b/bridges/primitives/chain-westend/src/lib.rs @@ -141,7 +141,7 @@ sp_api::decl_runtime_apis! { /// /// Returns `None` if message is too expensive to be sent to Westend from this chain. /// - /// Please keep in mind that this method returns lowest message fee required for message + /// Please keep in mind that this method returns the lowest message fee required for message /// to be accepted to the lane. It may be good idea to pay a bit over this price to account /// future exchange rate changes and guarantee that relayer would deliver your message /// to the target chain. @@ -171,7 +171,7 @@ sp_api::decl_runtime_apis! { pub trait FromWestendInboundLaneApi { /// Returns nonce of the latest message, received by given lane. fn latest_received_nonce(lane: LaneId) -> MessageNonce; - /// Nonce of latest message that has been confirmed to the bridged chain. + /// Nonce of the latest message that has been confirmed to the bridged chain. fn latest_confirmed_nonce(lane: LaneId) -> MessageNonce; /// State of the unrewarded relayers set at given lane. fn unrewarded_relayers_state(lane: LaneId) -> UnrewardedRelayersState; diff --git a/bridges/primitives/runtime/src/lib.rs b/bridges/primitives/runtime/src/lib.rs index aa457f1d0b0b..e12f484417dd 100644 --- a/bridges/primitives/runtime/src/lib.rs +++ b/bridges/primitives/runtime/src/lib.rs @@ -70,7 +70,7 @@ pub type InstanceId = [u8; 4]; /// Type of accounts on the source chain. pub enum SourceAccount { - /// An account that belongs to Root (priviledged origin). + /// An account that belongs to Root (privileged origin). Root, /// A non-priviledged account. /// @@ -82,7 +82,7 @@ pub enum SourceAccount { /// Derive an account ID from a foreign account ID. /// /// This function returns an encoded Blake2 hash. It is the responsibility of the caller to ensure -/// this can be succesfully decoded into an AccountId. +/// this can be successfully decoded into an AccountId. /// /// The `bridge_id` is used to provide extra entropy when producing account IDs. This helps prevent /// AccountId collisions between different bridges on a single target chain. diff --git a/node/core/pvf/src/artifacts.rs b/node/core/pvf/src/artifacts.rs index e84e11193d13..7c8188a323c2 100644 --- a/node/core/pvf/src/artifacts.rs +++ b/node/core/pvf/src/artifacts.rs @@ -168,7 +168,7 @@ impl Artifacts { .is_none()); } - /// Remove and retrive the artifacts from the table that are older than the supplied Time-To-Live. + /// Remove and retrieve the artifacts from the table that are older than the supplied Time-To-Live. pub fn prune(&mut self, artifact_ttl: Duration) -> Vec { let now = SystemTime::now(); diff --git a/node/core/pvf/src/host.rs b/node/core/pvf/src/host.rs index 4197acccb3a3..303daa53698d 100644 --- a/node/core/pvf/src/host.rs +++ b/node/core/pvf/src/host.rs @@ -52,8 +52,8 @@ impl ValidationHost { /// Execute PVF with the given code, params and priority. The result of execution will be sent /// to the provided result sender. /// - /// This is async to accomodate the fact a possibility of back-pressure. In vast majority of - /// situations this function should return immediatelly. + /// This is async to accommodate the fact a possibility of back-pressure. In the vast majority of + /// situations this function should return immediately. /// /// Returns an error if the request cannot be sent to the validation host, i.e. if it shut down. pub async fn execute_pvf( @@ -76,8 +76,8 @@ impl ValidationHost { /// Sends a signal to the validation host requesting to prepare a list of the given PVFs. /// - /// This is async to accomodate the fact a possibility of back-pressure. In vast majority of - /// situations this function should return immediatelly. + /// This is async to accommodate the fact a possibility of back-pressure. In the vast majority of + /// situations this function should return immediately. /// /// Returns an error if the request cannot be sent to the validation host, i.e. if it shut down. pub async fn heads_up(&mut self, active_pvfs: Vec) -> Result<(), String> { @@ -218,7 +218,7 @@ struct PendingExecutionRequest { result_tx: ResultSender, } -/// A mapping from an artifact ID which is in preparation state to the list of pending exeuction +/// A mapping from an artifact ID which is in preparation state to the list of pending execution /// requests that should be executed once the artifact's prepration is finished. #[derive(Default)] struct AwaitingPrepare(HashMap>); diff --git a/node/network/availability-distribution/src/error.rs b/node/network/availability-distribution/src/error.rs index 893bb6bc46d1..666e9a377690 100644 --- a/node/network/availability-distribution/src/error.rs +++ b/node/network/availability-distribution/src/error.rs @@ -68,7 +68,7 @@ pub enum Fatal { Runtime(#[from] #[source] runtime::Fatal), } -/// Non fatal errors of this subsystem. +/// Non-fatal errors of this subsystem. #[derive(Debug, Error)] pub enum NonFatal { /// av-store will drop the sender on any error that happens. diff --git a/node/network/availability-distribution/src/pov_requester/mod.rs b/node/network/availability-distribution/src/pov_requester/mod.rs index 5b0038cd7537..414fb0a41eee 100644 --- a/node/network/availability-distribution/src/pov_requester/mod.rs +++ b/node/network/availability-distribution/src/pov_requester/mod.rs @@ -170,7 +170,7 @@ async fn do_fetch_pov( } } -/// Get the session indeces for the given relay chain parents. +/// Get the session indices for the given relay chain parents. async fn get_activated_sessions(ctx: &mut Context, runtime: &mut RuntimeInfo, new_heads: impl Iterator) -> super::Result> where diff --git a/node/network/availability-distribution/src/requester/fetch_task/mod.rs b/node/network/availability-distribution/src/requester/fetch_task/mod.rs index bdb51bdd2a1c..42f5554ae839 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/mod.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/mod.rs @@ -221,13 +221,13 @@ impl FetchTask { } } - /// Whether or not there are still relay parents around with this candidate pending + /// Whether there are still relay parents around with this candidate pending /// availability. pub fn is_live(&self) -> bool { !self.live_in.is_empty() } - /// Whether or not this task can be considered finished. + /// Whether this task can be considered finished. /// /// That is, it is either canceled, succeeded or failed. pub fn is_finished(&self) -> bool { diff --git a/node/network/availability-distribution/src/requester/mod.rs b/node/network/availability-distribution/src/requester/mod.rs index 306cd1b96e76..4a610c23a553 100644 --- a/node/network/availability-distribution/src/requester/mod.rs +++ b/node/network/availability-distribution/src/requester/mod.rs @@ -91,7 +91,7 @@ impl Requester { } /// Update heads that need availability distribution. /// - /// For all active heads we will be fetching our chunks for availabilty distribution. + /// For all active heads we will be fetching our chunks for availability distribution. #[tracing::instrument(level = "trace", skip(self, ctx, runtime, update), fields(subsystem = LOG_TARGET))] pub async fn update_fetching_heads( &mut self, diff --git a/node/network/availability-distribution/src/tests/state.rs b/node/network/availability-distribution/src/tests/state.rs index 73fcd5f7ce91..7364b7fb3f26 100644 --- a/node/network/availability-distribution/src/tests/state.rs +++ b/node/network/availability-distribution/src/tests/state.rs @@ -68,7 +68,7 @@ pub struct TestState { pub relay_chain: Vec, /// Whenever the subsystem tries to fetch an erasure chunk one item of the given vec will be /// popped. So you can experiment with serving invalid chunks or no chunks on request and see - /// whether the subystem still succeds with its goal. + /// whether the subsystem still succeeds with its goal. pub chunks: HashMap<(CandidateHash, ValidatorIndex), Vec>>, /// All chunks that are valid and should be accepted. pub valid_chunks: HashSet<(CandidateHash, ValidatorIndex)>, diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index 92c160cfc235..848eeb02bb71 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -233,7 +233,7 @@ impl BitfieldDistribution { } } -/// Modify the reputation of a peer based on its behaviour. +/// Modify the reputation of a peer based on its behavior. #[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn modify_reputation( ctx: &mut Context, @@ -560,7 +560,7 @@ where } } -/// Handle the changes necassary when our view changes. +/// Handle the changes necessary when our view changes. #[tracing::instrument(level = "trace", fields(subsystem = LOG_TARGET))] fn handle_our_view_change(state: &mut ProtocolState, view: OurView) { let old_view = std::mem::replace(&mut (state.view), view); diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index b02ec9723e3f..bd4134ecea10 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -298,7 +298,7 @@ impl Subsystem for NetworkBridge } struct PeerData { - /// Latest view sent by the peer. + /// The Latest view sent by the peer. view: View, } diff --git a/node/network/bridge/src/network.rs b/node/network/bridge/src/network.rs index 2f6fa8d1f558..8eda64a2c262 100644 --- a/node/network/bridge/src/network.rs +++ b/node/network/bridge/src/network.rs @@ -93,7 +93,7 @@ where /// An action to be carried out by the network. /// -/// This type is used for implementing `Sink` in order to cummunicate asynchronously with the +/// This type is used for implementing `Sink` in order to communicate asynchronously with the /// underlying network implementation in the `Network` trait. #[derive(Debug, PartialEq)] pub enum NetworkAction { diff --git a/node/network/collator-protocol/src/collator_side.rs b/node/network/collator-protocol/src/collator_side.rs index acfcda217957..b51ca89dcb5e 100644 --- a/node/network/collator-protocol/src/collator_side.rs +++ b/node/network/collator-protocol/src/collator_side.rs @@ -1061,7 +1061,7 @@ mod tests { /// Generate a new relay parent and inform the subsystem about the new view. /// - /// If `merge_views == true` it means the subsystem will be informed that we working on the old `relay_parent` + /// If `merge_views == true` it means the subsystem will be informed that we are working on the old `relay_parent` /// and the new one. async fn advance_to_new_round(&mut self, virtual_overseer: &mut VirtualOverseer, merge_views: bool) { let old_relay_parent = self.relay_parent; @@ -1360,7 +1360,7 @@ mod tests { ); } - /// Check that the next received message is a collation advertisment message. + /// Check that the next received message is a collation advertisement message. async fn expect_advertise_collation_msg( virtual_overseer: &mut VirtualOverseer, peer: &PeerId, diff --git a/node/network/protocol/src/lib.rs b/node/network/protocol/src/lib.rs index 5f19a4af8992..75e9a8805245 100644 --- a/node/network/protocol/src/lib.rs +++ b/node/network/protocol/src/lib.rs @@ -324,14 +324,14 @@ pub mod v1 { LargeStatement(StatementMetadata), } - /// Data that maes a statement unique. + /// Data that makes a statement unique. #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq, Hash)] pub struct StatementMetadata { - /// Relayt parent this statement is relevant under. + /// Relay parent this statement is relevant under. pub relay_parent: Hash, /// Hash of the candidate that got validated. pub candidate_hash: CandidateHash, - /// Validator that attested the valididty. + /// Validator that attested the validity. pub signed_by: ValidatorIndex, /// Signature of seconding validator. pub signature: ValidatorSignature, @@ -369,7 +369,7 @@ pub mod v1 { } } - /// Whether or not this message contains a large statement. + /// Whether this message contains a large statement. pub fn is_large_statement(&self) -> bool { if let Self::LargeStatement(_) = self { true diff --git a/node/network/protocol/src/peer_set.rs b/node/network/protocol/src/peer_set.rs index 520567a5bd77..e83019d5d385 100644 --- a/node/network/protocol/src/peer_set.rs +++ b/node/network/protocol/src/peer_set.rs @@ -30,7 +30,7 @@ pub enum PeerSet { Collation, } -/// Whether or not a node is an authority or not. +/// Whether a node is an authority or not. /// /// Peer set configuration gets adjusted accordingly. #[derive(Copy, Clone, Debug, Eq, PartialEq)] diff --git a/node/network/protocol/src/request_response/request.rs b/node/network/protocol/src/request_response/request.rs index 69b86ff7f642..f8df5ee4a152 100644 --- a/node/network/protocol/src/request_response/request.rs +++ b/node/network/protocol/src/request_response/request.rs @@ -104,7 +104,7 @@ pub enum Recipient { /// discovery system. #[derive(Debug)] pub struct OutgoingRequest { - /// Intendent recipient of this request. + /// Intended recipient of this request. pub peer: Recipient, /// The actual request to send over the wire. pub payload: Req, diff --git a/node/network/statement-distribution/src/error.rs b/node/network/statement-distribution/src/error.rs index 82e918dfd18f..097b37099975 100644 --- a/node/network/statement-distribution/src/error.rs +++ b/node/network/statement-distribution/src/error.rs @@ -28,7 +28,7 @@ use crate::LOG_TARGET; /// General result. pub type Result = std::result::Result; -/// Result for non fatal only failures. +/// Result for non-fatal only failures. pub type NonFatalResult = std::result::Result; /// Result for fatal only failures. pub type FatalResult = std::result::Result; diff --git a/node/network/statement-distribution/src/requester.rs b/node/network/statement-distribution/src/requester.rs index 2368da3cd2c1..f2430ed10d75 100644 --- a/node/network/statement-distribution/src/requester.rs +++ b/node/network/statement-distribution/src/requester.rs @@ -39,7 +39,7 @@ const RETRY_TIMEOUT: Duration = Duration::from_millis(500); /// Messages coming from a background task. pub enum RequesterMessage { - /// Get an update of availble peers to try for fetching a given statement. + /// Get an update of available peers to try for fetching a given statement. GetMorePeers { relay_parent: Hash, candidate_hash: CandidateHash, @@ -69,7 +69,7 @@ pub enum RequesterMessage { /// A fetching task, taking care of fetching large statements via request/response. /// /// A fetch task does not know about a particular `Statement` instead it just tries fetching a -/// `CommittedCandidateReceipt` from peers, whether or not this can be used to re-assemble one ore +/// `CommittedCandidateReceipt` from peers, whether this can be used to re-assemble one ore /// many `SignedFullStatement`s needs to be verified by the caller. pub async fn fetch( relay_parent: Hash, diff --git a/node/network/statement-distribution/src/responder.rs b/node/network/statement-distribution/src/responder.rs index 4f572446abc2..da7e914edc93 100644 --- a/node/network/statement-distribution/src/responder.rs +++ b/node/network/statement-distribution/src/responder.rs @@ -35,7 +35,7 @@ const COST_INVALID_REQUEST: Rep = Rep::CostMajor("Peer sent unparsable request") /// Messages coming from a background task. pub enum ResponderMessage { - /// Get an update of availble peers to try for fetching a given statement. + /// Get an update of available peers to try for fetching a given statement. GetData { requesting_peer: PeerId, relay_parent: Hash, @@ -48,7 +48,7 @@ pub enum ResponderMessage { /// A fetching task, taking care of fetching large statements via request/response. /// /// A fetch task does not know about a particular `Statement` instead it just tries fetching a -/// `CommittedCandidateReceipt` from peers, whether or not this can be used to re-assemble one ore +/// `CommittedCandidateReceipt` from peers, whether this can be used to re-assemble one ore /// many `SignedFullStatement`s needs to be verified by the caller. pub async fn respond( mut receiver: mpsc::Receiver, diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index da48001fdc66..f0267226e49c 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -22,7 +22,7 @@ //! check out that guide, documentation in this crate will be mostly discussing //! technical stuff. //! -//! An `Overseer` is something that allows spawning/stopping and overseing +//! An `Overseer` is something that allows spawning/stopping and overseeing //! asynchronous tasks as well as establishing a well-defined and easy to use //! protocol that the tasks can use to communicate with each other. It is desired //! that this protocol is the only way tasks communicate with each other, however @@ -235,7 +235,7 @@ pub enum SubsystemError { /// Per origin (or subsystem) annotations to wrap an error. #[error("Error originated in {origin}")] FromOrigin { - /// An additional anotation tag for the origin of `source`. + /// An additional annotation tag for the origin of `source`. origin: &'static str, /// The wrapped error. Marked as source for tracking the error chain. #[source] source: Box @@ -364,8 +364,8 @@ pub trait SubsystemContext: Send + 'static { /// A trait that describes the [`Subsystem`]s that can run on the [`Overseer`]. /// /// It is generic over the message type circulating in the system. -/// The idea that we want some type contaning persistent state that -/// can spawn actually running subsystems when asked to. +/// The idea that we want some type containing persistent state that +/// can spawn actually running subsystems when asked. /// /// [`Overseer`]: struct.Overseer.html /// [`Subsystem`]: trait.Subsystem.html diff --git a/node/primitives/src/lib.rs b/node/primitives/src/lib.rs index a59f6cd1e05d..7646ca1ff204 100644 --- a/node/primitives/src/lib.rs +++ b/node/primitives/src/lib.rs @@ -51,7 +51,7 @@ pub const POV_BOMB_LIMIT: usize = MAX_POV_SIZE as usize; /// /// This is the committed candidate receipt instead of the bare candidate receipt. As such, /// it gives access to the commitments to validators who have not executed the candidate. This -/// is necessary to allow a block-producing validator to include candidates from outside of the para +/// is necessary to allow a block-producing validator to include candidates from outside the para /// it is assigned to. #[derive(Clone, PartialEq, Eq, Encode, Decode)] pub enum Statement { @@ -202,7 +202,7 @@ pub struct CollationResult { pub collation: Collation, /// An optional result sender that should be informed about a successfully seconded collation. /// - /// There is no guarantee that this sender is informed ever about any result, it is completly okay to just drop it. + /// There is no guarantee that this sender is informed ever about any result, it is completely okay to just drop it. /// However, if it is called, it should be called with the signed statement of a parachain validator seconding the /// collation. pub result_sender: Option>, diff --git a/node/service/src/chain_spec.rs b/node/service/src/chain_spec.rs index 278724746be2..5458b9bcfa22 100644 --- a/node/service/src/chain_spec.rs +++ b/node/service/src/chain_spec.rs @@ -59,16 +59,16 @@ pub struct Extensions { pub bad_blocks: sc_client_api::BadBlocks, } -/// The `ChainSpec` parametrised for the polkadot runtime. +/// The `ChainSpec` parameterized for the polkadot runtime. pub type PolkadotChainSpec = service::GenericChainSpec; -/// The `ChainSpec` parametrised for the kusama runtime. +/// The `ChainSpec` parameterized for the kusama runtime. pub type KusamaChainSpec = service::GenericChainSpec; -/// The `ChainSpec` parametrised for the westend runtime. +/// The `ChainSpec` parameterized for the westend runtime. pub type WestendChainSpec = service::GenericChainSpec; -/// The `ChainSpec` parametrized for the rococo runtime. +/// The `ChainSpec` parameterized for the rococo runtime. pub type RococoChainSpec = service::GenericChainSpec; /// Extension for the Rococo genesis config to support a custom changes to the genesis state. diff --git a/node/subsystem-util/src/error_handling.rs b/node/subsystem-util/src/error_handling.rs index b2040e53ab00..36e59bb7b2cb 100644 --- a/node/subsystem-util/src/error_handling.rs +++ b/node/subsystem-util/src/error_handling.rs @@ -20,8 +20,8 @@ //! //! - Ergonomic API with little repetition. //! - Still explicitness where it matters - fatal errors should be visible and justified. -//! - Easy recovering from non fatal errors. -//! - Errors start as non fatal and can be made fatal at the level where it is really clear they +//! - Easy recovering from non-fatal errors. +//! - Errors start as non-fatal and can be made fatal at the level where it is really clear they //! are fatal. E.g. cancellation of a oneshot might be fatal in one case, but absolutely expected //! in another. //! - Good error messages. Fatal errors don't need to be properly structured (as we won't handle @@ -37,7 +37,7 @@ use thiserror::Error; /// Errors might either be fatal and should bring the subsystem down or are at least at the point /// of occurrence deemed potentially recoverable. /// -/// Upper layers might have a better view and might make a non fatal error of a called function a +/// Upper layers might have a better view and might make a non-fatal error of a called function a /// fatal one. The opposite should not happen, therefore don't make an error fatal if you don't /// know it is in all cases. /// @@ -102,7 +102,7 @@ use thiserror::Error; /// } /// ``` /// Then mostly use `Error` in functions, you may also use `NonFatal` and `Fatal` directly in -/// functions that strictly only fail non fatal or fatal respectively, as `Fatal` and `NonFatal` +/// functions that strictly only fail non-fatal or fatal respectively, as `Fatal` and `NonFatal` /// can automatically converted into the above defined `Error`. /// ``` #[derive(Debug, Error)] @@ -135,7 +135,7 @@ impl Fault Self::Fatal(f.into()) } - /// Build an `Fault` from compatible non fatal error. + /// Build an `Fault` from compatible non-fatal error. pub fn from_non_fatal>(e: E1) -> Self { Self::Err(e.into()) } @@ -153,10 +153,10 @@ impl Fault } } -/// Unwrap non fatal error and report fatal one. +/// Unwrap non-fatal error and report fatal one. /// /// This function is useful for top level error handling. Fatal errors will be extracted, -/// non fatal error will be returned for handling. +/// non-fatal error will be returned for handling. /// /// Usage: /// diff --git a/node/subsystem-util/src/runtime/mod.rs b/node/subsystem-util/src/runtime/mod.rs index 5dda2e961f23..484df7b6cee1 100644 --- a/node/subsystem-util/src/runtime/mod.rs +++ b/node/subsystem-util/src/runtime/mod.rs @@ -63,7 +63,7 @@ pub struct ExtendedSessionInfo { pub validator_info: ValidatorInfo, } -/// Information about ourself, in case we are an `Authority`. +/// Information about ourselves, in case we are an `Authority`. /// /// This data is derived from the `SessionInfo` and our key as found in the keystore. pub struct ValidatorInfo { diff --git a/node/subsystem/src/errors.rs b/node/subsystem/src/errors.rs index e2c0d1af2a06..764c366c87eb 100644 --- a/node/subsystem/src/errors.rs +++ b/node/subsystem/src/errors.rs @@ -121,7 +121,7 @@ pub enum SubsystemError { /// Per origin (or subsystem) annotations to wrap an error. #[error("Error originated in {origin}")] FromOrigin { - /// An additional anotation tag for the origin of `source`. + /// An additional annotation tag for the origin of `source`. origin: &'static str, /// The wrapped error. Marked as source for tracking the error chain. #[source] source: Box diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 6509d37428d6..7f8aae8f0aaf 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -161,7 +161,7 @@ pub enum SubsystemError { /// Per origin (or subsystem) annotations to wrap an error. #[error("Error originated in {origin}")] FromOrigin { - /// An additional anotation tag for the origin of `source`. + /// An additional annotation tag for the origin of `source`. origin: &'static str, /// The wrapped error. Marked as source for tracking the error chain. #[source] source: Box diff --git a/node/subsystem/src/messages.rs b/node/subsystem/src/messages.rs index 95830ab5c539..87a68dafa95d 100644 --- a/node/subsystem/src/messages.rs +++ b/node/subsystem/src/messages.rs @@ -250,7 +250,7 @@ pub enum NetworkBridgeMessage { /// The underlying protocol to use for this request. peer_set: PeerSet, /// A request is revoked by dropping the `keep_alive` sender. - /// The revokation takes place upon the next connection request. + /// The revocation takes place upon the next connection request. keep_alive: oneshot::Receiver<()>, }, } diff --git a/node/subsystem/src/messages/network_bridge_event.rs b/node/subsystem/src/messages/network_bridge_event.rs index c0a58ada7704..8d7c2379858a 100644 --- a/node/subsystem/src/messages/network_bridge_event.rs +++ b/node/subsystem/src/messages/network_bridge_event.rs @@ -45,7 +45,7 @@ impl NetworkBridgeEvent { /// /// This tries to transform M in `PeerMessage` to a message type specific to a subsystem. /// It is used to dispatch events coming from a peer set to the various subsystems that are - /// handled within that peer set. More concretly a `ValidationProtocol` will be transformed + /// handled within that peer set. More concretely a `ValidationProtocol` will be transformed /// for example into a `BitfieldDistributionMessage` in case of the `BitfieldDistribution` /// constructor. /// diff --git a/node/test/service/src/chain_spec.rs b/node/test/service/src/chain_spec.rs index e0085fe17329..6c0bdace8521 100644 --- a/node/test/service/src/chain_spec.rs +++ b/node/test/service/src/chain_spec.rs @@ -29,7 +29,7 @@ use sp_runtime::Perbill; const DEFAULT_PROTOCOL_ID: &str = "dot"; -/// The `ChainSpec` parametrized for polkadot test runtime. +/// The `ChainSpec` parameterized for polkadot test runtime. pub type PolkadotChainSpec = service::GenericChainSpec; diff --git a/node/test/service/src/lib.rs b/node/test/service/src/lib.rs index 4104a53cdd61..e09d9adc4bd5 100644 --- a/node/test/service/src/lib.rs +++ b/node/test/service/src/lib.rs @@ -104,7 +104,7 @@ impl ClientHandle for TestClient { /// nodes if you want the future node to be connected to other nodes. /// /// The `storage_update_func` function will be executed in an externalities provided environment -/// and can be used to make adjustements to the runtime genesis storage. +/// and can be used to make adjustments to the runtime genesis storage. pub fn node_config( storage_update_func: impl Fn(), task_executor: TaskExecutor, @@ -210,7 +210,7 @@ pub fn node_config( /// want it to be connected to other nodes. /// /// The `storage_update_func` function will be executed in an externalities provided environment -/// and can be used to make adjustements to the runtime genesis storage. +/// and can be used to make adjustments to the runtime genesis storage. pub fn run_validator_node( task_executor: TaskExecutor, key: Sr25519Keyring, @@ -242,11 +242,11 @@ pub fn run_validator_node( /// want it to be connected to other nodes. /// /// The `storage_update_func` function will be executed in an externalities provided environment -/// and can be used to make adjustements to the runtime genesis storage. +/// and can be used to make adjustments to the runtime genesis storage. /// /// # Note /// -/// The collator functionionality still needs to be registered at the node! This can be done using +/// The collator functionality still needs to be registered at the node! This can be done using /// [`PolkadotTestNode::register_collator`]. pub fn run_collator_node( task_executor: TaskExecutor, diff --git a/primitives/src/v0.rs b/primitives/src/v0.rs index dc3c8dceb7d2..ab7c3fe18b8d 100644 --- a/primitives/src/v0.rs +++ b/primitives/src/v0.rs @@ -184,7 +184,7 @@ pub const PARACHAIN_INFO: Info = Info { scheduling: Scheduling::Always, }; -/// Auxilliary for when there's an attempt to swap two parachains/parathreads. +/// Auxiliary for when there's an attempt to swap two parachains/parathreads. pub trait SwapAux { /// Result describing whether it is possible to swap two parachains. Doesn't mutate state. fn ensure_can_swap(one: Id, other: Id) -> Result<(), &'static str>; @@ -435,7 +435,7 @@ pub struct AbridgedCandidateReceipt { /// A candidate-receipt with commitments directly included. pub struct CommitedCandidateReceipt { - /// The descriptor of the candidae. + /// The descriptor of the candidate. pub descriptor: CandidateDescriptor, /// The commitments of the candidate receipt. diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs index 48d6ad1e67e8..dc11e25fd5a9 100644 --- a/primitives/src/v1/mod.rs +++ b/primitives/src/v1/mod.rs @@ -181,7 +181,7 @@ pub const PARACHAINS_INHERENT_IDENTIFIER: InherentIdentifier = *b"parachn0"; pub const ASSIGNMENT_KEY_TYPE_ID: KeyTypeId = KeyTypeId(*b"asgn"); /// Maximum compressed code size we support right now. -/// At the moment we have runtime upgrade on chain, which restricts scalability severly. If we want +/// At the moment we have runtime upgrade on chain, which restricts scalability severely. If we want /// to have bigger values, we should fix that first. pub const MAX_CODE_SIZE: u32 = 3 * 1024 * 1024; @@ -370,7 +370,7 @@ impl CommittedCandidateReceipt { self.to_plain().hash() } - /// Does this committed candidate receipt corrensponds to the given [`CandidateReceipt`]? + /// Does this committed candidate receipt corresponds to the given [`CandidateReceipt`]? pub fn corresponds_to(&self, receipt: &CandidateReceipt) -> bool where H: PartialEq { receipt.descriptor == self.descriptor && receipt.commitments_hash == self.commitments.hash() } @@ -392,7 +392,7 @@ impl Ord for CommittedCandidateReceipt { } /// The validation data provides information about how to create the inputs for validation of a candidate. -/// This information is derived from the chain state and will vary from para to para, although some of the +/// This information is derived from the chain state and will vary from para to para, although some /// fields may be the same for every para. /// /// Since this data is used to form inputs to the validation function, it needs to be persisted by the @@ -408,7 +408,7 @@ impl Ord for CommittedCandidateReceipt { /// already been done. As such, there is no need for the validation data used to inform validators and /// collators about the checks the relay-chain will perform to be persisted by the availability system. /// -/// The `PersistedValidationData` should be relatively lightweight primarly because it is constructed +/// The `PersistedValidationData` should be relatively lightweight primarily because it is constructed /// during inclusion for each candidate and therefore lies on the critical path of inclusion. #[derive(PartialEq, Eq, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug, Default, MallocSizeOf))] @@ -1025,7 +1025,7 @@ pub enum ConsensusLog { /// A parachain or parathread upgraded its code. #[codec(index = 1)] ParaUpgradeCode(Id, Hash), - /// A parachain or parathread scheduled a code ugprade. + /// A parachain or parathread scheduled a code upgrade. #[codec(index = 2)] ParaScheduleUpgradeCode(Id, Hash, BlockNumber), /// Governance requests to auto-approve every candidate included up to the given block diff --git a/runtime/common/src/auctions.rs b/runtime/common/src/auctions.rs index 62c9ec75f612..a61ef590e8c9 100644 --- a/runtime/common/src/auctions.rs +++ b/runtime/common/src/auctions.rs @@ -133,7 +133,7 @@ decl_event!( /// [bidder, range, parachain_id, amount] WonDeploy(AccountId, SlotRange, ParaId, Balance), /// An existing parachain won the right to continue. - /// First balance is the extra amount reseved. Second is the total amount reserved. + /// First balance is the extra amount reserved. Second is the total amount reserved. /// [parachain_id, begin, count, total_amount] WonRenewal(ParaId, LeasePeriod, LeasePeriod, Balance), /// Funds were reserved for a winning bid. First balance is the extra amount reserved. diff --git a/runtime/common/src/claims.rs b/runtime/common/src/claims.rs index 31a937804f86..5824a7f5c4db 100644 --- a/runtime/common/src/claims.rs +++ b/runtime/common/src/claims.rs @@ -69,7 +69,7 @@ impl WeightInfo for TestWeightInfo { fn move_claim() -> Weight { 0 } } -/// The kind of a statement an account needs to make for a claim to be valid. +/// The kind of statement an account needs to make for a claim to be valid. #[derive(Encode, Decode, Clone, Copy, Eq, PartialEq, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] pub enum StatementKind { diff --git a/runtime/common/src/crowdloan.rs b/runtime/common/src/crowdloan.rs index c09c17736c29..4c9993a9ae0b 100644 --- a/runtime/common/src/crowdloan.rs +++ b/runtime/common/src/crowdloan.rs @@ -120,7 +120,7 @@ pub trait Config: frame_system::Config { /// Max number of storage keys to remove per extrinsic call. type RemoveKeysLimit: Get; - /// The parachain registrar type. We jus use this to ensure that only the manager of a para is able to + /// The parachain registrar type. We just use this to ensure that only the manager of a para is able to /// start a crowdloan for its slot. type Registrar: Registrar; From 4d7933348e8ed5b1dd421fe79f05e15e41a3832a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 21 May 2021 18:03:09 +0200 Subject: [PATCH 041/161] refactor, move things into the macro again, requires GATs for ctx init Fix the Ctx to the generated one. --- node/overseer/examples/minimal-example.rs | 1 + node/overseer/overseer-gen/Cargo.toml | 2 + node/overseer/overseer-gen/examples/dummy.rs | 13 +- .../proc-macro/src/impl_builder.rs | 317 ++++++++++++++++++ .../proc-macro/src/impl_dispatch.rs | 17 +- .../overseer-gen/proc-macro/src/impl_misc.rs | 225 +++++++++++++ .../proc-macro/src/impl_overseer.rs | 254 -------------- .../overseer-gen/proc-macro/src/lib.rs | 12 +- node/overseer/overseer-gen/src/lib.rs | 17 +- node/overseer/src/lib.rs | 195 +---------- 10 files changed, 608 insertions(+), 445 deletions(-) create mode 100644 node/overseer/overseer-gen/proc-macro/src/impl_builder.rs create mode 100644 node/overseer/overseer-gen/proc-macro/src/impl_misc.rs diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index 00e1ac7cada9..e8d45fcc6389 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -33,6 +33,7 @@ use polkadot_overseer::{Overseer, HeadSupportsParachains, AllSubsystems}; use polkadot_subsystem::{Subsystem, SubsystemContext, SpawnedSubsystem, FromOverseer}; use polkadot_subsystem::messages::{ CandidateValidationMessage, CandidateBackingMessage, + NetworkBridgeMessage, }; struct AlwaysSupportsParachains; diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index a15ff42d7ffa..1f5adc0c1583 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -17,6 +17,8 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } futures-timer = "3.0.2" tokio-util = { version = "0.6.7", features = [ "time" ] } pin-project = "1.0" +futures-util = "0.3" [dev-dependencies] trybuild = "1.0.41" +polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index fa43e28950e2..a46281bb12fe 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -1,5 +1,7 @@ //! A dummy to be used with cargo expand +use std::convert::Infallible; + use polkadot_overseer_gen::*; use polkadot_subsystem::messages::NetworkBridgeEvent; @@ -14,8 +16,9 @@ struct SigSigSig; #[derive(Debug, Clone)] struct EvX; + impl EvX { - pub fn focus<'a, T>(&'a self) -> Result, ()> { + pub fn focus<'a, T>(&'a self) -> Result { unimplemented!("dispatch") } } @@ -44,12 +47,18 @@ struct MsgStrukt(u8); struct Plinko; +impl From for MsgStrukt { + fn from(_event: EvX) -> Self { + MsgStrukt(1u8) + } +} + #[overlord(signal=SigSigSig, event=EvX, error=Yikes, gen=AllMessages)] struct Xxx { #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, - #[subsystem(Plinko)] + #[subsystem(no_dispatch, Plinko)] plinkos: AwesomeSubSys, } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs new file mode 100644 index 000000000000..29da343f268a --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -0,0 +1,317 @@ +use quote::quote; +use syn::{Ident, Result}; + +use super::*; + +/// Implement a builder pattern for the `Overseer`-type, +/// which acts as the gateway to constructing the overseer. +pub(crate) fn impl_builder( + info: &OverseerInfo, +) -> Result { + let overseer_name = info.overseer_name.clone(); + let builder = Ident::new(&(overseer_name.to_string() + "Builder"), overseer_name.span()); + let handler = Ident::new(&(overseer_name.to_string() + "Handler"), overseer_name.span()); + + let subsystem_name = &info.subsystem_names(); + let builder_generic_ty = &info.builder_generic_types(); + + let channel_name = &info.channel_names(""); + let channel_name_unbounded = &info.channel_names("_unbounded"); + + let channel_name_tx = &info.channel_names("_tx"); + let channel_name_unbounded_tx = &info.channel_names("_unbounded_tx"); + + let channel_name_rx = &info.channel_names("_rx"); + let channel_name_unbounded_rx = &info.channel_names("_unbounded_rx"); + + let baggage_generic_ty = &info.baggage_generic_types(); + let baggage_name = &info.baggage_names(); + let baggage_ty = &info.baggage_types(); + + let blocking = &info.subsystems().iter().map(|x| { + if x.blocking { + quote! { Blocking } + } else { + quote! { Regular } + } + }).collect::>(); + + let generics = quote! { + < S, #( #baggage_generic_ty, )* > + }; + let where_clause = quote! { + where + S: ::polkadot_overseer_gen::SpawnNamed, + }; + + let builder_generics = quote! { + + }; + + // all subsystems must have the same context + // even if the overseer does not impose such a limit. + let builder_additional_generics = quote! { + <#( #builder_generic_ty, )* > + }; + + let error_ty = &info.extern_error_ty; + let consumes = &info.consumes(); + + let builder_where_clause = quote! { + where + S: ::polkadot_overseer_gen::SpawnNamed, + #( #builder_generic_ty : Subsystem, #error_ty>, )* + }; + + let message_wrapper = &info.message_wrapper; + let event = &info.extern_event_ty; + let signal = &info.extern_signal_ty; + + let mut ts = quote! { + + impl #generics #overseer_name #generics #where_clause { + fn builder #builder_additional_generics () -> #builder #builder_generics + #builder_where_clause + { + #builder :: default() + } + } + + + pub type #handler = ::polkadot_overseer_gen::metered::MeteredSender< #event >; + + #[derive(Debug, Clone)] + struct #builder #builder_generics { + #( + #subsystem_name : ::std::option::Option< #builder_generic_ty >, + )* + #( + #baggage_name : ::std::option::Option< #baggage_ty >, + )* + spawner: ::std::option::Option< S >, + } + + impl #builder_generics Default for #builder #builder_generics { + fn default() -> Self { + Self { + #( + #subsystem_name: None, + )* + #( + #baggage_name: None, + )* + spawner: None, + } + } + } + + impl #builder_generics #builder #builder_generics #builder_where_clause { + #( + pub fn #subsystem_name (mut self, subsystem: #builder_generic_ty ) -> Self { + self. #subsystem_name = Some( subsystem ); + self + } + )* + + pub fn build(mut self) -> (#overseer_name #generics, #handler) + { + let (events_tx, events_rx) = ::polkadot_overseer_gen::metered::channel::< + #event + >(SIGNAL_CHANNEL_CAPACITY); + + let handler: #handler = events_tx.clone(); + + let (to_overseer_tx, to_overseer_rx) = ::polkadot_overseer_gen::metered::unbounded::< + ToOverseer + >(); + + #( + let (#channel_name_tx, #channel_name_rx) + : ( + ::polkadot_overseer_gen::metered::MeteredSender< MessagePacket< #consumes >>, + ::polkadot_overseer_gen::metered::MeteredReceiver< MessagePacket< #consumes >>, + ) = + ::polkadot_overseer_gen::metered::channel::< + MessagePacket< #consumes > + >(CHANNEL_CAPACITY); + )* + + #( + let (#channel_name_unbounded_tx, #channel_name_unbounded_rx): ( + ::polkadot_overseer_gen::metered::UnboundedMeteredSender< MessagePacket< #consumes >>, + ::polkadot_overseer_gen::metered::UnboundedMeteredReceiver< MessagePacket< #consumes >>, + ) = + ::polkadot_overseer_gen::metered::unbounded::< + MessagePacket< #consumes > + >(); + )* + + let channels_out = + ChannelsOut { + #( + #channel_name: #channel_name_tx, + )* + #( + #channel_name_unbounded: #channel_name_unbounded_tx, + )* + }; + + let spawner = self.spawner.expect("Spawner is set. qed"); + + let mut running_subsystems = ::polkadot_overseer_gen::FuturesUnordered::< + BoxFuture<'static, SubsystemResult<()>> + >::new(); + + #( + // FIXME generate a builder pattern that ensures this + let #subsystem_name = self. #subsystem_name .expect("All subsystem must exist with the builder pattern."); + + let message_rx: SubsystemIncomingMessages< #consumes > = ::polkadot_overseer_gen::select( + #channel_name_rx, #channel_name_unbounded_rx + ); + let (signal_tx, signal_rx) = ::polkadot_overseer_gen::metered::channel(SIGNAL_CHANNEL_CAPACITY); + let ctx = OverseerSubsystemContext::new( + signal_rx, + message_rx.clone(), + channels_out.clone(), + to_overseer_tx, + ); + + let #subsystem_name: OverseenSubsystem< #consumes > = + spawn::<_,_, #blocking, _, _>( + &mut spawner, + #channel_name_tx, + message_rx, + signal_tx, + channels_out.clone(), + to_overseer_tx, + #subsystem_name, + )?; + )* + + #( + let #baggage_name = self. #baggage_name .expect("Baggage must initialized"); + )* + + use ::polkadot_overseer_gen::StreamExt; + + let to_overseer_rx = to_overseer_rx.fuse(); + let overseer = #overseer_name { + #( + #subsystem_name, + )* + + #( + #baggage_name, + )* + + spawner, + running_subsystems, + events_rx, + to_overseer_rx, + }; + + (overseer, handler) + } + } + }; + eprintln!(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + {} +", +ts.to_string() +); + ts.extend(impl_task_kind(info)?); + Ok(ts) +} + +pub(crate) fn impl_task_kind( + info: &OverseerInfo, +) -> Result { + let signal = &info.extern_signal_ty; + + let ts = quote!{ + /// Task kind to launch. + pub trait TaskKind { + fn launch_task(spawner: &mut S, name: &'static str, future: BoxFuture<'static, ()>); + } + + struct Regular; + impl TaskKind for Regular { + fn launch_task(spawner: &mut S, name: &'static str, future: BoxFuture<'static, ()>) { + spawner.spawn(name, future) + } + } + + struct Blocking; + impl TaskKind for Blocking { + fn launch_task(spawner: &mut S, name: &'static str, future: BoxFuture<'static, ()>) { + spawner.spawn_blocking(name, future) + } + } + + /// Spawn task of kind `self` using spawner `S`. + pub fn spawn( + spawner: &mut S, + message_tx: ::polkadot_overseer_gen::metered::MeteredSender>, + // muxed incoming channels, bounded and unbouneded + message_rx: ::polkadot_overseer_gen::SubsystemIncomingMessages, + signal_tx: ::polkadot_overseer_gen::metered::MeteredSender< #signal >, + // meter for the unbounded channel + unbounded_meter: ::polkadot_overseer_gen::metered::Meter, + // connection to the subsystems + channels_out: ChannelsOut, + to_overseer_tx: ::polkadot_overseer_gen::metered::UnboundedMeteredSender, + ctx: Ctx, + s: impl ::polkadot_overseer_gen::Subsystem, + futures: &mut ::polkadot_overseer_gen::FuturesUnordered>>, + ) -> SubsystemResult> + where + S: ::polkadot_overseer_gen::SpawnNamed, + M: Send + 'static, + TK: TaskKind, + Ctx: ::polkadot_overseer_gen::SubsystemContext, + E: std::error::Error + Send + Sync + 'static + From, + { + let ::polkadot_overseer_gen::SpawnedSubsystem { future, name } = s.start(ctx); + + let (tx, rx) = ::polkadot_overseer_gen::oneshot::channel(); + + let fut = Box::pin(async move { + if let Err(e) = future.await { + ::polkadot_overseer_gen::tracing::error!(subsystem=name, err = ?e, "subsystem exited with error"); + } else { + ::polkadot_overseer_gen::tracing::debug!(subsystem=name, "subsystem exited without an error"); + } + let _ = tx.send(()); + }); + + ::launch_task(spawner, name, fut); + + futures.push(Box::pin(rx.map(|e| { tracing::warn!(err = ?e, "dropping error"); Ok(()) }))); + + let instance = Some(SubsystemInstance { + meters: ::polkadot_overseer_gen::SubsystemMeters { + unbounded: unbounded_meter, + bounded: message_tx.meter().clone(), + signals: signal_tx.meter().clone(), + }, + tx_signal: signal_tx, + tx_bounded: message_tx, + signals_received: 0, + name, + }); + + Ok(OverseenSubsystem { + instance, + }) + } + }; + + eprintln!(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + {} +", +ts.to_string() +); + + Ok(ts) +} diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index 011dba1ee14c..ce844f10f640 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -33,13 +33,20 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { pub fn dispatch_iter(event: #extern_event_ty) -> impl Iterator + Send { let mut iter = None.into_iter(); + use ::std::convert::TryFrom; + + // creates pretty errors when the inner variant + // does not impl `TryFrom< #extern_event_ty >` + fn dispatchable_message_impls_try_from_extern_event>() { + } + #( + dispatchable_message_impls_try_from_extern_event::< #dispatchable >(); + let mut iter = iter.chain( ::std::iter::once( - event.focus().ok().map(|event| { - #message_wrapper :: #dispatchable ( - #dispatchable :: from( event ) - ) + #dispatchable :: try_from( event ).ok().map(|event| { + #message_wrapper :: #dispatchable ( event ) }) ) ); @@ -48,6 +55,6 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { } } }; - + println!("{}", ts.to_string()); Ok(ts) } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs new file mode 100644 index 000000000000..cade566527f1 --- /dev/null +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -0,0 +1,225 @@ +use quote::quote; +use syn::{Ident, Result}; + +use super::*; + + +/// Implement a builder pattern for the `Overseer`-type, +/// which acts as the gateway to constructing the overseer. +pub(crate) fn impl_misc( + info: &OverseerInfo, +) -> Result { + let overseer_name = info.overseer_name.clone(); + let builder = Ident::new(&(overseer_name.to_string() + "Builder"), overseer_name.span()); + let handler = Ident::new(&(overseer_name.to_string() + "Handler"), overseer_name.span()); + + let subsystem_name = &info.subsystem_names(); + let builder_generic_ty = &info.builder_generic_types(); + + let channel_name = &info.channel_names(""); + let channel_name_unbounded = &info.channel_names("_unbounded"); + + let channel_name_tx = &info.channel_names("_tx"); + let channel_name_unbounded_tx = &info.channel_names("_unbounded_tx"); + + let channel_name_rx = &info.channel_names("_rx"); + let channel_name_unbounded_rx = &info.channel_names("_unbounded_rx"); + + let baggage_generic_ty = &info.baggage_generic_types(); + let baggage_name = &info.baggage_names(); + let baggage_ty = &info.baggage_types(); + + let blocking = &info.subsystems().iter().map(|x| { + if x.blocking { + quote! { Blocking } + } else { + quote! { Regular } + } + }).collect::>(); + + let consumes = &info.consumes(); + let message_wrapper = &info.message_wrapper; + let event = &info.extern_event_ty; + let signal = &info.extern_signal_ty; + + let generics = quote! { + < S, #( #baggage_generic_ty, )* > + }; + let where_clause = quote! { + where + S: ::polkadot_overseer_gen::SpawnNamed, + }; + + let builder_generics = quote! { + + }; + + // all subsystems must have the same context + // even if the overseer does not impose such a limit. + let builder_additional_generics = quote! { + < Ctx, #( #builder_generic_ty, )* > + }; + + let error_ty = &info.extern_error_ty; + + let builder_where_clause = quote! { + where + Ctx: SubsystemContext< #message_wrapper > + S: ::polkadot_overseer_gen::SpawnNamed, + #( #builder_generic_ty : Subsystem, )* + }; + + let ts = quote! { + //////////////////////////////////////////////////// + // `OverseerSubsystemSender` + + #[derive(Debug, Clone)] + pub struct OverseerSubsystemSender { + channels: ChannelsOut, + signals_received: SignalsReceived, + } + + #[::polkadot_overseer_gen::async_trait] + impl SubsystemSender< #message_wrapper > for OverseerSubsystemSender { + async fn send_message(&mut self, msg: #message_wrapper) { + self.channels.send_and_log_error(self.signals_received.load(), msg).await; + } + + async fn send_messages(&mut self, msgs: T) + where T: IntoIterator + Send, T::IntoIter: Send + { + // This can definitely be optimized if necessary. + for msg in msgs { + self.send_message(msg).await; + } + } + + fn send_unbounded_message(&mut self, msg: #message_wrapper) { + self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); + } + } + + /// A context type that is given to the [`Subsystem`] upon spawning. + /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s + /// or to spawn it's [`SubsystemJob`]s. + /// + /// [`Overseer`]: struct.Overseer.html + /// [`Subsystem`]: trait.Subsystem.html + /// [`SubsystemJob`]: trait.SubsystemJob.html + #[derive(Debug)] + pub struct OverseerSubsystemContext{ + signals: metered::MeteredReceiver< #signal >, + messages: SubsystemIncomingMessages, + to_subsystems: OverseerSubsystemSender, + to_overseer: metered::UnboundedMeteredSender, + signals_received: SignalsReceived, + pending_incoming: Option<(usize, M)>, + } + + impl OverseerSubsystemContext { + /// Create a new `OverseerSubsystemContext`. + fn new( + signals: metered::MeteredReceiver< #signal >, + messages: SubsystemIncomingMessages, + to_subsystems: ChannelsOut, + to_overseer: metered::UnboundedMeteredSender, + ) -> Self { + let signals_received = SignalsReceived::default(); + OverseerSubsystemContext { + signals, + messages, + to_subsystems: OverseerSubsystemSender { + channels: to_subsystems, + signals_received: signals_received.clone(), + }, + to_overseer, + signals_received, + pending_incoming: None, + } + } + } + + #[::polkadot_overseer_gen::async_trait] + impl SubsystemContext for OverseerSubsystemContext { + type Message = M; + type Sender = OverseerSubsystemSender; + + async fn try_recv(&mut self) -> Result>, ()> { + match ::polkadot_overseer_gen::poll!(self.recv()) { + ::polkadot_overseer_gen::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), + ::polkadot_overseer_gen::Poll::Pending => Ok(None), + } + } + + async fn recv(&mut self) -> SubsystemResult> { + loop { + // If we have a message pending an overseer signal, we only poll for signals + // in the meantime. + if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { + if needs_signals_received <= self.signals_received.load() { + return Ok(FromOverseer::Communication { msg }); + } else { + self.pending_incoming = Some((needs_signals_received, msg)); + + // wait for next signal. + let signal = self.signals.next().await + .ok_or(SubsystemError::Context( + "Signal channel is terminated and empty." + .to_owned() + ))?; + + self.signals_received.inc(); + return Ok(FromOverseer::Signal(signal)) + } + } + + let mut await_message = self.messages.next().fuse(); + let mut await_signal = self.signals.next().fuse(); + let signals_received = self.signals_received.load(); + let pending_incoming = &mut self.pending_incoming; + + // Otherwise, wait for the next signal or incoming message. + let from_overseer = futures::select_biased! { + signal = await_signal => { + let signal = signal + .ok_or(SubsystemError::Context( + "Signal channel is terminated and empty." + .to_owned() + ))?; + + FromOverseer::Signal(signal) + } + msg = await_message => { + let packet = msg + .ok_or(SubsystemError::Context( + "Message channel is terminated and empty." + .to_owned() + ))?; + + if packet.signals_received > signals_received { + // wait until we've received enough signals to return this message. + *pending_incoming = Some((packet.signals_received, packet.message)); + continue; + } else { + // we know enough to return this message. + FromOverseer::Communication { msg: packet.message} + } + } + }; + + if let FromOverseer::Signal(_) = from_overseer { + self.signals_received.inc(); + } + + return Ok(from_overseer); + } + } + + fn sender(&mut self) -> &mut Self::Sender { + &mut self.to_subsystems + } + } + }; + + Ok(ts) +} diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 50dbba3a4cac..2666335bb389 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -101,263 +101,9 @@ pub(crate) fn impl_overseer_struct( } }; - ts.extend(impl_builder(info)?); - ts.extend(impl_overseen_subsystem(info)?); Ok(ts) } -/// Implement a builder pattern for the `Overseer`-type, -/// which acts as the gateway to constructing the overseer. -pub(crate) fn impl_builder( - info: &OverseerInfo, -) -> Result { - let overseer_name = info.overseer_name.clone(); - let builder = Ident::new(&(overseer_name.to_string() + "Builder"), overseer_name.span()); - let handler = Ident::new(&(overseer_name.to_string() + "Handler"), overseer_name.span()); - - let subsystem_name = &info.subsystem_names(); - let builder_generic_ty = &info.builder_generic_types(); - - let channel_name = &info.channel_names(""); - let channel_name_unbounded = &info.channel_names("_unbounded"); - - let channel_name_tx = &info.channel_names("_tx"); - let channel_name_unbounded_tx = &info.channel_names("_unbounded_tx"); - - let channel_name_rx = &info.channel_names("_rx"); - let channel_name_unbounded_rx = &info.channel_names("_unbounded_rx"); - - let baggage_generic_ty = &info.baggage_generic_types(); - let baggage_name = &info.baggage_names(); - let baggage_ty = &info.baggage_types(); - - let blocking = &info.subsystems().iter().map(|x| x.blocking).collect::>(); - - let generics = quote! { - < S, #( #baggage_generic_ty, )* > - }; - let where_clause = quote! { - where - S: ::polkadot_overseer_gen::SpawnNamed, - }; - - let builder_generics = quote! { - - }; - - // all subsystems must have the same context - // even if the overseer does not impose such a limit. - let builder_additional_generics = quote! { - < Ctx, #( #builder_generic_ty, )* > - }; - - let error_ty = &info.extern_error_ty; - - let builder_where_clause = quote! { - where - Ctx: SubsystemContext, - S: ::polkadot_overseer_gen::SpawnNamed, - #( #builder_generic_ty : Subsystem, )* - }; - - let consumes = &info.consumes(); - let message_wrapper = &info.message_wrapper; - let event = &info.extern_event_ty; - let signal = &info.extern_signal_ty; - - let ts = quote! { - - impl #generics #overseer_name #generics #where_clause { - fn builder #builder_additional_generics () -> #builder #builder_generics - #builder_where_clause - { - #builder :: default() - } - } - - - pub type #handler = ::polkadot_overseer_gen::metered::MeteredSender< #event >; - - #[derive(Debug, Clone)] - struct #builder #builder_generics { - #( - #subsystem_name : ::std::option::Option< #builder_generic_ty >, - )* - #( - #baggage_name : ::std::option::Option< #baggage_ty >, - )* - spawner: ::std::option::Option< S >, - _phantom_ctx: ::std::marker::PhantomData< Ctx >, - } - - impl #builder_generics Default for #builder #builder_generics { - fn default() -> Self { - Self { - #( - #subsystem_name: None, - )* - #( - #baggage_name: None, - )* - spawner: None, - _phantom_ctx: ::std::marker::PhantomData, - } - } - } - - impl #builder_generics #builder #builder_generics #builder_where_clause { - #( - pub fn #subsystem_name (mut self, subsystem: #builder_generic_ty ) -> Self { - self. #subsystem_name = Some( subsystem ); - self - } - )* - - pub fn build(mut self, create_subsystem_ctx: F) -> (#overseer_name #generics, #handler) - where - F: FnMut( - ::polkadot_overseer_gen::metered::MeteredReceiver< #signal >, - SubsystemIncomingMessages< #message_wrapper >, - ChannelsOut, - ::polkadot_overseer_gen::metered::UnboundedMeteredSender, - ) -> Ctx, - { - - let (events_tx, events_rx) = ::polkadot_overseer_gen::metered::channel::< - #event - >(SIGNAL_CHANNEL_CAPACITY); - - let handler: #handler = events_tx.clone(); - - let (to_overseer_tx, to_overseer_rx) = ::polkadot_overseer_gen::metered::unbounded::< - ToOverseer - >(); - - let channels_out = { - #( - let (#channel_name_tx, #channel_name_rx) = - ::polkadot_overseer_gen::metered::channel::< - MessagePacket< #consumes > - >(CHANNEL_CAPACITY); - )* - - #( - let (#channel_name_unbounded_tx, #channel_name_unbounded_rx) = - ::polkadot_overseer_gen::metered::unbounded::< - MessagePacket< #consumes > - >(); - )* - - ChannelsOut { - #( - #channel_name: #channel_name_tx, - )* - #( - #channel_name_unbounded: #channel_name_unbounded_tx, - )* - } - }; - - let spawner = self.spawner.expect("Spawner is set. qed"); - - let mut running_subsystems = ::polkadot_overseer_gen::FuturesUnordered::< - BoxFuture<'static, SubsystemResult<()>> - >::new(); - - #( - // FIXME generate a builder pattern that ensures this - let #subsystem_name = self. #subsystem_name .expect("All subsystem must exist with the builder pattern."); - - let #subsystem_name: OverseenSubsystem< #consumes > = { - - let unbounded_meter = channels_out. #channel_name .meter().clone(); - - let message_tx: ::polkadot_overseer_gen::metered::MeteredSender< MessagePacket< #consumes > > - = channels_out. #channel_name .clone(); - - let message_rx: SubsystemIncomingMessages< #message_wrapper > = - ::polkadot_overseer_gen::select( - channels_out. #channel_name .clone(), - channels_out. #channel_name_unbounded .clone(), - ); - - let (signal_tx, signal_rx) = ::polkadot_overseer_gen::metered::channel::< #signal >(SIGNAL_CHANNEL_CAPACITY); - - let ctx = create_subsystem_ctx( - signal_rx, - message_rx, - channels_out.clone(), - to_overseer_tx.clone(), - ); - - let ::polkadot_overseer_gen::SpawnedSubsystem { future, name } = #subsystem_name .start(ctx); - - let (terminated_tx, terminated_rx) = ::polkadot_overseer_gen::oneshot::channel(); - - let fut = Box::pin(async move { - if let Err(e) = future.await { - } else { - } - let _ = terminated_tx.send(()); - }); - - if #blocking { - spawner.spawn_blocking(name, fut); - } else { - spawner.spawn(name, fut); - } - - running_subsystems.push(Box::pin(terminated_rx.map(|e| { - Ok(()) - }))); - - let instance = Some( - SubsystemInstance::< #consumes > { - meters: SubsystemMeters { - unbounded: unbounded_meter, - bounded: message_tx.meter().clone(), - signals: signal_tx.meter().clone(), - }, - tx_signal: signal_tx, - tx_bounded: message_tx, - signals_received: 0, - name, - } - ); - - OverseenSubsystem::< #consumes > { - instance, - } - }; - )* - - #( - let #baggage_name = self. #baggage_name .expect("Baggage must initialized"); - )* - - let overseer = #overseer_name { - #( - #subsystem_name, - )* - - #( - #baggage_name, - )* - - spawner, - running_subsystems, - events_rx, - to_overseer_rx, - }; - - (overseer, handler) - } - } - }; - Ok(ts) -} - - pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 97d696759ef1..119f40bb0fe0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -22,15 +22,18 @@ use std::collections::HashSet; mod parse_struct; mod parse_attr; mod impl_overseer; +mod impl_misc; +mod impl_builder; mod impl_replace; mod impl_channels_out; mod impl_message_wrapper; mod impl_dispatch; -// mod inc; use parse_struct::*; use parse_attr::*; use impl_overseer::*; +use impl_builder::*; +use impl_misc::*; use impl_replace::*; use impl_channels_out::*; use impl_message_wrapper::*; @@ -62,8 +65,13 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< }; let mut additive = impl_overseer_struct(&info)?; - additive.extend(impl_message_wrapper_enum(&info)?); + additive.extend(impl_builder(&info)?); + + additive.extend(impl_overseen_subsystem(&info)?); additive.extend(impl_channels_out_struct(&info)?); + additive.extend(impl_misc(&info)?); + + additive.extend(impl_message_wrapper_enum(&info)?); additive.extend(impl_dispatch(&info)?); Ok(additive) diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index f0267226e49c..32f07f31cc64 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -70,11 +70,12 @@ pub use sp_core::traits::SpawnNamed; #[doc(hidden)] pub use futures::{ select, + poll, future::{ Fuse, Future, BoxFuture }, stream::{ - select, FuturesUnordered + select, FuturesUnordered, }, channel::{mpsc, oneshot}, }; @@ -86,8 +87,12 @@ pub use std::time::Duration; use std::sync::atomic::{self, AtomicUsize}; use std::sync::Arc; -// pub use tokio_util::time; +#[doc(hidden)] pub use futures_timer::Delay; +#[doc(hidden)] +pub use futures_util::stream::StreamExt; + + /// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. /// /// Used to launch jobs. @@ -404,8 +409,8 @@ pub trait SubsystemSender: Send + Clone + 'static { -use futures::task::{Poll, Context}; -use std::pin::Pin; +pub use futures::task::{Poll, Context}; +pub use std::pin::Pin; /// A future that wraps another future with a `Delay` allowing for time-limited futures. #[pin_project::pin_project] @@ -453,5 +458,9 @@ impl Future for Timeout { + + + + #[cfg(test)] mod tests; diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index e941819726b8..22e9b4d0b115 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -80,6 +80,7 @@ use client::{BlockImportNotification, BlockchainEvents, FinalityNotification}; use sp_api::{ApiExt, ProvideRuntimeApi}; use polkadot_subsystem::messages::{ + NetworkBridgeMessage, CandidateValidationMessage, CandidateBackingMessage, CandidateSelectionMessage, ChainApiMessage, StatementDistributionMessage, AvailabilityDistributionMessage, BitfieldSigningMessage, BitfieldDistributionMessage, @@ -322,162 +323,26 @@ impl Debug for ToOverseer { } } } -/// A context type that is given to the [`Subsystem`] upon spawning. -/// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s -/// or to spawn it's [`SubsystemJob`]s. -/// -/// [`Overseer`]: struct.Overseer.html -/// [`Subsystem`]: trait.Subsystem.html -/// [`SubsystemJob`]: trait.SubsystemJob.html -#[derive(Debug)] -pub struct OverseerSubsystemContext{ - signals: metered::MeteredReceiver, - messages: SubsystemIncomingMessages, - to_subsystems: OverseerSubsystemSender, - to_overseer: metered::UnboundedMeteredSender, - signals_received: SignalsReceived, - pending_incoming: Option<(usize, M)>, - metrics: Metrics, -} - -impl OverseerSubsystemContext { - /// Create a new `OverseerSubsystemContext`. - fn new( - signals: metered::MeteredReceiver, - messages: SubsystemIncomingMessages, - to_subsystems: ChannelsOut, - to_overseer: metered::UnboundedMeteredSender, - metrics: Metrics, - ) -> Self { - let signals_received = SignalsReceived::default(); - OverseerSubsystemContext { - signals, - messages, - to_subsystems: OverseerSubsystemSender { - channels: to_subsystems, - signals_received: signals_received.clone(), - }, - to_overseer, - signals_received, - pending_incoming: None, - metrics, - } - } - - /// Create a new `OverseerSubsystemContext` with no metering. - /// - /// Intended for tests. - #[allow(unused)] - fn new_unmetered( - signals: metered::MeteredReceiver, - messages: SubsystemIncomingMessages, - to_subsystems: ChannelsOut, - to_overseer: metered::UnboundedMeteredSender, - ) -> Self { - let metrics = Metrics::default(); - OverseerSubsystemContext::new(signals, messages, to_subsystems, to_overseer, metrics) - } -} - -#[async_trait::async_trait] -impl SubsystemContext for OverseerSubsystemContext { - type Message = M; - type Signal = Signal; - type Sender = OverseerSubsystemSender; - - async fn try_recv(&mut self) -> Result>, ()> { - match poll!(self.recv()) { - Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), - Poll::Pending => Ok(None), - } - } - - async fn recv(&mut self) -> SubsystemResult> { - loop { - // If we have a message pending an overseer signal, we only poll for signals - // in the meantime. - if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { - if needs_signals_received <= self.signals_received.load() { - return Ok(FromOverseer::Communication { msg }); - } else { - self.pending_incoming = Some((needs_signals_received, msg)); - - // wait for next signal. - let signal = self.signals.next().await - .ok_or(SubsystemError::Context( - "Signal channel is terminated and empty." - .to_owned() - ))?; - - self.signals_received.inc(); - return Ok(FromOverseer::Signal(signal)) - } - } - - let mut await_message = self.messages.next().fuse(); - let mut await_signal = self.signals.next().fuse(); - let signals_received = self.signals_received.load(); - let pending_incoming = &mut self.pending_incoming; - - // Otherwise, wait for the next signal or incoming message. - let from_overseer = futures::select_biased! { - signal = await_signal => { - let signal = signal - .ok_or(SubsystemError::Context( - "Signal channel is terminated and empty." - .to_owned() - ))?; - - FromOverseer::Signal(signal) - } - msg = await_message => { - let packet = msg - .ok_or(SubsystemError::Context( - "Message channel is terminated and empty." - .to_owned() - ))?; - - if packet.signals_received > signals_received { - // wait until we've received enough signals to return this message. - *pending_incoming = Some((packet.signals_received, packet.message)); - continue; - } else { - // we know enough to return this message. - FromOverseer::Communication { msg: packet.message} - } - } - }; - if let FromOverseer::Signal(_) = from_overseer { - self.signals_received.inc(); - } - return Ok(from_overseer); - } - } +// async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) +// -> SubsystemResult<()> +// { +// self.sender().send(ToOverseer::SpawnJob { +// name, +// s, +// }).await.map_err(Into::into) +// } - async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) - -> SubsystemResult<()> - { - self.to_overseer.send(ToOverseer::SpawnJob { - name, - s, - }).await.map_err(Into::into) - } - - async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) - -> SubsystemResult<()> - { - self.to_overseer.send(ToOverseer::SpawnBlockingJob { - name, - s, - }).await.map_err(Into::into) - } +// async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) +// -> SubsystemResult<()> +// { +// self.sender().send(ToOverseer::SpawnBlockingJob { +// name, +// s, +// }).await.map_err(Into::into) +// } - fn sender(&mut self) -> &mut Self::Sender { - &mut self.to_subsystems - } -} /// The `Overseer` itself. #[overlord(gen=AllMessages, event=Event, signal=OverseerSignal, error=SubsystemError)] @@ -966,32 +831,6 @@ where } -#[derive(Debug, Clone)] -pub struct OverseerSubsystemSender { - channels: ChannelsOut, - signals_received: SignalsReceived, -} - -#[::polkadot_overseer_gen::async_trait] -impl SubsystemSender for OverseerSubsystemSender { - async fn send_message(&mut self, msg: AllMessages) { - self.channels.send_and_log_error(self.signals_received.load(), msg).await; - } - - async fn send_messages(&mut self, msgs: AllMessages) - where T: IntoIterator + Send, T::IntoIter: Send - { - // This can definitely be optimized if necessary. - for msg in msgs { - self.send_message(msg).await; - } - } - - fn send_unbounded_message(&mut self, msg: AllMessages) { - self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); - } -} - #[cfg(test)] mod tests; From 78e7c717c076e8e9536be3e0304f6a336140faff Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 26 May 2021 14:26:33 +0200 Subject: [PATCH 042/161] =?UTF-8?q?Hot=20like=20=E2=84=83=20=20or=20?= =?UTF-8?q?=E2=84=AA=20=3F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- node/overseer/overseer-gen/examples/dummy.rs | 16 ++++-- .../proc-macro/src/impl_builder.rs | 16 +++--- .../proc-macro/src/impl_channels_out.rs | 2 +- .../overseer-gen/proc-macro/src/impl_misc.rs | 42 +++++++++++++--- .../proc-macro/src/impl_overseer.rs | 2 +- node/overseer/overseer-gen/src/lib.rs | 50 ++++++++++++++----- 6 files changed, 95 insertions(+), 33 deletions(-) diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index a46281bb12fe..9763f904401e 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -9,6 +9,12 @@ use polkadot_subsystem::messages::NetworkBridgeEvent; #[derive(Default)] struct AwesomeSubSys; +impl ::polkadot_overseer_gen::Subsystem, Yikes> for AwesomeSubSys { + fn start(self, ctx: OverseerSubsystemContext) -> SpawnedSubsystem < Yikes > { + unimplemented!("starting yay!") + } +} + #[derive(Debug, Clone)] struct SigSigSig; @@ -58,8 +64,10 @@ struct Xxx { #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, - #[subsystem(no_dispatch, Plinko)] - plinkos: AwesomeSubSys, + // #[subsystem(no_dispatch, Plinko)] + // plinkos: AwesomeSubSys, + + // i_like_pi: f64, } #[derive(Debug, Clone)] @@ -71,8 +79,8 @@ struct DummyCtx; fn main() { let overseer = Xxx::builder() .sub0(AwesomeSubSys::default()) - .plinkos(AwesomeSubSys::default()) - .i_like_pie(std::f64::consts::PI) + // .plinkos(AwesomeSubSys::default()) + // .i_like_pie(std::f64::consts::PI) .spawner(DummySpawner) .build(|| -> DummyCtx { DummyCtx } ); } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 29da343f268a..1e4b407b4526 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -70,7 +70,7 @@ pub(crate) fn impl_builder( let mut ts = quote! { impl #generics #overseer_name #generics #where_clause { - fn builder #builder_additional_generics () -> #builder #builder_generics + pub fn builder #builder_additional_generics () -> #builder #builder_generics #builder_where_clause { #builder :: default() @@ -172,13 +172,13 @@ pub(crate) fn impl_builder( let (signal_tx, signal_rx) = ::polkadot_overseer_gen::metered::channel(SIGNAL_CHANNEL_CAPACITY); let ctx = OverseerSubsystemContext::new( signal_rx, - message_rx.clone(), + message_rx, channels_out.clone(), to_overseer_tx, ); let #subsystem_name: OverseenSubsystem< #consumes > = - spawn::<_,_, #blocking, _, _>( + spawn::<_,_, #blocking, _, _, _>( &mut spawner, #channel_name_tx, message_rx, @@ -186,7 +186,7 @@ pub(crate) fn impl_builder( channels_out.clone(), to_overseer_tx, #subsystem_name, - )?; + ).unwrap(); // FIXME )* #( @@ -230,6 +230,9 @@ pub(crate) fn impl_task_kind( let signal = &info.extern_signal_ty; let ts = quote!{ + + use ::polkadot_overseer_gen::FutureExt; + /// Task kind to launch. pub trait TaskKind { fn launch_task(spawner: &mut S, name: &'static str, future: BoxFuture<'static, ()>); @@ -250,7 +253,7 @@ pub(crate) fn impl_task_kind( } /// Spawn task of kind `self` using spawner `S`. - pub fn spawn( + pub fn spawn( spawner: &mut S, message_tx: ::polkadot_overseer_gen::metered::MeteredSender>, // muxed incoming channels, bounded and unbouneded @@ -262,7 +265,7 @@ pub(crate) fn impl_task_kind( channels_out: ChannelsOut, to_overseer_tx: ::polkadot_overseer_gen::metered::UnboundedMeteredSender, ctx: Ctx, - s: impl ::polkadot_overseer_gen::Subsystem, + s: SubSys, futures: &mut ::polkadot_overseer_gen::FuturesUnordered>>, ) -> SubsystemResult> where @@ -271,6 +274,7 @@ pub(crate) fn impl_task_kind( TK: TaskKind, Ctx: ::polkadot_overseer_gen::SubsystemContext, E: std::error::Error + Send + Sync + 'static + From, + SubSys: ::polkadot_overseer_gen::Subsystem, { let ::polkadot_overseer_gen::SpawnedSubsystem { future, name } = s.start(ctx); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index fdc542ffae06..4b2e57a8d095 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -41,7 +41,7 @@ pub(crate) fn impl_channels_out_struct( signals_received: usize, message: #message_wrapper, ) { - let res = match message { + let res: ::std::result::Result<_, _> = match message { #( #message_wrapper :: #consumes ( inner ) => { self. #channel_name .send( diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index cade566527f1..a24b31dc4f79 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -70,7 +70,7 @@ pub(crate) fn impl_misc( }; let ts = quote! { - //////////////////////////////////////////////////// + // ////////////////////////////////////////////////// // `OverseerSubsystemSender` #[derive(Debug, Clone)] @@ -79,14 +79,15 @@ pub(crate) fn impl_misc( signals_received: SignalsReceived, } + #( #[::polkadot_overseer_gen::async_trait] - impl SubsystemSender< #message_wrapper > for OverseerSubsystemSender { - async fn send_message(&mut self, msg: #message_wrapper) { - self.channels.send_and_log_error(self.signals_received.load(), msg).await; + impl SubsystemSender< #consumes > for OverseerSubsystemSender { + async fn send_message(&mut self, msg: #consumes) { + self.channels.send_and_log_error(self.signals_received.load(), msg.into()).await; } async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send + where T: IntoIterator + Send, T::IntoIter: Send { // This can definitely be optimized if necessary. for msg in msgs { @@ -94,10 +95,11 @@ pub(crate) fn impl_misc( } } - fn send_unbounded_message(&mut self, msg: #message_wrapper) { - self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); + fn send_unbounded_message(&mut self, msg: #consumes) { + self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg.into()); } } + )* /// A context type that is given to the [`Subsystem`] upon spawning. /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s @@ -140,8 +142,12 @@ pub(crate) fn impl_misc( } #[::polkadot_overseer_gen::async_trait] - impl SubsystemContext for OverseerSubsystemContext { + impl SubsystemContext for OverseerSubsystemContext + where + OverseerSubsystemSender: polkadot_overseer_gen::SubsystemSender + { type Message = M; + type Signal = #signal; type Sender = OverseerSubsystemSender; async fn try_recv(&mut self) -> Result>, ()> { @@ -218,6 +224,26 @@ pub(crate) fn impl_misc( fn sender(&mut self) -> &mut Self::Sender { &mut self.to_subsystems } + + #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn(ctx, name, fut)")] + async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) + -> SubsystemResult<()> + { + self.to_overseer.send(ToOverseer::SpawnJob { + name, + s, + }).await.map_err(Into::into) + } + + #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn_blocking(ctx, name, fut)")] + async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) + -> SubsystemResult<()> + { + self.to_overseer.send(ToOverseer::SpawnBlockingJob { + name, + s, + }).await.map_err(Into::into) + } } }; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 2666335bb389..517088169ca5 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -72,7 +72,7 @@ pub(crate) fn impl_overseer_struct( >, /// Gather running subsystems' outbound streams into one. - to_overseer_rx: ::polkadot_overseer_gen::Fuse< + to_overseer_rx: ::polkadot_overseer_gen::stream::Fuse< metered::UnboundedMeteredReceiver< ToOverseer > >, diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 32f07f31cc64..11db02ef4402 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -75,7 +75,7 @@ pub use futures::{ Fuse, Future, BoxFuture }, stream::{ - select, FuturesUnordered, + self, select, FuturesUnordered, }, channel::{mpsc, oneshot}, }; @@ -91,6 +91,8 @@ use std::sync::Arc; pub use futures_timer::Delay; #[doc(hidden)] pub use futures_util::stream::StreamExt; +#[doc(hidden)] +pub use futures_util::future::FutureExt; /// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. @@ -117,6 +119,18 @@ pub enum ToOverseer { }, } +use std::fmt; + +impl fmt::Debug for ToOverseer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SpawnJob{ name, .. } => writeln!(f, "SpawnJob{{ {}, ..}}", name), + Self::SpawnBlockingJob{ name, .. } => writeln!(f, "SpawnBlockingJob{{ {}, ..}}", name), + } + } + +} + /// A helper trait to map a subsystem to smth. else. @@ -140,8 +154,14 @@ impl MapSubsystem for F where F: Fn(T) -> U { // FIXME XXX elaborate the purpose of this. #[derive(Debug)] pub struct MessagePacket { - signals_received: usize, - message: T, + /// Signal level at the point of reception. + /// + /// Required to assure signals were consumed _before_ + /// consuming messages that are based on the assumption + /// that a certain signal was assumed. + pub signals_received: usize, + /// The message to be sent/consumed. + pub message: T, } /// Create a packet from its parts. @@ -153,7 +173,7 @@ pub fn make_packet(signals_received: usize, message: T) -> MessagePacket { } /// Incoming messages from both the bounded and unbounded channel. -pub type SubsystemIncomingMessages = ::futures::stream::Select< +pub type SubsystemIncomingMessages = self::stream::Select< self::metered::MeteredReceiver>, self::metered::UnboundedMeteredReceiver>, >; @@ -334,7 +354,7 @@ pub trait SubsystemContext: Send + 'static { /// And the same for signals. type Signal: Send + 'static; /// The sender type as provided by `sender()` and underlying. - type Sender: SubsystemSender; + type Sender: SubsystemSender + std::fmt::Debug + Clone + Send; /// Try to asynchronously receive a message. /// @@ -356,14 +376,19 @@ pub trait SubsystemContext: Send + 'static { ) -> SubsystemResult<()>; /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: Self::Message); + async fn send_message(&mut self, msg: Self::Message) { + self.sender().send_message(msg).await + } /// Send multiple direct messages to other `Subsystem`s, routed based on message type. async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send; + where T: IntoIterator + Send, T::IntoIter: Send + { + self.sender().send_messages(msgs).await + } /// Obtain the sender. - fn sender(&self) -> Self::Sender; + fn sender(&mut self) -> &mut Self::Sender; } /// A trait that describes the [`Subsystem`]s that can run on the [`Overseer`]. @@ -386,23 +411,22 @@ where /// TODO FIXME #[async_trait::async_trait] -pub trait SubsystemSender: Send + Clone + 'static { +pub trait SubsystemSender: Send + Clone + 'static { // Inner message type. - // type Message: Send + Clone + 'static; /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: AllMessages); + async fn send_message(&mut self, msg: M); /// Send multiple direct messages to other `Subsystem`s, routed based on message type. async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send; + where T: IntoIterator + Send, T::IntoIter: Send; /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message /// type. /// /// This function should be used only when there is some other bounding factor on the messages /// sent with it. Otherwise, it risks a memory leak. - fn send_unbounded_message(&mut self, msg: AllMessages); + fn send_unbounded_message(&mut self, msg: M); } From 3709117604ff2eaab60407c6ad33082e7ca0474a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 26 May 2021 15:09:08 +0200 Subject: [PATCH 043/161] fixins --- Cargo.lock | 33 ++++++++++++++++++- node/overseer/overseer-gen/examples/dummy.rs | 12 ++++++- .../proc-macro/src/impl_builder.rs | 7 +++- .../proc-macro/src/impl_channels_out.rs | 7 ++-- 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a6040e02a77..f52f4ad6156b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2511,7 +2511,7 @@ dependencies = [ "log", "slab", "tokio 0.2.21", - "tokio-util", + "tokio-util 0.3.1", ] [[package]] @@ -6172,6 +6172,7 @@ dependencies = [ "polkadot-node-network-protocol", "polkadot-node-primitives", "polkadot-node-subsystem-test-helpers", + "polkadot-overseer-gen", "polkadot-primitives", "polkadot-procmacro-subsystem-dispatch-gen", "polkadot-statement-table", @@ -6225,6 +6226,7 @@ dependencies = [ "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", + "polkadot-overseer-gen", "polkadot-primitives", "rand 0.8.3", "sc-network", @@ -6266,10 +6268,14 @@ dependencies = [ "async-trait", "futures 0.3.14", "futures-timer 3.0.2", + "futures-util", "metered-channel", + "pin-project 1.0.4", + "polkadot-node-subsystem", "polkadot-overseer-gen-proc-macro", "sp-core", "thiserror", + "tokio-util 0.6.7", "tracing", "trybuild", ] @@ -10399,6 +10405,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "tokio" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37" +dependencies = [ + "autocfg", + "pin-project-lite 0.2.4", +] + [[package]] name = "tokio-buf" version = "0.1.1" @@ -10627,6 +10643,21 @@ dependencies = [ "tokio 0.2.21", ] +[[package]] +name = "tokio-util" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" +dependencies = [ + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.4", + "slab", + "tokio 1.6.0", +] + [[package]] name = "toml" version = "0.5.6" diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index 9763f904401e..e4ec460f69f8 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -73,6 +73,16 @@ struct Xxx { #[derive(Debug, Clone)] struct DummySpawner; +impl SpawnNamed for DummySpawner{ + fn spawn_blocking(&self, name: &'static str, future: futures::future::BoxFuture<'static, ()>) { + unimplemented!("spawn blocking") + } + + fn spawn(&self, name: &'static str, future: futures::future::BoxFuture<'static, ()>) { + unimplemented!("spawn") + } +} + #[derive(Debug, Clone)] struct DummyCtx; @@ -82,5 +92,5 @@ fn main() { // .plinkos(AwesomeSubSys::default()) // .i_like_pie(std::f64::consts::PI) .spawner(DummySpawner) - .build(|| -> DummyCtx { DummyCtx } ); + .build(); } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 1e4b407b4526..343460a809a8 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -77,7 +77,6 @@ pub(crate) fn impl_builder( } } - pub type #handler = ::polkadot_overseer_gen::metered::MeteredSender< #event >; #[derive(Debug, Clone)] @@ -106,6 +105,12 @@ pub(crate) fn impl_builder( } impl #builder_generics #builder #builder_generics #builder_where_clause { + /// The spawner to use for spawning tasks. + pub fn spawner(mut self, spawner: S) -> Self where S: ::polkadot_overseer_gen::SpawnNamed + Send { + self.spawner = Some(spawner); + self + } + #( pub fn #subsystem_name (mut self, subsystem: #builder_generic_ty ) -> Self { self. #subsystem_name = Some( subsystem ); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 4b2e57a8d095..8c93f5f75b26 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -65,12 +65,15 @@ pub(crate) fn impl_channels_out_struct( signals_received: usize, message: AllMessages, ) { - let res = match message { + use ::std::sync::mpsc::TrySendError; + + let res: ::std::result::Result<_, _> = match message { #( #message_wrapper :: #consumes (inner) => { - self. #channel_name_unbounded .send( + self. #channel_name_unbounded .unbounded_send( make_packet(signals_received, inner) ) + .map_err(|e| e.into_send_error()) }, )* }; From 0f772893b075204acce527e57bd51cfeb3be661f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 26 May 2021 16:02:04 +0200 Subject: [PATCH 044/161] dummy example works --- node/overseer/overseer-gen/examples/dummy.rs | 15 +++++---- .../proc-macro/src/impl_builder.rs | 32 ++++++++----------- .../proc-macro/src/impl_channels_out.rs | 2 +- .../proc-macro/src/impl_message_wrapper.rs | 2 +- 4 files changed, 24 insertions(+), 27 deletions(-) diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index e4ec460f69f8..ca2ac1ed1698 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -6,8 +6,9 @@ use polkadot_overseer_gen::*; use polkadot_subsystem::messages::NetworkBridgeEvent; +/// Concrete subsystem implementation for `MsgStrukt` msg type. #[derive(Default)] -struct AwesomeSubSys; +pub struct AwesomeSubSys; impl ::polkadot_overseer_gen::Subsystem, Yikes> for AwesomeSubSys { fn start(self, ctx: OverseerSubsystemContext) -> SpawnedSubsystem < Yikes > { @@ -15,12 +16,14 @@ impl ::polkadot_overseer_gen::Subsystem, Yik } } +/// A signal sent by the overseer. #[derive(Debug, Clone)] -struct SigSigSig; +pub struct SigSigSig; +/// The external event. #[derive(Debug, Clone)] -struct EvX; +pub struct EvX; impl EvX { @@ -30,7 +33,7 @@ impl EvX { } #[derive(Debug, Clone, Copy)] -struct Yikes; +pub struct Yikes; impl std::fmt::Display for Yikes { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -47,10 +50,10 @@ impl From for Yikes { } #[derive(Debug, Clone)] -struct MsgStrukt(u8); +pub struct MsgStrukt(u8); #[derive(Debug, Clone, Copy)] -struct Plinko; +pub struct Plinko; impl From for MsgStrukt { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 343460a809a8..ee3bd5ee5785 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -80,7 +80,7 @@ pub(crate) fn impl_builder( pub type #handler = ::polkadot_overseer_gen::metered::MeteredSender< #event >; #[derive(Debug, Clone)] - struct #builder #builder_generics { + pub struct #builder #builder_generics { #( #subsystem_name : ::std::option::Option< #builder_generic_ty >, )* @@ -118,7 +118,7 @@ pub(crate) fn impl_builder( } )* - pub fn build(mut self) -> (#overseer_name #generics, #handler) + pub fn build(mut self) -> SubsystemResult<(#overseer_name #generics, #handler)> { let (events_tx, events_rx) = ::polkadot_overseer_gen::metered::channel::< #event @@ -132,20 +132,14 @@ pub(crate) fn impl_builder( #( let (#channel_name_tx, #channel_name_rx) - : ( - ::polkadot_overseer_gen::metered::MeteredSender< MessagePacket< #consumes >>, - ::polkadot_overseer_gen::metered::MeteredReceiver< MessagePacket< #consumes >>, - ) = + = ::polkadot_overseer_gen::metered::channel::< MessagePacket< #consumes > >(CHANNEL_CAPACITY); )* #( - let (#channel_name_unbounded_tx, #channel_name_unbounded_rx): ( - ::polkadot_overseer_gen::metered::UnboundedMeteredSender< MessagePacket< #consumes >>, - ::polkadot_overseer_gen::metered::UnboundedMeteredReceiver< MessagePacket< #consumes >>, - ) = + let (#channel_name_unbounded_tx, #channel_name_unbounded_rx) = ::polkadot_overseer_gen::metered::unbounded::< MessagePacket< #consumes > >(); @@ -154,14 +148,14 @@ pub(crate) fn impl_builder( let channels_out = ChannelsOut { #( - #channel_name: #channel_name_tx, + #channel_name: #channel_name_tx .clone(), )* #( #channel_name_unbounded: #channel_name_unbounded_tx, )* }; - let spawner = self.spawner.expect("Spawner is set. qed"); + let mut spawner = self.spawner.expect("Spawner is set. qed"); let mut running_subsystems = ::polkadot_overseer_gen::FuturesUnordered::< BoxFuture<'static, SubsystemResult<()>> @@ -171,6 +165,8 @@ pub(crate) fn impl_builder( // FIXME generate a builder pattern that ensures this let #subsystem_name = self. #subsystem_name .expect("All subsystem must exist with the builder pattern."); + let unbounded_meter = #channel_name_unbounded_rx.meter().clone(); + let message_rx: SubsystemIncomingMessages< #consumes > = ::polkadot_overseer_gen::select( #channel_name_rx, #channel_name_unbounded_rx ); @@ -186,12 +182,13 @@ pub(crate) fn impl_builder( spawn::<_,_, #blocking, _, _, _>( &mut spawner, #channel_name_tx, - message_rx, signal_tx, + unbounded_meter, channels_out.clone(), - to_overseer_tx, + ctx, #subsystem_name, - ).unwrap(); // FIXME + &mut running_subsystems, + )?; )* #( @@ -216,7 +213,7 @@ pub(crate) fn impl_builder( to_overseer_rx, }; - (overseer, handler) + Ok((overseer, handler)) } } }; @@ -261,14 +258,11 @@ pub(crate) fn impl_task_kind( pub fn spawn( spawner: &mut S, message_tx: ::polkadot_overseer_gen::metered::MeteredSender>, - // muxed incoming channels, bounded and unbouneded - message_rx: ::polkadot_overseer_gen::SubsystemIncomingMessages, signal_tx: ::polkadot_overseer_gen::metered::MeteredSender< #signal >, // meter for the unbounded channel unbounded_meter: ::polkadot_overseer_gen::metered::Meter, // connection to the subsystems channels_out: ChannelsOut, - to_overseer_tx: ::polkadot_overseer_gen::metered::UnboundedMeteredSender, ctx: Ctx, s: SubSys, futures: &mut ::polkadot_overseer_gen::FuturesUnordered>>, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 8c93f5f75b26..9552cc0e7515 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -63,7 +63,7 @@ pub(crate) fn impl_channels_out_struct( pub fn send_unbounded_and_log_error( &self, signals_received: usize, - message: AllMessages, + message: #message_wrapper, ) { use ::std::sync::mpsc::TrySendError; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index a6860a8fb899..8d8ab8b8341e 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -14,7 +14,7 @@ pub(crate) fn impl_message_wrapper_enum( let ts = quote! { /// Generated message type wrapper #[derive(Debug, Clone)] - enum #message_wrapper { + pub enum #message_wrapper { #( #consumes ( #consumes ), )* From a028b1623044a9fccab1f67041f9d91bfd7e20d3 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 26 May 2021 16:13:45 +0200 Subject: [PATCH 045/161] extend example, improved fixins --- node/overseer/overseer-gen/examples/dummy.rs | 20 ++++++++++++++----- .../proc-macro/src/impl_builder.rs | 9 ++++++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index ca2ac1ed1698..dbbb0be6f36e 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -16,6 +16,16 @@ impl ::polkadot_overseer_gen::Subsystem, Yik } } +#[derive(Default)] +pub struct GoblinTower; + +impl ::polkadot_overseer_gen::Subsystem, Yikes> for GoblinTower { + fn start(self, ctx: OverseerSubsystemContext) -> SpawnedSubsystem < Yikes > { + unimplemented!("starting yay!") + } +} + + /// A signal sent by the overseer. #[derive(Debug, Clone)] pub struct SigSigSig; @@ -67,10 +77,10 @@ struct Xxx { #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, - // #[subsystem(no_dispatch, Plinko)] - // plinkos: AwesomeSubSys, + #[subsystem(no_dispatch, blocking, Plinko)] + plinkos: GoblinTower, - // i_like_pi: f64, + i_like_pi: f64, } #[derive(Debug, Clone)] @@ -92,8 +102,8 @@ struct DummyCtx; fn main() { let overseer = Xxx::builder() .sub0(AwesomeSubSys::default()) - // .plinkos(AwesomeSubSys::default()) - // .i_like_pie(std::f64::consts::PI) + .plinkos(GoblinTower::default()) + .i_like_pi(::std::f64::consts::PI) .spawner(DummySpawner) .build(); } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index ee3bd5ee5785..382af43bff29 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -118,6 +118,13 @@ pub(crate) fn impl_builder( } )* + #( + pub fn #baggage_name (mut self, baggage: #baggage_ty ) -> Self { + self. #baggage_name = Some( baggage ); + self + } + )* + pub fn build(mut self) -> SubsystemResult<(#overseer_name #generics, #handler)> { let (events_tx, events_rx) = ::polkadot_overseer_gen::metered::channel::< @@ -175,7 +182,7 @@ pub(crate) fn impl_builder( signal_rx, message_rx, channels_out.clone(), - to_overseer_tx, + to_overseer_tx.clone(), ); let #subsystem_name: OverseenSubsystem< #consumes > = From 97bc44782ab7069fd0d76a3258d111ff98f75f52 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 26 May 2021 20:06:22 +0200 Subject: [PATCH 046/161] chores --- .../proc-macro/src/impl_builder.rs | 80 ++-- .../proc-macro/src/impl_channels_out.rs | 126 +++--- .../proc-macro/src/impl_dispatch.rs | 14 +- .../proc-macro/src/impl_message_wrapper.rs | 6 +- .../overseer-gen/proc-macro/src/impl_misc.rs | 417 +++++++++--------- .../proc-macro/src/impl_overseer.rs | 16 +- .../overseer-gen/proc-macro/src/lib.rs | 27 +- .../overseer-gen/proc-macro/src/parse_attr.rs | 54 ++- .../proc-macro/src/parse_struct.rs | 102 ++--- .../overseer-gen/proc-macro/src/tests.rs | 127 +++--- 10 files changed, 466 insertions(+), 503 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 382af43bff29..82177c4bd14a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -5,9 +5,7 @@ use super::*; /// Implement a builder pattern for the `Overseer`-type, /// which acts as the gateway to constructing the overseer. -pub(crate) fn impl_builder( - info: &OverseerInfo, -) -> Result { +pub(crate) fn impl_builder(info: &OverseerInfo) -> Result { let overseer_name = info.overseer_name.clone(); let builder = Ident::new(&(overseer_name.to_string() + "Builder"), overseer_name.span()); let handler = Ident::new(&(overseer_name.to_string() + "Handler"), overseer_name.span()); @@ -28,13 +26,17 @@ pub(crate) fn impl_builder( let baggage_name = &info.baggage_names(); let baggage_ty = &info.baggage_types(); - let blocking = &info.subsystems().iter().map(|x| { - if x.blocking { - quote! { Blocking } - } else { - quote! { Regular } - } - }).collect::>(); + let blocking = &info + .subsystems() + .iter() + .map(|x| { + if x.blocking { + quote! { Blocking } + } else { + quote! { Regular } + } + }) + .collect::>(); let generics = quote! { < S, #( #baggage_generic_ty, )* > @@ -63,9 +65,9 @@ pub(crate) fn impl_builder( #( #builder_generic_ty : Subsystem, #error_ty>, )* }; - let message_wrapper = &info.message_wrapper; + let _message_wrapper = &info.message_wrapper; let event = &info.extern_event_ty; - let signal = &info.extern_signal_ty; + let _signal = &info.extern_signal_ty; let mut ts = quote! { @@ -174,9 +176,9 @@ pub(crate) fn impl_builder( let unbounded_meter = #channel_name_unbounded_rx.meter().clone(); - let message_rx: SubsystemIncomingMessages< #consumes > = ::polkadot_overseer_gen::select( - #channel_name_rx, #channel_name_unbounded_rx - ); + let message_rx: SubsystemIncomingMessages< #consumes > = ::polkadot_overseer_gen::select( + #channel_name_rx, #channel_name_unbounded_rx + ); let (signal_tx, signal_rx) = ::polkadot_overseer_gen::metered::channel(SIGNAL_CHANNEL_CAPACITY); let ctx = OverseerSubsystemContext::new( signal_rx, @@ -186,23 +188,23 @@ pub(crate) fn impl_builder( ); let #subsystem_name: OverseenSubsystem< #consumes > = - spawn::<_,_, #blocking, _, _, _>( - &mut spawner, - #channel_name_tx, - signal_tx, + spawn::<_,_, #blocking, _, _, _>( + &mut spawner, + #channel_name_tx, + signal_tx, unbounded_meter, - channels_out.clone(), + channels_out.clone(), ctx, - #subsystem_name, + #subsystem_name, &mut running_subsystems, - )?; + )?; )* #( let #baggage_name = self. #baggage_name .expect("Baggage must initialized"); )* - use ::polkadot_overseer_gen::StreamExt; + use ::polkadot_overseer_gen::StreamExt; let to_overseer_rx = to_overseer_rx.fuse(); let overseer = #overseer_name { @@ -224,21 +226,20 @@ pub(crate) fn impl_builder( } } }; - eprintln!(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + eprintln!( + ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> {} ", -ts.to_string() -); - ts.extend(impl_task_kind(info)?); + ts.to_string() + ); + ts.extend(impl_task_kind(info)?); Ok(ts) } -pub(crate) fn impl_task_kind( - info: &OverseerInfo, -) -> Result { - let signal = &info.extern_signal_ty; +pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result { + let signal = &info.extern_signal_ty; - let ts = quote!{ + let ts = quote! { use ::polkadot_overseer_gen::FutureExt; @@ -279,9 +280,9 @@ pub(crate) fn impl_task_kind( M: Send + 'static, TK: TaskKind, Ctx: ::polkadot_overseer_gen::SubsystemContext, - E: std::error::Error + Send + Sync + 'static + From, + E: std::error::Error + Send + Sync + 'static + From, SubSys: ::polkadot_overseer_gen::Subsystem, - { + { let ::polkadot_overseer_gen::SpawnedSubsystem { future, name } = s.start(ctx); let (tx, rx) = ::polkadot_overseer_gen::oneshot::channel(); @@ -315,13 +316,14 @@ pub(crate) fn impl_task_kind( instance, }) } - }; + }; - eprintln!(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + eprintln!( + ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> {} ", -ts.to_string() -); + ts.to_string() + ); - Ok(ts) + Ok(ts) } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 9552cc0e7515..ad719ac1392d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -1,91 +1,87 @@ - use quote::quote; use syn::Result; use super::*; /// Implement the helper type `ChannelsOut` and `MessagePacket`. -pub(crate) fn impl_channels_out_struct( - info: &OverseerInfo, -) -> Result { - - let message_wrapper = info.message_wrapper.clone(); +pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result { + let message_wrapper = info.message_wrapper.clone(); - let channel_name = &info.channel_names(""); - let channel_name_unbounded = &info.channel_names("_unbounded"); + let channel_name = &info.channel_names(""); + let channel_name_unbounded = &info.channel_names("_unbounded"); let consumes = &info.consumes(); - let ts = quote! { - #[derive(Debug, Clone)] + let ts = quote! { + #[derive(Debug, Clone)] pub struct ChannelsOut { #( pub #channel_name: - ::polkadot_overseer_gen::metered::MeteredSender< - MessagePacket< #consumes > - >, + ::polkadot_overseer_gen::metered::MeteredSender< + MessagePacket< #consumes > + >, )* - #( + #( pub #channel_name_unbounded: - ::polkadot_overseer_gen::metered::UnboundedMeteredSender< - MessagePacket< #consumes > - >, + ::polkadot_overseer_gen::metered::UnboundedMeteredSender< + MessagePacket< #consumes > + >, )* } - impl ChannelsOut { - /// Send a message via a bounded channel. - pub async fn send_and_log_error( - &mut self, - signals_received: usize, - message: #message_wrapper, - ) { - let res: ::std::result::Result<_, _> = match message { - #( - #message_wrapper :: #consumes ( inner ) => { - self. #channel_name .send( - ::polkadot_overseer_gen::make_packet(signals_received, inner) - ).await - } - )* - }; + impl ChannelsOut { + /// Send a message via a bounded channel. + pub async fn send_and_log_error( + &mut self, + signals_received: usize, + message: #message_wrapper, + ) { + let res: ::std::result::Result<_, _> = match message { + #( + #message_wrapper :: #consumes ( inner ) => { + self. #channel_name .send( + ::polkadot_overseer_gen::make_packet(signals_received, inner) + ).await + } + )* + }; - if res.is_err() { - ::polkadot_overseer_gen::tracing::debug!( - target: LOG_TARGET, - "Failed to send a message to another subsystem", - ); - } - } + if res.is_err() { + ::polkadot_overseer_gen::tracing::debug!( + target: LOG_TARGET, + "Failed to send a message to another subsystem", + ); + } + } - /// Send a message to another subsystem via an unbounded channel. - pub fn send_unbounded_and_log_error( - &self, - signals_received: usize, - message: #message_wrapper, - ) { - use ::std::sync::mpsc::TrySendError; + /// Send a message to another subsystem via an unbounded channel. + pub fn send_unbounded_and_log_error( + &self, + signals_received: usize, + message: #message_wrapper, + ) { + use ::std::sync::mpsc::TrySendError; - let res: ::std::result::Result<_, _> = match message { - #( - #message_wrapper :: #consumes (inner) => { - self. #channel_name_unbounded .unbounded_send( - make_packet(signals_received, inner) - ) - .map_err(|e| e.into_send_error()) - }, - )* - }; + let res: ::std::result::Result<_, _> = match message { + #( + #message_wrapper :: #consumes (inner) => { + self. #channel_name_unbounded .unbounded_send( + make_packet(signals_received, inner) + ) + .map_err(|e| e.into_send_error()) + }, + )* + }; - if res.is_err() { - ::polkadot_overseer_gen::tracing::debug!( - target: LOG_TARGET, - "Failed to send a message to another subsystem", - ); - } - } - } + if res.is_err() { + ::polkadot_overseer_gen::tracing::debug!( + target: LOG_TARGET, + "Failed to send a message to another subsystem", + ); + } + } + } }; Ok(ts) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index ce844f10f640..d7d3ad2d0804 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -14,18 +14,22 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use super::*; use proc_macro2::TokenStream; use quote::quote; -use syn::{Ident, Path, Result}; -use super::*; - +use syn::{Path, Result}; pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { let message_wrapper = &info.message_wrapper; - let dispatchable = info.subsystems().into_iter().filter(|ssf| !ssf.no_dispatch).map(|ssf| ssf.consumes.clone()).collect::>(); + let dispatchable = info + .subsystems() + .into_iter() + .filter(|ssf| !ssf.no_dispatch) + .map(|ssf| ssf.consumes.clone()) + .collect::>(); - let extern_event_ty= &info.extern_event_ty.clone(); + let extern_event_ty = &info.extern_event_ty.clone(); let ts = quote! { impl #message_wrapper { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 8d8ab8b8341e..bf4ce1b58a5a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -4,12 +4,10 @@ use syn::Result; use super::*; /// Generates the wrapper type enum. -pub(crate) fn impl_message_wrapper_enum( - info: &OverseerInfo, -) -> Result { +pub(crate) fn impl_message_wrapper_enum(info: &OverseerInfo) -> Result { let consumes = info.consumes(); - let message_wrapper = &info.message_wrapper; + let message_wrapper = &info.message_wrapper; let ts = quote! { /// Generated message type wrapper diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index a24b31dc4f79..3caaaef60d3b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -3,249 +3,250 @@ use syn::{Ident, Result}; use super::*; - /// Implement a builder pattern for the `Overseer`-type, /// which acts as the gateway to constructing the overseer. -pub(crate) fn impl_misc( - info: &OverseerInfo, -) -> Result { +pub(crate) fn impl_misc(info: &OverseerInfo) -> Result { let overseer_name = info.overseer_name.clone(); - let builder = Ident::new(&(overseer_name.to_string() + "Builder"), overseer_name.span()); - let handler = Ident::new(&(overseer_name.to_string() + "Handler"), overseer_name.span()); + let _builder = Ident::new(&(overseer_name.to_string() + "Builder"), overseer_name.span()); + let _handler = Ident::new(&(overseer_name.to_string() + "Handler"), overseer_name.span()); - let subsystem_name = &info.subsystem_names(); + let _subsystem_name = &info.subsystem_names(); let builder_generic_ty = &info.builder_generic_types(); - let channel_name = &info.channel_names(""); - let channel_name_unbounded = &info.channel_names("_unbounded"); + let _channel_name = &info.channel_names(""); + let _channel_name_unbounded = &info.channel_names("_unbounded"); - let channel_name_tx = &info.channel_names("_tx"); - let channel_name_unbounded_tx = &info.channel_names("_unbounded_tx"); + let _channel_name_tx = &info.channel_names("_tx"); + let _channel_name_unbounded_tx = &info.channel_names("_unbounded_tx"); - let channel_name_rx = &info.channel_names("_rx"); - let channel_name_unbounded_rx = &info.channel_names("_unbounded_rx"); + let _channel_name_rx = &info.channel_names("_rx"); + let _channel_name_unbounded_rx = &info.channel_names("_unbounded_rx"); let baggage_generic_ty = &info.baggage_generic_types(); - let baggage_name = &info.baggage_names(); - let baggage_ty = &info.baggage_types(); - - let blocking = &info.subsystems().iter().map(|x| { - if x.blocking { - quote! { Blocking } - } else { - quote! { Regular } - } - }).collect::>(); + let _baggage_name = &info.baggage_names(); + let _baggage_ty = &info.baggage_types(); + + let _blocking = &info + .subsystems() + .iter() + .map(|x| { + if x.blocking { + quote! { Blocking } + } else { + quote! { Regular } + } + }) + .collect::>(); let consumes = &info.consumes(); let message_wrapper = &info.message_wrapper; - let event = &info.extern_event_ty; + let _event = &info.extern_event_ty; let signal = &info.extern_signal_ty; - let generics = quote! { + let _generics = quote! { < S, #( #baggage_generic_ty, )* > }; - let where_clause = quote! { + let _where_clause = quote! { where S: ::polkadot_overseer_gen::SpawnNamed, }; - let builder_generics = quote! { + let _builder_generics = quote! { }; // all subsystems must have the same context // even if the overseer does not impose such a limit. - let builder_additional_generics = quote! { + let _builder_additional_generics = quote! { < Ctx, #( #builder_generic_ty, )* > }; let error_ty = &info.extern_error_ty; - let builder_where_clause = quote! { + let _builder_where_clause = quote! { where - Ctx: SubsystemContext< #message_wrapper > + Ctx: SubsystemContext< #message_wrapper > S: ::polkadot_overseer_gen::SpawnNamed, #( #builder_generic_ty : Subsystem, )* }; let ts = quote! { - // ////////////////////////////////////////////////// - // `OverseerSubsystemSender` - - #[derive(Debug, Clone)] - pub struct OverseerSubsystemSender { - channels: ChannelsOut, - signals_received: SignalsReceived, - } - - #( - #[::polkadot_overseer_gen::async_trait] - impl SubsystemSender< #consumes > for OverseerSubsystemSender { - async fn send_message(&mut self, msg: #consumes) { - self.channels.send_and_log_error(self.signals_received.load(), msg.into()).await; - } - - async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send - { - // This can definitely be optimized if necessary. - for msg in msgs { - self.send_message(msg).await; - } - } - - fn send_unbounded_message(&mut self, msg: #consumes) { - self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg.into()); - } - } - )* - - /// A context type that is given to the [`Subsystem`] upon spawning. - /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s - /// or to spawn it's [`SubsystemJob`]s. - /// - /// [`Overseer`]: struct.Overseer.html - /// [`Subsystem`]: trait.Subsystem.html - /// [`SubsystemJob`]: trait.SubsystemJob.html - #[derive(Debug)] - pub struct OverseerSubsystemContext{ - signals: metered::MeteredReceiver< #signal >, - messages: SubsystemIncomingMessages, - to_subsystems: OverseerSubsystemSender, - to_overseer: metered::UnboundedMeteredSender, - signals_received: SignalsReceived, - pending_incoming: Option<(usize, M)>, - } - - impl OverseerSubsystemContext { - /// Create a new `OverseerSubsystemContext`. - fn new( - signals: metered::MeteredReceiver< #signal >, - messages: SubsystemIncomingMessages, - to_subsystems: ChannelsOut, - to_overseer: metered::UnboundedMeteredSender, - ) -> Self { - let signals_received = SignalsReceived::default(); - OverseerSubsystemContext { - signals, - messages, - to_subsystems: OverseerSubsystemSender { - channels: to_subsystems, - signals_received: signals_received.clone(), - }, - to_overseer, - signals_received, - pending_incoming: None, - } - } - } - - #[::polkadot_overseer_gen::async_trait] - impl SubsystemContext for OverseerSubsystemContext - where - OverseerSubsystemSender: polkadot_overseer_gen::SubsystemSender - { - type Message = M; - type Signal = #signal; - type Sender = OverseerSubsystemSender; - - async fn try_recv(&mut self) -> Result>, ()> { - match ::polkadot_overseer_gen::poll!(self.recv()) { - ::polkadot_overseer_gen::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), - ::polkadot_overseer_gen::Poll::Pending => Ok(None), - } - } - - async fn recv(&mut self) -> SubsystemResult> { - loop { - // If we have a message pending an overseer signal, we only poll for signals - // in the meantime. - if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { - if needs_signals_received <= self.signals_received.load() { - return Ok(FromOverseer::Communication { msg }); - } else { - self.pending_incoming = Some((needs_signals_received, msg)); - - // wait for next signal. - let signal = self.signals.next().await - .ok_or(SubsystemError::Context( - "Signal channel is terminated and empty." - .to_owned() - ))?; - - self.signals_received.inc(); - return Ok(FromOverseer::Signal(signal)) - } - } - - let mut await_message = self.messages.next().fuse(); - let mut await_signal = self.signals.next().fuse(); - let signals_received = self.signals_received.load(); - let pending_incoming = &mut self.pending_incoming; - - // Otherwise, wait for the next signal or incoming message. - let from_overseer = futures::select_biased! { - signal = await_signal => { - let signal = signal - .ok_or(SubsystemError::Context( - "Signal channel is terminated and empty." - .to_owned() - ))?; - - FromOverseer::Signal(signal) - } - msg = await_message => { - let packet = msg - .ok_or(SubsystemError::Context( - "Message channel is terminated and empty." - .to_owned() - ))?; - - if packet.signals_received > signals_received { - // wait until we've received enough signals to return this message. - *pending_incoming = Some((packet.signals_received, packet.message)); - continue; - } else { - // we know enough to return this message. - FromOverseer::Communication { msg: packet.message} - } - } - }; - - if let FromOverseer::Signal(_) = from_overseer { - self.signals_received.inc(); - } - - return Ok(from_overseer); - } - } - - fn sender(&mut self) -> &mut Self::Sender { - &mut self.to_subsystems - } - - #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn(ctx, name, fut)")] - async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) - -> SubsystemResult<()> - { - self.to_overseer.send(ToOverseer::SpawnJob { - name, - s, - }).await.map_err(Into::into) - } - - #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn_blocking(ctx, name, fut)")] - async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) - -> SubsystemResult<()> - { - self.to_overseer.send(ToOverseer::SpawnBlockingJob { - name, - s, - }).await.map_err(Into::into) - } - } - }; - - Ok(ts) + // ////////////////////////////////////////////////// + // `OverseerSubsystemSender` + + #[derive(Debug, Clone)] + pub struct OverseerSubsystemSender { + channels: ChannelsOut, + signals_received: SignalsReceived, + } + + #( + #[::polkadot_overseer_gen::async_trait] + impl SubsystemSender< #consumes > for OverseerSubsystemSender { + async fn send_message(&mut self, msg: #consumes) { + self.channels.send_and_log_error(self.signals_received.load(), msg.into()).await; + } + + async fn send_messages(&mut self, msgs: T) + where T: IntoIterator + Send, T::IntoIter: Send + { + // This can definitely be optimized if necessary. + for msg in msgs { + self.send_message(msg).await; + } + } + + fn send_unbounded_message(&mut self, msg: #consumes) { + self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg.into()); + } + } + )* + + /// A context type that is given to the [`Subsystem`] upon spawning. + /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s + /// or to spawn it's [`SubsystemJob`]s. + /// + /// [`Overseer`]: struct.Overseer.html + /// [`Subsystem`]: trait.Subsystem.html + /// [`SubsystemJob`]: trait.SubsystemJob.html + #[derive(Debug)] + pub struct OverseerSubsystemContext{ + signals: metered::MeteredReceiver< #signal >, + messages: SubsystemIncomingMessages, + to_subsystems: OverseerSubsystemSender, + to_overseer: metered::UnboundedMeteredSender, + signals_received: SignalsReceived, + pending_incoming: Option<(usize, M)>, + } + + impl OverseerSubsystemContext { + /// Create a new `OverseerSubsystemContext`. + fn new( + signals: metered::MeteredReceiver< #signal >, + messages: SubsystemIncomingMessages, + to_subsystems: ChannelsOut, + to_overseer: metered::UnboundedMeteredSender, + ) -> Self { + let signals_received = SignalsReceived::default(); + OverseerSubsystemContext { + signals, + messages, + to_subsystems: OverseerSubsystemSender { + channels: to_subsystems, + signals_received: signals_received.clone(), + }, + to_overseer, + signals_received, + pending_incoming: None, + } + } + } + + #[::polkadot_overseer_gen::async_trait] + impl SubsystemContext for OverseerSubsystemContext + where + OverseerSubsystemSender: polkadot_overseer_gen::SubsystemSender + { + type Message = M; + type Signal = #signal; + type Sender = OverseerSubsystemSender; + + async fn try_recv(&mut self) -> Result>, ()> { + match ::polkadot_overseer_gen::poll!(self.recv()) { + ::polkadot_overseer_gen::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), + ::polkadot_overseer_gen::Poll::Pending => Ok(None), + } + } + + async fn recv(&mut self) -> SubsystemResult> { + loop { + // If we have a message pending an overseer signal, we only poll for signals + // in the meantime. + if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { + if needs_signals_received <= self.signals_received.load() { + return Ok(FromOverseer::Communication { msg }); + } else { + self.pending_incoming = Some((needs_signals_received, msg)); + + // wait for next signal. + let signal = self.signals.next().await + .ok_or(SubsystemError::Context( + "Signal channel is terminated and empty." + .to_owned() + ))?; + + self.signals_received.inc(); + return Ok(FromOverseer::Signal(signal)) + } + } + + let mut await_message = self.messages.next().fuse(); + let mut await_signal = self.signals.next().fuse(); + let signals_received = self.signals_received.load(); + let pending_incoming = &mut self.pending_incoming; + + // Otherwise, wait for the next signal or incoming message. + let from_overseer = futures::select_biased! { + signal = await_signal => { + let signal = signal + .ok_or(SubsystemError::Context( + "Signal channel is terminated and empty." + .to_owned() + ))?; + + FromOverseer::Signal(signal) + } + msg = await_message => { + let packet = msg + .ok_or(SubsystemError::Context( + "Message channel is terminated and empty." + .to_owned() + ))?; + + if packet.signals_received > signals_received { + // wait until we've received enough signals to return this message. + *pending_incoming = Some((packet.signals_received, packet.message)); + continue; + } else { + // we know enough to return this message. + FromOverseer::Communication { msg: packet.message} + } + } + }; + + if let FromOverseer::Signal(_) = from_overseer { + self.signals_received.inc(); + } + + return Ok(from_overseer); + } + } + + fn sender(&mut self) -> &mut Self::Sender { + &mut self.to_subsystems + } + + #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn(ctx, name, fut)")] + async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) + -> SubsystemResult<()> + { + self.to_overseer.send(ToOverseer::SpawnJob { + name, + s, + }).await.map_err(Into::into) + } + + #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn_blocking(ctx, name, fut)")] + async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) + -> SubsystemResult<()> + { + self.to_overseer.send(ToOverseer::SpawnBlockingJob { + name, + s, + }).await.map_err(Into::into) + } + } + }; + + Ok(ts) } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 517088169ca5..720706b2c037 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -1,16 +1,14 @@ use quote::quote; -use syn::{Ident, Result}; +use syn::Result; use super::*; -pub(crate) fn impl_overseer_struct( - info: &OverseerInfo, -) -> Result { +pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result { let message_wrapper = &info.message_wrapper.clone(); let overseer_name = info.overseer_name.clone(); let subsystem_name = &info.subsystem_names(); - let builder_generic_ty = &info.builder_generic_types(); + let _builder_generic_ty = &info.builder_generic_types(); let baggage_name = &info.baggage_names(); let baggage_ty = &info.baggage_types(); @@ -38,7 +36,7 @@ pub(crate) fn impl_overseer_struct( let log_target = syn::LitStr::new(overseer_name.to_string().to_lowercase().as_str(), overseer_name.span()); - let mut ts = quote! { + let ts = quote! { const STOP_DELAY: ::std::time::Duration = ::std::time::Duration::from_secs(1); /// Capacity of a bounded message channel between overseer and subsystem @@ -104,12 +102,10 @@ pub(crate) fn impl_overseer_struct( Ok(ts) } - - pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { let signal = &info.extern_signal_ty; - let message_wrapper = &info.message_wrapper; - let consumes = &info.consumes(); + let _message_wrapper = &info.message_wrapper; + let _consumes = &info.consumes(); let ts = quote::quote! { diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 119f40bb0fe0..426530c289dc 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -15,29 +15,26 @@ // along with Polkadot. If not, see . use proc_macro2::TokenStream; -use syn::{parse2, Error, Result}; -use syn::spanned::Spanned; -use std::collections::HashSet; +use syn::{parse2, Result}; -mod parse_struct; -mod parse_attr; -mod impl_overseer; -mod impl_misc; mod impl_builder; -mod impl_replace; +mod impl_misc; +mod impl_overseer; +mod parse_attr; +mod parse_struct; +// mod impl_replace; mod impl_channels_out; -mod impl_message_wrapper; mod impl_dispatch; +mod impl_message_wrapper; -use parse_struct::*; -use parse_attr::*; -use impl_overseer::*; use impl_builder::*; -use impl_misc::*; -use impl_replace::*; use impl_channels_out::*; -use impl_message_wrapper::*; use impl_dispatch::*; +use impl_message_wrapper::*; +use impl_misc::*; +use impl_overseer::*; +use parse_attr::*; +use parse_struct::*; #[proc_macro_attribute] pub fn overlord(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream { diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs index 66cd8449a8c4..0f5cc750cd51 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs @@ -1,17 +1,16 @@ use proc_macro2::Span; -use std::collections::{HashMap, hash_map::RandomState}; -use syn::Path; -use syn::Error; -use syn::Ident; -use syn::LitInt; +use std::collections::{hash_map::RandomState, HashMap}; use syn::parse::Parse; use syn::parse::ParseBuffer; use syn::punctuated::Punctuated; -use syn::Result; use syn::spanned::Spanned; +use syn::Error; +use syn::Ident; +use syn::LitInt; +use syn::Path; +use syn::Result; use syn::Token; - #[derive(Clone, Debug)] enum AttrItem { ExternEventType(Path), @@ -63,13 +62,13 @@ impl Parse for AttrItem { Ok(if key == TAG_EXT_SIGNAL_TY { let path = input.parse::()?; AttrItem::ExternOverseerSignalType(path) - } else if key == TAG_EXT_EVENT_TY { + } else if key == TAG_EXT_EVENT_TY { let path = input.parse::()?; AttrItem::ExternEventType(path) - } else if key == TAG_EXT_ERROR_TY { + } else if key == TAG_EXT_ERROR_TY { let path = input.parse::()?; AttrItem::ExternErrorType(path) - } else if key == TAG_GEN_TY { + } else if key == TAG_GEN_TY { let wrapper_message = input.parse::()?; AttrItem::MessageWrapperName(wrapper_message) } else if key == TAG_SIGNAL_CAPACITY { @@ -79,7 +78,7 @@ impl Parse for AttrItem { let value = input.parse::()?; AttrItem::MessageChannelCapacity(value) } else { - return Err(Error::new(span, "Expected one of `gen`, `signal_capacity`, or `message_capacity`.")) + return Err(Error::new(span, "Expected one of `gen`, `signal_capacity`, or `message_capacity`.")); }) } } @@ -105,7 +104,7 @@ impl Parse for AttrArgs { if let Some(first) = unique.insert(item.key(), item.clone()) { let mut e = Error::new(item.span(), format!("Duplicate definition of `{}` found", item.key())); e.combine(Error::new(first.span(), "previously defined here.")); - return Err(e) + return Err(e); } } @@ -128,38 +127,37 @@ impl Parse for AttrArgs { } else { 1024_usize }; - let extern_error_ty = unique.remove(TAG_EXT_ERROR_TY) - .map(|x| if let AttrItem::ExternErrorType(x) = x { x.clone() } else { unreachable!() } ) + let extern_error_ty = unique + .remove(TAG_EXT_ERROR_TY) + .map(|x| if let AttrItem::ExternErrorType(x) = x { x.clone() } else { unreachable!() }) .ok_or_else(|| { Error::new(span, format!("Must declare the overseer signals type via `{}=..`.", TAG_EXT_ERROR_TY)) })?; - let extern_signal_ty = unique.remove(TAG_EXT_SIGNAL_TY) - .map(|x| if let AttrItem::ExternOverseerSignalType(x) = x { x.clone() } else { unreachable!() } ) + let extern_signal_ty = unique + .remove(TAG_EXT_SIGNAL_TY) + .map(|x| if let AttrItem::ExternOverseerSignalType(x) = x { x.clone() } else { unreachable!() }) .ok_or_else(|| { Error::new(span, format!("Must declare the overseer signals type via `{}=..`.", TAG_EXT_SIGNAL_TY)) })?; - let extern_event_ty = unique.remove(TAG_EXT_EVENT_TY) - .map(|x| if let AttrItem::ExternEventType(x) = x { x.clone() } else { unreachable!() } ) + let extern_event_ty = unique + .remove(TAG_EXT_EVENT_TY) + .map(|x| if let AttrItem::ExternEventType(x) = x { x.clone() } else { unreachable!() }) .ok_or_else(|| { Error::new(span, format!("Must declare the external event type via `{}=..`.", TAG_EXT_EVENT_TY)) - })?; - let message_wrapper = unique.remove(TAG_GEN_TY) - .map(|x| if let AttrItem::MessageWrapperName(x) = x { x.clone() } else { unreachable!() } ) - .ok_or_else(|| { - Error::new(span, format!("Must declare the generated type via `{}=..`.", TAG_GEN_TY)) - })?; + let message_wrapper = unique + .remove(TAG_GEN_TY) + .map(|x| if let AttrItem::MessageWrapperName(x) = x { x.clone() } else { unreachable!() }) + .ok_or_else(|| Error::new(span, format!("Must declare the generated type via `{}=..`.", TAG_GEN_TY)))?; if !unique.is_empty() { - let v = unique.into_iter().map(|(tag, _attr)| -> String { - format!("`{}`", tag) - }).collect::>(); + let v = unique.into_iter().map(|(tag, _attr)| -> String { format!("`{}`", tag) }).collect::>(); let s = v.join(", "); - return Err(Error::new(span, format!("Found unknown arguments to the overseer macro {}.", s))) + return Err(Error::new(span, format!("Found unknown arguments to the overseer macro {}.", s))); } Ok(AttrArgs { diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 37b58c4d2444..4caef9921ff4 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -1,17 +1,17 @@ use proc_macro2::Span; -use std::collections::{HashSet, hash_map::RandomState}; -use syn::{AttrStyle, Path}; +use std::collections::{hash_map::RandomState, HashSet}; +use syn::parse::Parse; +use syn::parse::ParseStream; +use syn::punctuated::Punctuated; +use syn::spanned::Spanned; use syn::Attribute; use syn::Field; use syn::FieldsNamed; use syn::Ident; -use syn::parse::Parse; -use syn::punctuated::Punctuated; -use syn::spanned::Spanned; use syn::Token; use syn::Type; -use syn::{parse2, ItemStruct, Error, GenericParam, Result}; -use syn::parse::ParseStream; +use syn::{AttrStyle, Path}; +use syn::{Error, GenericParam, ItemStruct, Result}; /// A field of the struct annotated with /// `#[subsystem(no_dispatch, , A | B | C)]` @@ -40,12 +40,6 @@ fn try_type_to_path(ty: Type, span: Span) -> Result { } } - -fn try_type_to_ident(ty: Type, span: Span) -> Result { - try_type_to_path(ty, span.clone())?.get_ident().cloned().ok_or_else(|| Error::new(span, "Expected an identifier, but got a path.")) -} - - pub(crate) struct SubSystemTag { #[allow(dead_code)] pub(crate) attrs: Vec, @@ -75,10 +69,10 @@ impl Parse for SubSystemTag { let mut unique = HashSet::<_, RandomState>::default(); while let Some(ident) = parse_tags()? { if ident != "no_dispatch" && ident != "blocking" { - return Err(Error::new(ident.span(), "Allowed tags are only `no_dispatch` or `blocking`.")) + return Err(Error::new(ident.span(), "Allowed tags are only `no_dispatch` or `blocking`.")); } if !unique.insert(ident.to_string()) { - return Err(Error::new(ident.span(), "Found duplicate tag.")) + return Err(Error::new(ident.span(), "Found duplicate tag.")); } } let no_dispatch = unique.take("no_dispatch").is_some(); @@ -86,16 +80,10 @@ impl Parse for SubSystemTag { let consumes = content.parse_terminated(Path::parse)?; - Ok(Self { - attrs, - no_dispatch, - blocking, - consumes, - }) + Ok(Self { attrs, no_dispatch, blocking, consumes }) } } - /// Fields that are _not_ subsystems. #[derive(Debug, Clone)] pub(crate) struct BaggageField { @@ -104,7 +92,6 @@ pub(crate) struct BaggageField { pub(crate) generic: bool, } - #[derive(Clone, Debug)] pub(crate) struct OverseerInfo { /// Fields annotated with `#[subsystem(..)]`. @@ -161,19 +148,22 @@ impl OverseerInfo { } pub(crate) fn baggage_generic_types(&self) -> Vec { - self.baggage.iter().filter(|bag| bag.generic).filter_map(|bag| bag.field_ty.get_ident().cloned()).collect::>() + self.baggage + .iter() + .filter(|bag| bag.generic) + .filter_map(|bag| bag.field_ty.get_ident().cloned()) + .collect::>() } pub(crate) fn channel_names(&self, suffix: &'static str) -> Vec { - self.subsystems.iter() - .map(|ssf| Ident::new(&(ssf.name.to_string() + suffix), ssf.name.span())) - .collect::>() + self.subsystems + .iter() + .map(|ssf| Ident::new(&(ssf.name.to_string() + suffix), ssf.name.span())) + .collect::>() } pub(crate) fn consumes(&self) -> Vec { - self.subsystems.iter() - .map(|ssf| ssf.consumes.clone()) - .collect::>() + self.subsystems.iter().map(|ssf| ssf.consumes.clone()).collect::>() } } @@ -191,42 +181,33 @@ impl OverseerGuts { let mut subsystems = Vec::with_capacity(n); let mut baggage = Vec::with_capacity(n); for (idx, Field { attrs, vis: _, ident, ty, .. }) in fields.named.into_iter().enumerate() { - let mut consumes = attrs.iter() - .filter(|attr| attr.style == AttrStyle::Outer) - .filter_map(|attr| { - let span = attr.path.span(); - attr.path.get_ident().filter(|ident| *ident == "subsystem").map(move |_ident| { - let attr_tokens = attr.tokens.clone(); - (attr_tokens, span) - }) - }); - let ident = ident.ok_or_else(|| { - Error::new(ty.span(), "Missing identifier for member. BUG") - })?; + let mut consumes = attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| { + let span = attr.path.span(); + attr.path.get_ident().filter(|ident| *ident == "subsystem").map(move |_ident| { + let attr_tokens = attr.tokens.clone(); + (attr_tokens, span) + }) + }); + let ident = ident.ok_or_else(|| Error::new(ty.span(), "Missing identifier for member. BUG"))?; if let Some((attr_tokens, span)) = consumes.next() { if let Some((_attr_tokens2, span2)) = consumes.next() { return Err({ let mut err = Error::new(span, "The first subsystem annotation is at"); - err.combine( - Error::new(span2, "but another here for the same field.") - ); + err.combine(Error::new(span2, "but another here for the same field.")); err - }) + }); } let mut consumes_paths = Vec::with_capacity(attrs.len()); let attr_tokens = attr_tokens.clone(); let variant: SubSystemTag = syn::parse2(attr_tokens.clone())?; if variant.consumes.len() != 1 { - return Err(Error::new(attr_tokens.span(), "Exactly one message can be consumed per subsystem.")) + return Err(Error::new(attr_tokens.span(), "Exactly one message can be consumed per subsystem.")); } consumes_paths.extend(variant.consumes.into_iter()); - if consumes_paths.is_empty() { - return Err( - Error::new(span, "Subsystem must consume at least one message") - ) + return Err(Error::new(span, "Subsystem must consume at least one message")); } subsystems.push(SubSysField { @@ -239,19 +220,12 @@ impl OverseerGuts { }); } else { let field_ty: Path = try_type_to_path(ty, ident.span())?; - let generic: bool = if let Some(ident) = field_ty.get_ident() { - baggage_generics.contains(ident) - } else { - false - }; - baggage.push(BaggageField { - field_name: ident, - generic, - field_ty, - }); + let generic: bool = + if let Some(ident) = field_ty.get_ident() { baggage_generics.contains(ident) } else { false }; + baggage.push(BaggageField { field_name: ident, generic, field_ty }); } } - Ok( Self { name, subsystems, baggage }) + Ok(Self { name, subsystems, baggage }) } } @@ -286,7 +260,9 @@ impl Parse for OverseerGuts { Self::parse_fields(name, baggage_generic_idents, named) } - syn::Fields::Unit => Err(Error::new(ds.fields.span(), "Must be a struct with named fields. Not an unit struct.")), + syn::Fields::Unit => { + Err(Error::new(ds.fields.span(), "Must be a struct with named fields. Not an unit struct.")) + } syn::Fields::Unnamed(unnamed) => { Err(Error::new(unnamed.span(), "Must be a struct with named fields. Not an unnamed fields struct.")) } diff --git a/node/overseer/overseer-gen/proc-macro/src/tests.rs b/node/overseer/overseer-gen/proc-macro/src/tests.rs index 98f96bfb3ae3..56b230786d37 100644 --- a/node/overseer/overseer-gen/proc-macro/src/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/tests.rs @@ -1,99 +1,94 @@ use super::*; -use quote::quote; use assert_matches::assert_matches; +use quote::quote; use syn::parse_quote; - - #[test] fn print() { - let attr = quote! { - gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, - }; + let attr = quote! { + gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, + }; - let item = quote! { - pub struct Ooooh where X: Secrit { - #[subsystem(no_dispatch, Foo)] - sub0: FooSubsystem, + let item = quote! { + pub struct Ooooh where X: Secrit { + #[subsystem(no_dispatch, Foo)] + sub0: FooSubsystem, - #[subsystem(blocking, Bar)] - yyy: BaersBuyBilliardBalls, + #[subsystem(blocking, Bar)] + yyy: BaersBuyBilliardBalls, - #[subsystem(no_dispatch, blocking, Twain)] - fff: Beeeeep, + #[subsystem(no_dispatch, blocking, Twain)] + fff: Beeeeep, - #[subsystem(Rope)] - mc: MountainCave, + #[subsystem(Rope)] + mc: MountainCave, - metrics: Metrics, - } - }; + metrics: Metrics, + } + }; - let output = impl_overseer_gen(attr, item).expect("Simple example always works. qed"); - println!("//generated:"); - println!("{}", output); + let output = impl_overseer_gen(attr, item).expect("Simple example always works. qed"); + println!("//generated:"); + println!("{}", output); } - #[test] fn struct_parse_full() { - let item: OverseerGuts = parse_quote! { - pub struct Ooooh where X: Secrit { - #[subsystem(no_dispatch, Foo)] - sub0: FooSubsystem, + let item: OverseerGuts = parse_quote! { + pub struct Ooooh where X: Secrit { + #[subsystem(no_dispatch, Foo)] + sub0: FooSubsystem, - #[subsystem(blocking, Bar)] - yyy: BaersBuyBilliardBalls, + #[subsystem(blocking, Bar)] + yyy: BaersBuyBilliardBalls, - #[subsystem(no_dispatch, blocking, Twain)] - fff: Beeeeep, + #[subsystem(no_dispatch, blocking, Twain)] + fff: Beeeeep, - #[subsystem(Rope)] - mc: MountainCave, + #[subsystem(Rope)] + mc: MountainCave, - metrics: Metrics, - } - }; - let _ = dbg!(item); + metrics: Metrics, + } + }; + let _ = dbg!(item); } #[test] fn struct_parse_basic() { - let item: OverseerGuts = parse_quote! { - pub struct Ooooh { - #[subsystem(Foo)] - sub0: FooSubsystem, - } - }; - let _ = dbg!(item); + let item: OverseerGuts = parse_quote! { + pub struct Ooooh { + #[subsystem(Foo)] + sub0: FooSubsystem, + } + }; + let _ = dbg!(item); } - #[test] fn attr_full() { - let attr: AttrArgs = parse_quote! { - gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, - }; - assert_matches!(attr, AttrArgs { - message_channel_capacity, - signal_channel_capacity, - .. - } => { - assert_eq!(message_channel_capacity, 222); - assert_eq!(signal_channel_capacity, 111); - }); + let attr: AttrArgs = parse_quote! { + gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, + }; + assert_matches!(attr, AttrArgs { + message_channel_capacity, + signal_channel_capacity, + .. + } => { + assert_eq!(message_channel_capacity, 222); + assert_eq!(signal_channel_capacity, 111); + }); } - #[test] fn attr_partial() { - let attr: AttrArgs = parse_quote! { - gen=AllMessage, event=::some::why::ExternEvent, signal=::foo::SigSigSig, - }; - assert_matches!(attr, AttrArgs { - message_channel_capacity, - signal_channel_capacity, - .. - } => { - }); + let attr: AttrArgs = parse_quote! { + gen=AllMessage, event=::some::why::ExternEvent, signal=::foo::SigSigSig, + }; + assert_matches!(attr, AttrArgs { + message_channel_capacity: _, + signal_channel_capacity: _, + .. + } => { + }); } From 425140522aac49c3cf72caffe24e8439ff03cb8d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 26 May 2021 20:07:27 +0200 Subject: [PATCH 047/161] spelling --- node/overseer/overseer-gen/proc-macro/src/parse_struct.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 4caef9921ff4..2e6ed5bcf1b6 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -25,10 +25,10 @@ pub(crate) struct SubSysField { pub(crate) ty: Path, /// Type to be consumed by the subsystem. pub(crate) consumes: Path, - /// If `no_dispatch` is present, if the message is incomming via + /// If `no_dispatch` is present, if the message is incoming via /// an extern `Event`, it will not be dispatched to all subsystems. pub(crate) no_dispatch: bool, - /// If the subsystem imlementation is blocking execution and hence + /// If the subsystem implementation is blocking execution and hence /// has to be spawned on a separate thread or thread pool. pub(crate) blocking: bool, } @@ -110,7 +110,7 @@ pub(crate) struct OverseerInfo { /// Size of the bounded signal channel. pub(crate) signal_channel_capacity: usize, - /// Signals to be sent, sparse information that is used intermittendly. + /// Signals to be sent, sparse information that is used intermittently. pub(crate) extern_signal_ty: Path, /// Incoming event type from the outer world, commonly from the network. From 4bc034fbebc2dc6f2fea4b9e8731d7f28c0f82cf Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 26 May 2021 20:12:59 +0200 Subject: [PATCH 048/161] remove pointless vars --- .../proc-macro/src/impl_builder.rs | 15 ----- .../proc-macro/src/impl_dispatch.rs | 1 - .../overseer-gen/proc-macro/src/impl_misc.rs | 56 ------------------- .../proc-macro/src/impl_overseer.rs | 4 -- 4 files changed, 76 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 82177c4bd14a..b644a34daa9a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -65,9 +65,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result, #error_ty>, )* }; - let _message_wrapper = &info.message_wrapper; let event = &info.extern_event_ty; - let _signal = &info.extern_signal_ty; let mut ts = quote! { @@ -226,12 +224,6 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - {} -", - ts.to_string() - ); ts.extend(impl_task_kind(info)?); Ok(ts) } @@ -318,12 +310,5 @@ pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - {} -", - ts.to_string() - ); - Ok(ts) } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index d7d3ad2d0804..be8ff74cfa53 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -59,6 +59,5 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { } } }; - println!("{}", ts.to_string()); Ok(ts) } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index 3caaaef60d3b..6d6ea80c51f5 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -7,69 +7,13 @@ use super::*; /// which acts as the gateway to constructing the overseer. pub(crate) fn impl_misc(info: &OverseerInfo) -> Result { let overseer_name = info.overseer_name.clone(); - let _builder = Ident::new(&(overseer_name.to_string() + "Builder"), overseer_name.span()); - let _handler = Ident::new(&(overseer_name.to_string() + "Handler"), overseer_name.span()); - - let _subsystem_name = &info.subsystem_names(); let builder_generic_ty = &info.builder_generic_types(); - - let _channel_name = &info.channel_names(""); - let _channel_name_unbounded = &info.channel_names("_unbounded"); - - let _channel_name_tx = &info.channel_names("_tx"); - let _channel_name_unbounded_tx = &info.channel_names("_unbounded_tx"); - - let _channel_name_rx = &info.channel_names("_rx"); - let _channel_name_unbounded_rx = &info.channel_names("_unbounded_rx"); - - let baggage_generic_ty = &info.baggage_generic_types(); - let _baggage_name = &info.baggage_names(); - let _baggage_ty = &info.baggage_types(); - - let _blocking = &info - .subsystems() - .iter() - .map(|x| { - if x.blocking { - quote! { Blocking } - } else { - quote! { Regular } - } - }) - .collect::>(); - let consumes = &info.consumes(); let message_wrapper = &info.message_wrapper; - let _event = &info.extern_event_ty; let signal = &info.extern_signal_ty; - let _generics = quote! { - < S, #( #baggage_generic_ty, )* > - }; - let _where_clause = quote! { - where - S: ::polkadot_overseer_gen::SpawnNamed, - }; - - let _builder_generics = quote! { - - }; - - // all subsystems must have the same context - // even if the overseer does not impose such a limit. - let _builder_additional_generics = quote! { - < Ctx, #( #builder_generic_ty, )* > - }; - let error_ty = &info.extern_error_ty; - let _builder_where_clause = quote! { - where - Ctx: SubsystemContext< #message_wrapper > - S: ::polkadot_overseer_gen::SpawnNamed, - #( #builder_generic_ty : Subsystem, )* - }; - let ts = quote! { // ////////////////////////////////////////////////// // `OverseerSubsystemSender` diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 720706b2c037..bfb5e8f1d5f7 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -8,8 +8,6 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result Result Result { let signal = &info.extern_signal_ty; - let _message_wrapper = &info.message_wrapper; - let _consumes = &info.consumes(); let ts = quote::quote! { From 097c65aa1e1074e0c8caa085f2d746d71fd82ac1 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 27 May 2021 09:42:48 +0200 Subject: [PATCH 049/161] make sure all prefixes are there --- node/overseer/overseer-gen/examples/dummy.rs | 8 +++---- .../proc-macro/src/impl_builder.rs | 6 +++-- .../overseer-gen/proc-macro/src/impl_misc.rs | 24 +++++++++---------- .../proc-macro/src/impl_replace.rs | 2 +- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index dbbb0be6f36e..665afdf4edba 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -10,8 +10,8 @@ use polkadot_subsystem::messages::NetworkBridgeEvent; #[derive(Default)] pub struct AwesomeSubSys; -impl ::polkadot_overseer_gen::Subsystem, Yikes> for AwesomeSubSys { - fn start(self, ctx: OverseerSubsystemContext) -> SpawnedSubsystem < Yikes > { +impl ::polkadot_overseer_gen::Subsystem, Yikes> for AwesomeSubSys { + fn start(self, ctx: XxxSubsystemContext) -> SpawnedSubsystem < Yikes > { unimplemented!("starting yay!") } } @@ -19,8 +19,8 @@ impl ::polkadot_overseer_gen::Subsystem, Yik #[derive(Default)] pub struct GoblinTower; -impl ::polkadot_overseer_gen::Subsystem, Yikes> for GoblinTower { - fn start(self, ctx: OverseerSubsystemContext) -> SpawnedSubsystem < Yikes > { +impl ::polkadot_overseer_gen::Subsystem, Yikes> for GoblinTower { + fn start(self, ctx: XxxSubsystemContext) -> SpawnedSubsystem < Yikes > { unimplemented!("starting yay!") } } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index b644a34daa9a..5e64010aba54 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -59,10 +59,12 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result, #error_ty>, )* + #( #builder_generic_ty : Subsystem<#subsyste_ctx_name< #consumes >, #error_ty>, )* }; let event = &info.extern_event_ty; @@ -178,7 +180,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result::new( signal_rx, message_rx, channels_out.clone(), diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index 6d6ea80c51f5..27d577c27cba 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -7,26 +7,24 @@ use super::*; /// which acts as the gateway to constructing the overseer. pub(crate) fn impl_misc(info: &OverseerInfo) -> Result { let overseer_name = info.overseer_name.clone(); - let builder_generic_ty = &info.builder_generic_types(); + let subsyste_sender_name = Ident::new(&(overseer_name.to_string() + "SubsystemSender"), overseer_name.span()); + let subsyste_ctx_name = Ident::new(&(overseer_name.to_string() + "SubsystemContext"), overseer_name.span()); let consumes = &info.consumes(); - let message_wrapper = &info.message_wrapper; let signal = &info.extern_signal_ty; - let error_ty = &info.extern_error_ty; - let ts = quote! { // ////////////////////////////////////////////////// // `OverseerSubsystemSender` #[derive(Debug, Clone)] - pub struct OverseerSubsystemSender { + pub struct #subsyste_sender_name { channels: ChannelsOut, signals_received: SignalsReceived, } #( #[::polkadot_overseer_gen::async_trait] - impl SubsystemSender< #consumes > for OverseerSubsystemSender { + impl SubsystemSender< #consumes > for #subsyste_sender_name { async fn send_message(&mut self, msg: #consumes) { self.channels.send_and_log_error(self.signals_received.load(), msg.into()).await; } @@ -54,17 +52,17 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result /// [`Subsystem`]: trait.Subsystem.html /// [`SubsystemJob`]: trait.SubsystemJob.html #[derive(Debug)] - pub struct OverseerSubsystemContext{ + pub struct #subsyste_ctx_name{ signals: metered::MeteredReceiver< #signal >, messages: SubsystemIncomingMessages, - to_subsystems: OverseerSubsystemSender, + to_subsystems: #subsyste_sender_name, to_overseer: metered::UnboundedMeteredSender, signals_received: SignalsReceived, pending_incoming: Option<(usize, M)>, } - impl OverseerSubsystemContext { - /// Create a new `OverseerSubsystemContext`. + impl #subsyste_ctx_name { + /// Create a new context. fn new( signals: metered::MeteredReceiver< #signal >, messages: SubsystemIncomingMessages, @@ -72,10 +70,10 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result to_overseer: metered::UnboundedMeteredSender, ) -> Self { let signals_received = SignalsReceived::default(); - OverseerSubsystemContext { + #subsyste_ctx_name { signals, messages, - to_subsystems: OverseerSubsystemSender { + to_subsystems: #subsyste_sender_name { channels: to_subsystems, signals_received: signals_received.clone(), }, @@ -87,7 +85,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result } #[::polkadot_overseer_gen::async_trait] - impl SubsystemContext for OverseerSubsystemContext + impl SubsystemContext for #subsyste_ctx_name where OverseerSubsystemSender: polkadot_overseer_gen::SubsystemSender { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs index 29162226e99b..f3298b7dc1c1 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs @@ -34,7 +34,7 @@ pub(crate) fn impl_replacable_subsystem( for SubSysField { name: replacable_item, ty: _replacable_item_ty, generic, .. } in info.subsystems.iter() { let keeper = info.subsystems.iter().filter(|&ssf| ssf.generic != *generic).map(|ssf| ssf.name.clone()); - let fn_name = Ident::new(&format!("replace_{}", replacable_item), replacable_item.span()); + let fn_name = Ident::new(format!("replace_{}", replacable_item).as_str(), replacable_item.span()); // adjust the generics such that the appropriate member type is replaced let new = Ident::new("NEW", replacable_item.span()); From 2ad93bebdb0bacc3e6240dcfca41bc94ba692d8b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 27 May 2021 15:11:42 +0200 Subject: [PATCH 050/161] more fixins --- node/overseer/overseer-gen/examples/dummy.rs | 2 +- .../proc-macro/src/impl_builder.rs | 4 +- .../proc-macro/src/impl_message_wrapper.rs | 2 +- .../overseer-gen/proc-macro/src/impl_misc.rs | 45 ++- .../proc-macro/src/impl_overseer.rs | 46 ++- node/overseer/overseer-gen/src/lib.rs | 35 +- node/overseer/src/lib.rs | 54 +-- node/overseer/src/subsystems.rs | 24 +- node/overseer/src/tests.rs | 2 +- node/subsystem-util/src/lib.rs | 326 +++--------------- node/subsystem-util/src/runtime/mod.rs | 40 ++- node/subsystem-util/src/tests.rs | 246 +++++++++++++ 12 files changed, 443 insertions(+), 383 deletions(-) create mode 100644 node/subsystem-util/src/tests.rs diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index 665afdf4edba..5f9bcccc91ee 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -21,7 +21,7 @@ pub struct GoblinTower; impl ::polkadot_overseer_gen::Subsystem, Yikes> for GoblinTower { fn start(self, ctx: XxxSubsystemContext) -> SpawnedSubsystem < Yikes > { - unimplemented!("starting yay!") + unimplemented!("welcum") } } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 5e64010aba54..afeb99a88dd3 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -235,7 +235,7 @@ pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result Result, - E: std::error::Error + Send + Sync + 'static + From, + E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::SubsystemError>, SubSys: ::polkadot_overseer_gen::Subsystem, { let ::polkadot_overseer_gen::SpawnedSubsystem { future, name } = s.start(ctx); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index bf4ce1b58a5a..38e342ce298d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -11,7 +11,7 @@ pub(crate) fn impl_message_wrapper_enum(info: &OverseerInfo) -> Result Result let signal = &info.extern_signal_ty; let ts = quote! { - // ////////////////////////////////////////////////// - // `OverseerSubsystemSender` - #[derive(Debug, Clone)] pub struct #subsyste_sender_name { channels: ChannelsOut, @@ -53,10 +50,12 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result /// [`SubsystemJob`]: trait.SubsystemJob.html #[derive(Debug)] pub struct #subsyste_ctx_name{ - signals: metered::MeteredReceiver< #signal >, + signals: ::polkadot_overseer_gen::metered::MeteredReceiver< #signal >, messages: SubsystemIncomingMessages, to_subsystems: #subsyste_sender_name, - to_overseer: metered::UnboundedMeteredSender, + to_overseer: ::polkadot_overseer_gen::metered::UnboundedMeteredSender< + ::polkadot_overseer_gen::ToOverseer + >, signals_received: SignalsReceived, pending_incoming: Option<(usize, M)>, } @@ -64,10 +63,10 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result impl #subsyste_ctx_name { /// Create a new context. fn new( - signals: metered::MeteredReceiver< #signal >, + signals: ::polkadot_overseer_gen::metered::MeteredReceiver< #signal >, messages: SubsystemIncomingMessages, to_subsystems: ChannelsOut, - to_overseer: metered::UnboundedMeteredSender, + to_overseer: ::polkadot_overseer_gen::metered::UnboundedMeteredSender, ) -> Self { let signals_received = SignalsReceived::default(); #subsyste_ctx_name { @@ -87,11 +86,11 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result #[::polkadot_overseer_gen::async_trait] impl SubsystemContext for #subsyste_ctx_name where - OverseerSubsystemSender: polkadot_overseer_gen::SubsystemSender + #subsyste_sender_name: polkadot_overseer_gen::SubsystemSender, { type Message = M; type Signal = #signal; - type Sender = OverseerSubsystemSender; + type Sender = #subsyste_sender_name; async fn try_recv(&mut self) -> Result>, ()> { match ::polkadot_overseer_gen::poll!(self.recv()) { @@ -100,25 +99,25 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result } } - async fn recv(&mut self) -> SubsystemResult> { + async fn recv(&mut self) -> ::polkadot_overseer_gen::SubsystemResult> { loop { // If we have a message pending an overseer signal, we only poll for signals // in the meantime. if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { if needs_signals_received <= self.signals_received.load() { - return Ok(FromOverseer::Communication { msg }); + return Ok(::polkadot_overseer_gen::FromOverseer::Communication { msg }); } else { self.pending_incoming = Some((needs_signals_received, msg)); // wait for next signal. let signal = self.signals.next().await - .ok_or(SubsystemError::Context( + .ok_or(::polkadot_overseer_gen::SubsystemError::Context( "Signal channel is terminated and empty." .to_owned() ))?; self.signals_received.inc(); - return Ok(FromOverseer::Signal(signal)) + return Ok(::polkadot_overseer_gen::FromOverseer::Signal(signal)) } } @@ -128,19 +127,19 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result let pending_incoming = &mut self.pending_incoming; // Otherwise, wait for the next signal or incoming message. - let from_overseer = futures::select_biased! { + let from_overseer = ::polkadot_overseer_gen::futures::select_biased! { signal = await_signal => { let signal = signal - .ok_or(SubsystemError::Context( + .ok_or(::polkadot_overseer_gen::SubsystemError::Context( "Signal channel is terminated and empty." .to_owned() ))?; - FromOverseer::Signal(signal) + ::polkadot_overseer_gen::FromOverseer::Signal(signal) } msg = await_message => { let packet = msg - .ok_or(SubsystemError::Context( + .ok_or(::polkadot_overseer_gen::SubsystemError::Context( "Message channel is terminated and empty." .to_owned() ))?; @@ -151,12 +150,12 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result continue; } else { // we know enough to return this message. - FromOverseer::Communication { msg: packet.message} + ::polkadot_overseer_gen::FromOverseer::Communication { msg: packet.message} } } }; - if let FromOverseer::Signal(_) = from_overseer { + if let ::polkadot_overseer_gen::FromOverseer::Signal(_) = from_overseer { self.signals_received.inc(); } @@ -170,9 +169,9 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn(ctx, name, fut)")] async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) - -> SubsystemResult<()> + -> ::polkadot_overseer_gen::SubsystemResult<()> { - self.to_overseer.send(ToOverseer::SpawnJob { + self.to_overseer.send(::polkadot_overseer_gen::ToOverseer::SpawnJob { name, s, }).await.map_err(Into::into) @@ -180,9 +179,9 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn_blocking(ctx, name, fut)")] async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) - -> SubsystemResult<()> + -> ::polkadot_overseer_gen::SubsystemResult<()> { - self.to_overseer.send(ToOverseer::SpawnBlockingJob { + self.to_overseer.send(::polkadot_overseer_gen::ToOverseer::SpawnBlockingJob { name, s, }).await.map_err(Into::into) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index bfb5e8f1d5f7..8012cb1678df 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -11,7 +11,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result @@ -21,7 +21,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result Result SubsystemResult<()> { + /// Send the given signal, a terminatin signal, to all subsystems + /// and wait for all subsystems to go down. + /// + /// The definition of a termination signal is up to the user and + /// implementation specific. + pub async fn wait_terminate(&mut self, signal: #signal_ty, timeout: ::std::time::Duration) -> ::polkadot_overseer_gen::SubsystemResult<()> { + #( + self. #subsystem_name .send_signal(signal.clone()).await; + )* + let _ = signal; + + let mut timeout_fut = ::polkadot_overseer_gen::Delay::new( + timeout + ).fuse(); + + loop { + select! { + _ = self.running_subsystems.next() => { + if self.running_subsystems.is_empty() { + break; + } + }, + _ = timeout_fut => break, + complete => break, + } + } + + Ok(()) + } + + pub async fn broadcast_signal(&mut self, signal: #signal_ty) -> ::polkadot_overseer_gen::SubsystemResult<()> { #( self. #subsystem_name .send_signal(signal.clone()).await; )* @@ -86,7 +116,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result SubsystemResult<()> { + pub async fn route_message(&mut self, message: #message_wrapper) -> ::polkadot_overseer_gen::SubsystemResult<()> { match message { #( #message_wrapper :: #consumes ( inner ) => self. #subsystem_name .send_message( inner ).await?, @@ -122,7 +152,7 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result SubsystemResult<()> { + pub async fn send_message(&mut self, message: M) -> ::polkadot_overseer_gen::SubsystemResult<()> { const MESSAGE_TIMEOUT: Duration = Duration::from_secs(10); if let Some(ref mut instance) = self.instance { @@ -132,7 +162,7 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { - Err(SubsystemError::SubsystemStalled(instance.name)) + Err(::polkadot_overseer_gen::SubsystemError::SubsystemStalled(instance.name)) } Some(res) => res.map_err(Into::into), } @@ -145,12 +175,12 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result SubsystemResult<()> { - const SIGNAL_TIMEOUT: Duration = Duration::from_secs(10); + const SIGNAL_TIMEOUT: ::std::time::Duration = ::std::time::Duration::from_secs(10); if let Some(ref mut instance) = self.instance { match instance.tx_signal.send(signal).timeout(SIGNAL_TIMEOUT).await { None => { - Err(SubsystemError::SubsystemStalled(instance.name)) + Err(::polkadot_overseer_gen::SubsystemError::SubsystemStalled(instance.name)) } Some(res) => { let res = res.map_err(Into::into); diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 11db02ef4402..c5dd160edf82 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -69,6 +69,7 @@ pub use sp_core::traits::SpawnNamed; #[doc(hidden)] pub use futures::{ + self, select, poll, future::{ @@ -77,9 +78,14 @@ pub use futures::{ stream::{ self, select, FuturesUnordered, }, + task::{ + Poll, Context, + }, channel::{mpsc, oneshot}, }; #[doc(hidden)] +pub use std::pin::Pin; +#[doc(hidden)] pub use async_trait::async_trait; #[doc(hidden)] @@ -94,7 +100,6 @@ pub use futures_util::stream::StreamExt; #[doc(hidden)] pub use futures_util::future::FutureExt; - /// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. /// /// Used to launch jobs. @@ -216,11 +221,12 @@ pub trait AnnotateErrorOrigin: 'static + Send + Sync + std::error::Error { /// /// In essence it's just a newtype wrapping a `BoxFuture`. pub struct SpawnedSubsystem -where -E: std::error::Error - + 'static - + Send - + From + where + E: std::error::Error + + Send + + Sync + + 'static + + From { /// Name of the subsystem being spawned. pub name: &'static str, @@ -350,9 +356,9 @@ pub enum FromOverseer { pub trait SubsystemContext: Send + 'static { /// The message type of this context. Subsystems launched with this context will expect /// to receive messages of this type. - type Message: Send + 'static; + type Message: std::fmt::Debug + Send + 'static; /// And the same for signals. - type Signal: Send + 'static; + type Signal: std::fmt::Debug + Send + 'static; /// The sender type as provided by `sender()` and underlying. type Sender: SubsystemSender + std::fmt::Debug + Clone + Send; @@ -412,8 +418,6 @@ where /// TODO FIXME #[async_trait::async_trait] pub trait SubsystemSender: Send + Clone + 'static { - // Inner message type. - /// Send a direct message to some other `Subsystem`, routed based on message type. async fn send_message(&mut self, msg: M); @@ -429,13 +433,6 @@ pub trait SubsystemSender: Send + Clone + 'static { fn send_unbounded_message(&mut self, msg: M); } - - - - -pub use futures::task::{Poll, Context}; -pub use std::pin::Pin; - /// A future that wraps another future with a `Delay` allowing for time-limited futures. #[pin_project::pin_project] pub struct Timeout { @@ -460,9 +457,9 @@ pub trait TimeoutExt: Future { } } -impl TimeoutExt for F {} +impl TimeoutExt for F where F: Future{} -impl Future for Timeout { +impl Future for Timeout where F: Future { type Output = Option; fn poll(self: Pin<&mut Self>, ctx: &mut Context) -> Poll { diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 22e9b4d0b115..9c0f1f236660 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -65,6 +65,7 @@ use std::sync::{atomic::{self, AtomicUsize}, Arc}; use std::task::Poll; use std::time::Duration; use std::collections::{hash_map, HashMap}; +use std::iter::FromIterator; use futures::channel::oneshot; use futures::{ @@ -80,7 +81,6 @@ use client::{BlockImportNotification, BlockchainEvents, FinalityNotification}; use sp_api::{ApiExt, ProvideRuntimeApi}; use polkadot_subsystem::messages::{ - NetworkBridgeMessage, CandidateValidationMessage, CandidateBackingMessage, CandidateSelectionMessage, ChainApiMessage, StatementDistributionMessage, AvailabilityDistributionMessage, BitfieldSigningMessage, BitfieldDistributionMessage, @@ -91,7 +91,7 @@ use polkadot_subsystem::messages::{ }; pub use polkadot_subsystem::{ OverseerSignal, - SubsystemResult, ActiveLeavesUpdate, ActivatedLeaf, DummySubsystem, jaeger, + SubsystemResult, ActiveLeavesUpdate, ActivatedLeaf, jaeger, }; /// TODO legacy, to be deleted, left for easier integration @@ -101,8 +101,10 @@ use self::subsystems::{AllSubsystems, AllSubsystemsSame}; mod metrics; use self::metrics::Metrics; -use polkadot_node_subsystem_util::{TimeoutExt, metrics::{prometheus, Metrics as MetricsTrait}, metered, Metronome}; +use polkadot_overseer_gen::*; +use polkadot_node_subsystem_util::{metrics::{prometheus, Metrics as MetricsTrait}, metered, Metronome}; use polkadot_overseer_gen::{ + TimeoutExt, SpawnNamed, SpawnedSubsystem, SubsystemError, @@ -113,6 +115,7 @@ use polkadot_overseer_gen::{ MessagePacket, make_packet, SignalsReceived, + FromOverseer, }; // Target for logs. @@ -425,6 +428,7 @@ pub struct Overseer { impl Overseer where SupportsParachains: HeadSupportsParachains, + S: SpawnNamed, { /// Create a new instance of the `Overseer` with a fixed set of [`Subsystem`]s. /// @@ -516,32 +520,34 @@ where /// # /// # }); } /// ``` - pub fn new( + pub fn new( leaves: impl IntoIterator, all_subsystems: AllSubsystems, prometheus_registry: Option<&prometheus::Registry>, supports_parachains: SupportsParachains, mut s: S, - ) -> SubsystemResult<(Self, OverseerHandler)> + ) -> ::polkadot_overseer_gen::SubsystemResult<(Self, OverseerHandler)> where - CV: Subsystem> + Send, - CB: Subsystem> + Send, - CS: Subsystem> + Send, - SD: Subsystem> + Send, - AD: Subsystem> + Send, - AR: Subsystem> + Send, - BS: Subsystem> + Send, - BD: Subsystem> + Send, - P: Subsystem> + Send, - RA: Subsystem> + Send, - AS: Subsystem> + Send, - NB: Subsystem> + Send, - CA: Subsystem> + Send, - CG: Subsystem> + Send, - CP: Subsystem> + Send, - ApD: Subsystem> + Send, - ApV: Subsystem> + Send, - GS: Subsystem> + Send, + CV: Subsystem, E> + Send, + CB: Subsystem, E> + Send, + CS: Subsystem, E> + Send, + SD: Subsystem, E> + Send, + AD: Subsystem, E> + Send, + AR: Subsystem, E> + Send, + BS: Subsystem, E> + Send, + BD: Subsystem, E> + Send, + P: Subsystem, E> + Send, + RA: Subsystem, E> + Send, + AS: Subsystem, E> + Send, + NB: Subsystem, E> + Send, + CA: Subsystem, E> + Send, + CG: Subsystem, E> + Send, + CP: Subsystem, E> + Send, + ApD: Subsystem, E> + Send, + ApV: Subsystem, E> + Send, + GS: Subsystem, E> + Send, + E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::SubsystemError>, + S: SpawnNamed, { let metrics: Metrics = ::register(prometheus_registry)?; @@ -614,7 +620,7 @@ where // Stop the overseer. async fn stop(mut self) { - self.wait_for_terminate(OverseerSignal::Conclude).await + let _ = self.wait_terminate(OverseerSignal::Conclude, ::std::time::Duration::from_secs(1_u64)).await; } /// Run the `Overseer`. diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index 86d0ca8054b0..d6b910078ccd 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -1,17 +1,25 @@ -use ::polkadot_overseer_gen::MapSubsystem; -use crate::SubsystemContext; -use crate::Subsystem; +use ::polkadot_overseer_gen::{ + MapSubsystem, SubsystemContext, + Subsystem, + SpawnedSubsystem, + FromOverseer, +}; + +use crate::OverseerSignal; /// A dummy subsystem that implements [`Subsystem`] for all /// types of messages. Used for tests or as a placeholder. +#[derive(Clone, Copy, Debug)] pub struct DummySubsystem; -impl Subsystem for DummySubsystem +impl Subsystem for DummySubsystem where - C::Message: std::fmt::Debug + Context: SubsystemContext, + ::Message: std::fmt::Debug + Send + 'static, + E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::SubsystemError>, { - fn start(self, mut ctx: C) -> SpawnedSubsystem { + fn start(self, mut ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { loop { match ctx.recv().await { @@ -146,7 +154,7 @@ impl } } - fn as_ref(&self) -> AllSubsystems<&'_ CV, &'_ CB, &'_ CS, &'_ SD, &'_ AD, &'_ AR, &'_ BS, &'_ BD, &'_ P, &'_ RA, &'_ AS, &'_ NB, &'_ CA, &'_ CG, &'_ CP, &'_ ApD, &'_ ApV, &'_ GS> { + pub fn as_ref(&self) -> AllSubsystems<&'_ CV, &'_ CB, &'_ CS, &'_ SD, &'_ AD, &'_ AR, &'_ BS, &'_ BD, &'_ P, &'_ RA, &'_ AS, &'_ NB, &'_ CA, &'_ CG, &'_ CP, &'_ ApD, &'_ ApV, &'_ GS> { AllSubsystems { candidate_validation: &self.candidate_validation, candidate_backing: &self.candidate_backing, @@ -169,7 +177,7 @@ impl } } - fn map_subsystems(self, m: M) + pub fn map_subsystems(self, m: M) -> AllSubsystems< >::Output, >::Output, diff --git a/node/overseer/src/tests.rs b/node/overseer/src/tests.rs index cc427f2bbf78..66d27d189574 100644 --- a/node/overseer/src/tests.rs +++ b/node/overseer/src/tests.rs @@ -46,7 +46,7 @@ struct TestSubsystem2(metered::MeteredSender); impl Subsystem for TestSubsystem2 where C: SubsystemContext { - fn start(self, mut ctx: C) -> SpawnedSubsystem { + fn start(self, mut ctx: C) -> SpawnedSubsystem<::::polkadot_overseer_gen::SubsystemError> { let sender = self.0.clone(); SpawnedSubsystem { name: "test-subsystem-2", diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 575a9e42a337..029335bfa7a0 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -30,7 +30,7 @@ use polkadot_node_subsystem::{ ActiveLeavesUpdate, OverseerSignal, }; -use polkadot_overseer_gen::{ +pub use polkadot_overseer_gen::{ FromOverseer, SpawnedSubsystem, Subsystem, @@ -121,15 +121,15 @@ pub enum Error { pub type RuntimeApiReceiver = oneshot::Receiver>; /// Request some data from the `RuntimeApi`. -pub async fn request_from_runtime( +pub async fn request_from_runtime( parent: Hash, sender: &mut Sender, request_builder: RequestBuilder, ) -> RuntimeApiReceiver where RequestBuilder: FnOnce(RuntimeApiSender) -> RuntimeApiRequest, - Sender: SubsystemSender, - AllMessages: From, + Sender: SubsystemSender, + M: From, { let (tx, rx) = oneshot::channel(); @@ -154,14 +154,18 @@ macro_rules! specialize_requests { #[doc = "Request `"] #[doc = $doc_name] #[doc = "` from the runtime"] - pub async fn $func_name > ( + pub async fn $func_name ( parent: Hash, $( $param_name: $param_ty, )* - sender: &mut impl SubsystemSender , - ) -> RuntimeApiReceiver<$return_ty> { - request_from_runtime::<_, _, _, AllMessages>(parent, sender, |tx| RuntimeApiRequest::$request_variant( + sender: &mut Sender, + ) -> RuntimeApiReceiver<$return_ty> + where + M: From, + Sender: SubsystemSender, + { + request_from_runtime::<_, _, Sender, M>(parent, sender, |tx| RuntimeApiRequest::$request_variant( $( $param_name, )* tx )).await } @@ -268,13 +272,13 @@ pub struct Validator { impl Validator { /// Get a struct representing this node's validator if this node is in fact a validator in the context of the given block. - pub async fn new( + pub async fn new( parent: Hash, keystore: SyncCryptoStorePtr, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender , ) -> Result where - AllMessages: From, + M: From, { // Note: request_validators and request_session_index_for_child do not and cannot // run concurrently: they both have a mutable handle to the same sender. @@ -404,26 +408,26 @@ pub enum FromJobCommand { /// A sender for messages from jobs, as well as commands to the overseer. #[derive(Clone)] -pub struct JobSender, AllMessages> { +pub struct JobSender, M> { sender: S, from_job: mpsc::Sender, - _phantom: std::marker::PhantomData, + _phantom: std::marker::PhantomData, } -impl, AllMessages> JobSender { +impl, M> JobSender { /// Get access to the underlying subsystem sender. pub fn subsystem_sender(&mut self) -> &mut S { &mut self.sender } /// Send a direct message to some other `Subsystem`, routed based on message type. - pub async fn send_message(&mut self, msg: AllMessages) { + pub async fn send_message(&mut self, msg: M) { self.sender.send_message(msg).await } /// Send multiple direct messages to other `Subsystem`s, routed based on message type. pub async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send + where T: IntoIterator + Send, T::IntoIter: Send { self.sender.send_messages(msgs).await } @@ -434,7 +438,7 @@ impl, AllMessages> JobSender { /// /// This function should be used only when there is some other bounding factor on the messages /// sent with it. Otherwise, it risks a memory leak. - pub fn send_unbounded_message(&mut self, msg: AllMessages) { + pub fn send_unbounded_message(&mut self, msg: M) { self.sender.send_unbounded_message(msg) } @@ -445,21 +449,22 @@ impl, AllMessages> JobSender { } #[async_trait::async_trait] -impl, AllMessages> SubsystemSender for JobSender +impl SubsystemSender for JobSender where - AllMessages: Clone + Send + 'static + M: Clone + Send + 'static, + S: SubsystemSender { - async fn send_message(&mut self, msg: AllMessages) { + async fn send_message(&mut self, msg: M) { self.sender.send_message(msg).await } async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send + where T: IntoIterator + Send, T::IntoIter: Send { self.sender.send_messages(msgs).await } - fn send_unbounded_message(&mut self, msg: AllMessages) { + fn send_unbounded_message(&mut self, msg: M) { self.sender.send_unbounded_message(msg) } } @@ -493,22 +498,19 @@ pub trait JobTrait: Unpin + Sized { /// The `delegate_subsystem!` macro should take care of this. type Metrics: 'static + metrics::Metrics + Send; - /// The generated wrapping message type. - type Message: 'static + Send; - /// Name of the job, i.e. `CandidateBackingJob` const NAME: &'static str; /// Run a job for the given relay `parent`. /// /// The job should be ended when `receiver` returns `None`. - fn run>( + fn run>( parent: Hash, span: Arc, run_args: Self::RunArgs, metrics: Self::Metrics, receiver: mpsc::Receiver, - sender: JobSender, + sender: JobSender, ) -> Pin> + Send>>; } @@ -539,7 +541,11 @@ struct Jobs { outgoing_msgs: StreamUnordered>, } -impl Jobs { +impl Jobs +where + Spawner: SpawnNamed, + ToJob: Send + 'static, +{ /// Create a new Jobs manager which handles spawning appropriate jobs. pub fn new(spawner: Spawner) -> Self { Self { @@ -550,7 +556,7 @@ impl Jobs { } /// Spawn a new job for this `parent_hash`, with whatever args are appropriate. - fn spawn_job( + fn spawn_job( &mut self, parent_hash: Hash, span: Arc, @@ -560,7 +566,7 @@ impl Jobs { ) where Job: JobTrait, - Sender: SubsystemSender, + Sender: SubsystemSender, { let (to_job_tx, to_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY); let (from_job_tx, from_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY); @@ -575,6 +581,7 @@ impl Jobs { JobSender { sender, from_job: from_job_tx, + _phantom: Default::default(), }, ).await { tracing::error!( @@ -687,12 +694,11 @@ impl JobSubsystem { pub async fn run(self, mut ctx: Context) where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, - Context: SubsystemContext, - Job: 'static + JobTrait::Message> + Send, + Context: SubsystemContext::ToJob, Signal=OverseerSignal>, + Job: 'static + JobTrait + Send, Job::RunArgs: Clone + Sync, - Job::ToJob: From<::Message> + Sync, + Job::ToJob: From, Job::Metrics: Sync, - ::Message: From, { let JobSubsystem { params: JobSubsystemParams { @@ -703,7 +709,7 @@ impl JobSubsystem { .. } = self; - let mut jobs = Jobs::::new(spawner); + let mut jobs = Jobs::::new(spawner); loop { select! { @@ -715,7 +721,7 @@ impl JobSubsystem { }))) => { for activated in activated { let sender: Context::Sender = ctx.sender().clone(); - jobs.spawn_job::( + jobs.spawn_job::( activated.hash, activated.span, run_args.clone(), @@ -768,10 +774,10 @@ impl JobSubsystem { impl Subsystem for JobSubsystem where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, - Context: SubsystemContext, + Context: SubsystemContext, Job: 'static + JobTrait + Send, Job::RunArgs: Clone + Sync, - Job::ToJob: From<::Message> + Sync, + Job::ToJob: Sync + From, Job::Metrics: Sync, { fn start(self, ctx: Context) -> SpawnedSubsystem { @@ -842,246 +848,4 @@ impl futures::Stream for Metronome } #[cfg(test)] -mod tests { - use super::*; - use executor::block_on; - use thiserror::Error; - use polkadot_node_jaeger as jaeger; - use polkadot_node_subsystem::{ - messages::CandidateSelectionMessage, ActiveLeavesUpdate, - ActivatedLeaf, - }; - use assert_matches::assert_matches; - use futures::{channel::mpsc, executor, StreamExt, future, Future, FutureExt, SinkExt}; - use polkadot_primitives::v1::Hash; - use polkadot_node_subsystem_test_helpers::{self as test_helpers, make_subsystem_context}; - use std::{pin::Pin, sync::{Arc, atomic::{AtomicUsize, Ordering}}, time::Duration}; - - // basic usage: in a nutshell, when you want to define a subsystem, just focus on what its jobs do; - // you can leave the subsystem itself to the job manager. - - // for purposes of demonstration, we're going to whip up a fake subsystem. - // this will 'select' candidates which are pre-loaded in the job - - // job structs are constructed within JobTrait::run - // most will want to retain the sender and receiver, as well as whatever other data they like - struct FakeCandidateSelectionJob { - receiver: mpsc::Receiver, - } - - // Error will mostly be a wrapper to make the try operator more convenient; - // deriving From implementations for most variants is recommended. - // It must implement Debug for logging. - #[derive(Debug, Error)] - enum Error { - #[error(transparent)] - Sending(#[from]mpsc::SendError), - } - - impl JobTrait for FakeCandidateSelectionJob { - type ToJob = CandidateSelectionMessage; - type Error = Error; - type RunArgs = bool; - type Metrics = (); - - const NAME: &'static str = "FakeCandidateSelectionJob"; - - /// Run a job for the parent block indicated - // - // this function is in charge of creating and executing the job's main loop - fn run( - _: Hash, - _: Arc, - run_args: Self::RunArgs, - _metrics: Self::Metrics, - receiver: mpsc::Receiver, - mut sender: JobSender, - ) -> Pin> + Send>> { - async move { - let job = FakeCandidateSelectionJob { receiver }; - - if run_args { - sender.send_message(CandidateSelectionMessage::Invalid( - Default::default(), - Default::default(), - ).into()).await; - } - - // it isn't necessary to break run_loop into its own function, - // but it's convenient to separate the concerns in this way - job.run_loop().await - } - .boxed() - } - } - - impl FakeCandidateSelectionJob { - async fn run_loop(mut self) -> Result<(), Error> { - loop { - match self.receiver.next().await { - Some(_csm) => { - unimplemented!("we'd report the collator to the peer set manager here, but that's not implemented yet"); - } - None => break, - } - } - - Ok(()) - } - } - - // with the job defined, it's straightforward to get a subsystem implementation. - type FakeCandidateSelectionSubsystem = - JobSubsystem; - - // this type lets us pretend to be the overseer - type OverseerHandle = test_helpers::TestSubsystemContextHandle; - - fn test_harness>( - run_args: bool, - test: impl FnOnce(OverseerHandle) -> T, - ) { - let _ = env_logger::builder() - .is_test(true) - .filter( - None, - log::LevelFilter::Trace, - ) - .try_init(); - - let pool = sp_core::testing::TaskExecutor::new(); - let (context, overseer_handle) = make_subsystem_context(pool.clone()); - - let subsystem = FakeCandidateSelectionSubsystem::new( - pool, - run_args, - (), - ).run(context); - let test_future = test(overseer_handle); - - futures::pin_mut!(subsystem, test_future); - - executor::block_on(async move { - future::join(subsystem, test_future) - .timeout(Duration::from_secs(2)) - .await - .expect("test timed out instead of completing") - }); - } - - #[test] - fn starting_and_stopping_job_works() { - let relay_parent: Hash = [0; 32].into(); - - test_harness(true, |mut overseer_handle| async move { - overseer_handle - .send(FromOverseer::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: relay_parent, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - }), - ))) - .await; - assert_matches!( - overseer_handle.recv().await, - AllMessages::CandidateSelection(_) - ); - overseer_handle - .send(FromOverseer::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::stop_work(relay_parent), - ))) - .await; - - overseer_handle - .send(FromOverseer::Signal(OverseerSignal::Conclude)) - .await; - }); - } - - #[test] - fn sending_to_a_non_running_job_do_not_stop_the_subsystem() { - let relay_parent = Hash::repeat_byte(0x01); - - test_harness(true, |mut overseer_handle| async move { - overseer_handle - .send(FromOverseer::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: relay_parent, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - }), - ))) - .await; - - // send to a non running job - overseer_handle - .send(FromOverseer::Communication { - msg: Default::default(), - }) - .await; - - // the subsystem is still alive - assert_matches!( - overseer_handle.recv().await, - AllMessages::CandidateSelection(_) - ); - - overseer_handle - .send(FromOverseer::Signal(OverseerSignal::Conclude)) - .await; - }); - } - - #[test] - fn test_subsystem_impl_and_name_derivation() { - let pool = sp_core::testing::TaskExecutor::new(); - let (context, _) = make_subsystem_context::(pool.clone()); - - let SpawnedSubsystem { name, .. } = - FakeCandidateSelectionSubsystem::new(pool, false, ()).start(context); - assert_eq!(name, "FakeCandidateSelection"); - } - - - #[test] - fn tick_tack_metronome() { - let n = Arc::new(AtomicUsize::default()); - - let (tick, mut block) = mpsc::unbounded(); - - let metronome = { - let n = n.clone(); - let stream = Metronome::new(Duration::from_millis(137_u64)); - stream.for_each(move |_res| { - let _ = n.fetch_add(1, Ordering::Relaxed); - let mut tick = tick.clone(); - async move { - tick.send(()).await.expect("Test helper channel works. qed"); - } - }).fuse() - }; - - let f2 = async move { - block.next().await; - assert_eq!(n.load(Ordering::Relaxed), 1_usize); - block.next().await; - assert_eq!(n.load(Ordering::Relaxed), 2_usize); - block.next().await; - assert_eq!(n.load(Ordering::Relaxed), 3_usize); - block.next().await; - assert_eq!(n.load(Ordering::Relaxed), 4_usize); - }.fuse(); - - futures::pin_mut!(f2); - futures::pin_mut!(metronome); - - block_on(async move { - // futures::join!(metronome, f2) - futures::select!( - _ = metronome => unreachable!("Metronome never stops. qed"), - _ = f2 => (), - ) - }); - } -} +mod tests; diff --git a/node/subsystem-util/src/runtime/mod.rs b/node/subsystem-util/src/runtime/mod.rs index 484df7b6cee1..5daeef91e844 100644 --- a/node/subsystem-util/src/runtime/mod.rs +++ b/node/subsystem-util/src/runtime/mod.rs @@ -25,6 +25,9 @@ use sp_keystore::{CryptoStore, SyncCryptoStorePtr}; use polkadot_primitives::v1::{CoreState, EncodeAs, GroupIndex, GroupRotationInfo, Hash, OccupiedCore, SessionIndex, SessionInfo, Signed, SigningContext, UncheckedSigned, ValidatorId, ValidatorIndex}; use polkadot_overseer_gen::SubsystemContext; +use polkadot_node_subsystem::messages::RuntimeApiMessage; +use polkadot_node_subsystem::OverseerSignal; + use crate::{ request_session_index_for_child, request_session_info, @@ -86,13 +89,14 @@ impl RuntimeInfo { } /// Retrieve the current session index. - pub async fn get_session_index( + pub async fn get_session_index( &mut self, ctx: &mut Context, parent: Hash, ) -> Result where - Context: SubsystemContext, + M: From, + Context: SubsystemContext, { match self.session_index_cache.get(&parent) { Some(index) => Ok(*index), @@ -107,13 +111,14 @@ impl RuntimeInfo { } /// Get `ExtendedSessionInfo` by relay parent hash. - pub async fn get_session_info<'a, Context>( + pub async fn get_session_info<'a, Context, M>( &'a mut self, ctx: &mut Context, parent: Hash, ) -> Result<&'a ExtendedSessionInfo> where - Context: SubsystemContext, + M: From, + Context: SubsystemContext, { let session_index = self.get_session_index(ctx, parent).await?; @@ -124,14 +129,15 @@ impl RuntimeInfo { /// /// `request_session_info` still requires the parent to be passed in, so we take the parent /// in addition to the `SessionIndex`. - pub async fn get_session_info_by_index<'a, Context>( + pub async fn get_session_info_by_index<'a, Context, M>( &'a mut self, ctx: &mut Context, parent: Hash, session_index: SessionIndex, ) -> Result<&'a ExtendedSessionInfo> where - Context: SubsystemContext, + M: From, + Context: SubsystemContext, { if !self.session_info_cache.contains(&session_index) { let session_info = @@ -154,14 +160,15 @@ impl RuntimeInfo { } /// Convenience function for checking the signature of something signed. - pub async fn check_signature( + pub async fn check_signature( &mut self, ctx: &mut Context, parent: Hash, signed: UncheckedSigned, ) -> Result, UncheckedSigned>> where - Context: SubsystemContext, + M: From, + Context: SubsystemContext, Payload: EncodeAs + Clone, RealPayload: Encode + Clone, { @@ -242,21 +249,23 @@ where } /// Request availability cores from the runtime. -pub async fn get_availability_cores(ctx: &mut Context, relay_parent: Hash) +pub async fn get_availability_cores(ctx: &mut Context, relay_parent: Hash) -> Result> where - Context: SubsystemContext, + M: From, + Context: SubsystemContext, { recv_runtime(request_availability_cores(relay_parent, ctx.sender()).await).await } /// Variant of `request_availability_cores` that only returns occupied ones. -pub async fn get_occupied_cores( +pub async fn get_occupied_cores( ctx: &mut Context, relay_parent: Hash, ) -> Result> where - Context: SubsystemContext, +M: From, +Context: SubsystemContext, { let cores = get_availability_cores(ctx, relay_parent).await?; @@ -274,10 +283,11 @@ where } /// Get group rotation info based on the given relay_parent. -pub async fn get_group_rotation_info(ctx: &mut Context, relay_parent: Hash) +pub async fn get_group_rotation_info(ctx: &mut Context, relay_parent: Hash) -> Result - where - Context: SubsystemContext +where + M: From, + Context: SubsystemContext, { // We drop `groups` here as we don't need them, because of `RuntimeInfo`. Ideally we would not // fetch them in the first place. diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs new file mode 100644 index 000000000000..18425295d950 --- /dev/null +++ b/node/subsystem-util/src/tests.rs @@ -0,0 +1,246 @@ + + +use super::*; +use executor::block_on; +use thiserror::Error; +use polkadot_node_jaeger as jaeger; +use polkadot_node_subsystem::{ + messages::CandidateSelectionMessage, ActiveLeavesUpdate, + ActivatedLeaf, +}; +use assert_matches::assert_matches; +use futures::{channel::mpsc, executor, StreamExt, future, Future, FutureExt, SinkExt}; +use polkadot_primitives::v1::Hash; +use polkadot_node_subsystem_test_helpers::{self as test_helpers, make_subsystem_context}; +use std::{pin::Pin, sync::{Arc, atomic::{AtomicUsize, Ordering}}, time::Duration}; + +// basic usage: in a nutshell, when you want to define a subsystem, just focus on what its jobs do; +// you can leave the subsystem itself to the job manager. + +// for purposes of demonstration, we're going to whip up a fake subsystem. +// this will 'select' candidates which are pre-loaded in the job + +// job structs are constructed within JobTrait::run +// most will want to retain the sender and receiver, as well as whatever other data they like +struct FakeCandidateSelectionJob { + receiver: mpsc::Receiver, +} + +// Error will mostly be a wrapper to make the try operator more convenient; +// deriving From implementations for most variants is recommended. +// It must implement Debug for logging. +#[derive(Debug, Error)] +enum Error { + #[error(transparent)] + Sending(#[from]mpsc::SendError), +} + +impl JobTrait for FakeCandidateSelectionJob { + type ToJob = CandidateSelectionMessage; + type Error = Error; + type RunArgs = bool; + type Metrics = (); + + const NAME: &'static str = "FakeCandidateSelectionJob"; + + /// Run a job for the parent block indicated + // + // this function is in charge of creating and executing the job's main loop + fn run( + _: Hash, + _: Arc, + run_args: Self::RunArgs, + _metrics: Self::Metrics, + receiver: mpsc::Receiver, + mut sender: JobSender, + ) -> Pin> + Send>> + where + S: SubsystemSender, + { + async move { + let job = FakeCandidateSelectionJob { receiver }; + + if run_args { + sender.send_message(CandidateSelectionMessage::Invalid( + Default::default(), + Default::default(), + ).into()).await; + } + + // it isn't necessary to break run_loop into its own function, + // but it's convenient to separate the concerns in this way + job.run_loop().await + } + .boxed() + } +} + +impl FakeCandidateSelectionJob { + async fn run_loop(mut self) -> Result<(), Error> { + loop { + match self.receiver.next().await { + Some(_csm) => { + unimplemented!("we'd report the collator to the peer set manager here, but that's not implemented yet"); + } + None => break, + } + } + + Ok(()) + } +} + +// with the job defined, it's straightforward to get a subsystem implementation. +type FakeCandidateSelectionSubsystem = + JobSubsystem; + +// this type lets us pretend to be the overseer +type OverseerHandle = test_helpers::TestSubsystemContextHandle; + +fn test_harness>( + run_args: bool, + test: impl FnOnce(OverseerHandle) -> T, +) { + let _ = env_logger::builder() + .is_test(true) + .filter( + None, + log::LevelFilter::Trace, + ) + .try_init(); + + let pool = sp_core::testing::TaskExecutor::new(); + let (context, overseer_handle) = make_subsystem_context(pool.clone()); + + let subsystem = FakeCandidateSelectionSubsystem::new( + pool, + run_args, + (), + ).run(context); + let test_future = test(overseer_handle); + + futures::pin_mut!(subsystem, test_future); + + executor::block_on(async move { + future::join(subsystem, test_future) + .timeout(Duration::from_secs(2)) + .await + .expect("test timed out instead of completing") + }); +} + +#[test] +fn starting_and_stopping_job_works() { + let relay_parent: Hash = [0; 32].into(); + + test_harness(true, |mut overseer_handle| async move { + overseer_handle + .send(FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: relay_parent, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + }), + ))) + .await; + assert_matches!( + overseer_handle.recv().await, + M::CandidateSelection(_) + ); + overseer_handle + .send(FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::stop_work(relay_parent), + ))) + .await; + + overseer_handle + .send(FromOverseer::Signal(OverseerSignal::Conclude)) + .await; + }); +} + +#[test] +fn sending_to_a_non_running_job_do_not_stop_the_subsystem() { + let relay_parent = Hash::repeat_byte(0x01); + + test_harness(true, |mut overseer_handle| async move { + overseer_handle + .send(FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: relay_parent, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + }), + ))) + .await; + + // send to a non running job + overseer_handle + .send(FromOverseer::Communication { + msg: Default::default(), + }) + .await; + + // the subsystem is still alive + assert_matches!( + overseer_handle.recv().await, + M::CandidateSelection(_) + ); + + overseer_handle + .send(FromOverseer::Signal(OverseerSignal::Conclude)) + .await; + }); +} + +#[test] +fn test_subsystem_impl_and_name_derivation() { + let pool = sp_core::testing::TaskExecutor::new(); + let (context, _) = make_subsystem_context::(pool.clone()); + + let SpawnedSubsystem { name, .. } = + FakeCandidateSelectionSubsystem::new(pool, false, ()).start(context); + assert_eq!(name, "FakeCandidateSelection"); +} + + +#[test] +fn tick_tack_metronome() { + let n = Arc::new(AtomicUsize::default()); + + let (tick, mut block) = mpsc::unbounded(); + + let metronome = { + let n = n.clone(); + let stream = Metronome::new(Duration::from_millis(137_u64)); + stream.for_each(move |_res| { + let _ = n.fetch_add(1, Ordering::Relaxed); + let mut tick = tick.clone(); + async move { + tick.send(()).await.expect("Test helper channel works. qed"); + } + }).fuse() + }; + + let f2 = async move { + block.next().await; + assert_eq!(n.load(Ordering::Relaxed), 1_usize); + block.next().await; + assert_eq!(n.load(Ordering::Relaxed), 2_usize); + block.next().await; + assert_eq!(n.load(Ordering::Relaxed), 3_usize); + block.next().await; + assert_eq!(n.load(Ordering::Relaxed), 4_usize); + }.fuse(); + + futures::pin_mut!(f2); + futures::pin_mut!(metronome); + + block_on(async move { + // futures::join!(metronome, f2) + futures::select!( + _ = metronome => unreachable!("Metronome never stops. qed"), + _ = f2 => (), + ) + }); +} From fdf7057b179ea5e452ccade30a2ef053b45349bd Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 27 May 2021 19:02:07 +0200 Subject: [PATCH 051/161] more fallout fixins --- .../overseer-gen/proc-macro/src/impl_misc.rs | 24 +-- .../proc-macro/src/impl_overseer.rs | 6 +- node/overseer/overseer-gen/src/lib.rs | 2 +- node/overseer/src/handler.rs | 53 ------ node/overseer/src/lib.rs | 177 +++++++----------- node/overseer/src/subsystems.rs | 4 +- node/overseer/src/tests.rs | 2 +- 7 files changed, 91 insertions(+), 177 deletions(-) delete mode 100644 node/overseer/src/handler.rs diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index d88d7cf30957..fc14e7a42003 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -7,21 +7,21 @@ use super::*; /// which acts as the gateway to constructing the overseer. pub(crate) fn impl_misc(info: &OverseerInfo) -> Result { let overseer_name = info.overseer_name.clone(); - let subsyste_sender_name = Ident::new(&(overseer_name.to_string() + "SubsystemSender"), overseer_name.span()); - let subsyste_ctx_name = Ident::new(&(overseer_name.to_string() + "SubsystemContext"), overseer_name.span()); + let subsystem_sender_name = Ident::new(&(overseer_name.to_string() + "SubsystemSender"), overseer_name.span()); + let subsystem_ctx_name = Ident::new(&(overseer_name.to_string() + "SubsystemContext"), overseer_name.span()); let consumes = &info.consumes(); let signal = &info.extern_signal_ty; let ts = quote! { #[derive(Debug, Clone)] - pub struct #subsyste_sender_name { + pub struct #subsystem_sender_name { channels: ChannelsOut, signals_received: SignalsReceived, } #( #[::polkadot_overseer_gen::async_trait] - impl SubsystemSender< #consumes > for #subsyste_sender_name { + impl SubsystemSender< #consumes > for #subsystem_sender_name { async fn send_message(&mut self, msg: #consumes) { self.channels.send_and_log_error(self.signals_received.load(), msg.into()).await; } @@ -49,10 +49,10 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result /// [`Subsystem`]: trait.Subsystem.html /// [`SubsystemJob`]: trait.SubsystemJob.html #[derive(Debug)] - pub struct #subsyste_ctx_name{ + pub struct #subsystem_ctx_name{ signals: ::polkadot_overseer_gen::metered::MeteredReceiver< #signal >, messages: SubsystemIncomingMessages, - to_subsystems: #subsyste_sender_name, + to_subsystems: #subsystem_sender_name, to_overseer: ::polkadot_overseer_gen::metered::UnboundedMeteredSender< ::polkadot_overseer_gen::ToOverseer >, @@ -60,7 +60,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result pending_incoming: Option<(usize, M)>, } - impl #subsyste_ctx_name { + impl #subsystem_ctx_name { /// Create a new context. fn new( signals: ::polkadot_overseer_gen::metered::MeteredReceiver< #signal >, @@ -69,10 +69,10 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result to_overseer: ::polkadot_overseer_gen::metered::UnboundedMeteredSender, ) -> Self { let signals_received = SignalsReceived::default(); - #subsyste_ctx_name { + #subsystem_ctx_name { signals, messages, - to_subsystems: #subsyste_sender_name { + to_subsystems: #subsystem_sender_name { channels: to_subsystems, signals_received: signals_received.clone(), }, @@ -84,13 +84,13 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result } #[::polkadot_overseer_gen::async_trait] - impl SubsystemContext for #subsyste_ctx_name + impl SubsystemContext for #subsystem_ctx_name where - #subsyste_sender_name: polkadot_overseer_gen::SubsystemSender, + #subsystem_sender_name: polkadot_overseer_gen::SubsystemSender, { type Message = M; type Signal = #signal; - type Sender = #subsyste_sender_name; + type Sender = #subsystem_sender_name; async fn try_recv(&mut self) -> Result>, ()> { match ::polkadot_overseer_gen::poll!(self.recv()) { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 8012cb1678df..af98cfdf0ca2 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -64,12 +64,12 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result> + BoxFuture<'static, ::polkadot_overseer_gen::SubsystemResult<()>> >, /// Gather running subsystems' outbound streams into one. to_overseer_rx: ::polkadot_overseer_gen::stream::Fuse< - metered::UnboundedMeteredReceiver< ToOverseer > + ::polkadot_overseer_gen::metered::UnboundedMeteredReceiver< ToOverseer > >, /// Events that are sent to the overseer from the outside world. @@ -144,7 +144,7 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { pub instance: std::option::Option< - SubsystemInstance + ::polkadot_overseer_gen::SubsystemInstance >, } diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index c5dd160edf82..dd02e01a110e 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -366,7 +366,7 @@ pub trait SubsystemContext: Send + 'static { /// /// This has to be used with caution, if you loop over this without /// using `pending!()` macro you will end up with a busy loop! - async fn try_recv(&mut self) -> Result>, ()>; + async fn try_recv(&mut self) -> SubsystemResult>>; /// Receive a message. async fn recv(&mut self) -> SubsystemResult>; diff --git a/node/overseer/src/handler.rs b/node/overseer/src/handler.rs deleted file mode 100644 index 1b56eca6be25..000000000000 --- a/node/overseer/src/handler.rs +++ /dev/null @@ -1,53 +0,0 @@ -/// A handler used to communicate with the [`Overseer`]. -/// -/// [`Overseer`]: struct.Overseer.html -#[derive(Clone)] -pub struct OverseerHandler { - events_tx: metered::MeteredSender, -} - -impl OverseerHandler { - /// Inform the `Overseer` that that some block was imported. - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - pub async fn block_imported(&mut self, block: BlockInfo) { - self.send_and_log_error(Event::BlockImported(block)).await - } - - /// Send some message to one of the `Subsystem`s. - #[tracing::instrument(level = "trace", skip(self, msg), fields(subsystem = LOG_TARGET))] - pub async fn send_msg(&mut self, msg: impl Into) { - self.send_and_log_error(Event::MsgToSubsystem(msg.into())).await - } - - /// Inform the `Overseer` that some block was finalized. - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - pub async fn block_finalized(&mut self, block: BlockInfo) { - self.send_and_log_error(Event::BlockFinalized(block)).await - } - - /// Wait for a block with the given hash to be in the active-leaves set. - /// - /// The response channel responds if the hash was activated and is closed if the hash was deactivated. - /// Note that due the fact the overseer doesn't store the whole active-leaves set, only deltas, - /// the response channel may never return if the hash was deactivated before this call. - /// In this case, it's the caller's responsibility to ensure a timeout is set. - #[tracing::instrument(level = "trace", skip(self, response_channel), fields(subsystem = LOG_TARGET))] - pub async fn wait_for_activation(&mut self, hash: Hash, response_channel: oneshot::Sender>) { - self.send_and_log_error(Event::ExternalRequest(ExternalRequest::WaitForActivation { - hash, - response_channel - })).await - } - - /// Tell `Overseer` to shutdown. - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - pub async fn stop(&mut self) { - self.send_and_log_error(Event::Stop).await - } - - async fn send_and_log_error(&mut self, event: Event) { - if self.events_tx.send(event).await.is_err() { - tracing::info!(target: LOG_TARGET, "Failed to send an event to Overseer"); - } - } -} diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 9c0f1f236660..3ca0532f9635 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -101,21 +101,25 @@ use self::subsystems::{AllSubsystems, AllSubsystemsSame}; mod metrics; use self::metrics::Metrics; -use polkadot_overseer_gen::*; use polkadot_node_subsystem_util::{metrics::{prometheus, Metrics as MetricsTrait}, metered, Metronome}; use polkadot_overseer_gen::{ TimeoutExt, SpawnNamed, SpawnedSubsystem, + Subsystem, SubsystemError, SubsystemMeterReadouts, SubsystemMeters, SubsystemIncomingMessages, + SubsystemInstance, + SubsystemSender, + SubsystemContext, overlord, MessagePacket, make_packet, SignalsReceived, FromOverseer, + ToOverseer, }; // Target for logs. @@ -150,84 +154,58 @@ impl HeadSupportsParachains for Arc where self.runtime_api().has_api::>(&id).unwrap_or(false) } } + + /// A handler used to communicate with the [`Overseer`]. /// /// [`Overseer`]: struct.Overseer.html #[derive(Clone)] -pub struct OverseerHandler2 { - events_tx: OverseerHandler, -} +pub struct Handler(pub OverseerHandler); -impl OverseerHandler2 { - /// Inform the `Overseer` that that some block was imported. - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - pub async fn block_imported(&mut self, block: BlockInfo) { - self.send_and_log_error(Event::BlockImported(block)).await - } - - /// Send some message to one of the `Subsystem`s. - #[tracing::instrument(level = "trace", skip(self, msg), fields(subsystem = LOG_TARGET))] - pub async fn send_msg(&mut self, msg: impl Into) { - self.send_and_log_error(Event::MsgToSubsystem(msg.into())).await - } - - /// Inform the `Overseer` that some block was finalized. - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - pub async fn block_finalized(&mut self, block: BlockInfo) { - self.send_and_log_error(Event::BlockFinalized(block)).await - } - - /// Wait for a block with the given hash to be in the active-leaves set. - /// - /// The response channel responds if the hash was activated and is closed if the hash was deactivated. - /// Note that due the fact the overseer doesn't store the whole active-leaves set, only deltas, - /// the response channel may never return if the hash was deactivated before this call. - /// In this case, it's the caller's responsibility to ensure a timeout is set. - #[tracing::instrument(level = "trace", skip(self, response_channel), fields(subsystem = LOG_TARGET))] - pub async fn wait_for_activation(&mut self, hash: Hash, response_channel: oneshot::Sender>) { - self.send_and_log_error(Event::ExternalRequest(ExternalRequest::WaitForActivation { - hash, - response_channel - })).await - } - - /// Tell `Overseer` to shutdown. - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] - pub async fn stop(&mut self) { - self.send_and_log_error(Event::Stop).await - } - - async fn send_and_log_error(&mut self, event: Event) { - if self.events_tx.send(event).await.is_err() { - tracing::info!(target: LOG_TARGET, "Failed to send an event to Overseer"); - } - } -} +impl Handler { + /// Inform the `Overseer` that that some block was imported. + #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] + pub async fn block_imported(&mut self, block: BlockInfo) { + self.send_and_log_error(Event::BlockImported(block)).await + } + /// Send some message to one of the `Subsystem`s. + #[tracing::instrument(level = "trace", skip(self, msg), fields(subsystem = LOG_TARGET))] + pub async fn send_msg(&mut self, msg: impl Into) { + self.send_and_log_error(Event::MsgToSubsystem(msg.into())).await + } -/// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. -/// -/// It wraps a system-wide [`AllMessages`] type that represents all possible -/// messages in the system. -/// -/// [`AllMessages`]: enum.AllMessages.html -/// [`Subsystem`]: trait.Subsystem.html -/// [`Overseer`]: struct.Overseer.html -enum ToOverseer { - /// A message that wraps something the `Subsystem` is desiring to - /// spawn on the overseer and a `oneshot::Sender` to signal the result - /// of the spawn. - SpawnJob { - name: &'static str, - s: BoxFuture<'static, ()>, - }, + /// Inform the `Overseer` that some block was finalized. + #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] + pub async fn block_finalized(&mut self, block: BlockInfo) { + self.send_and_log_error(Event::BlockFinalized(block)).await + } - /// Same as `SpawnJob` but for blocking tasks to be executed on a - /// dedicated thread pool. - SpawnBlockingJob { - name: &'static str, - s: BoxFuture<'static, ()>, - }, + /// Wait for a block with the given hash to be in the active-leaves set. + /// + /// The response channel responds if the hash was activated and is closed if the hash was deactivated. + /// Note that due the fact the overseer doesn't store the whole active-leaves set, only deltas, + /// the response channel may never return if the hash was deactivated before this call. + /// In this case, it's the caller's responsibility to ensure a timeout is set. + #[tracing::instrument(level = "trace", skip(self, response_channel), fields(subsystem = LOG_TARGET))] + pub async fn wait_for_activation(&mut self, hash: Hash, response_channel: oneshot::Sender>) { + self.send_and_log_error(Event::ExternalRequest(ExternalRequest::WaitForActivation { + hash, + response_channel + })).await + } + + /// Tell `Overseer` to shutdown. + #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] + pub async fn stop(&mut self) { + self.send_and_log_error(Event::Stop).await; + } + + async fn send_and_log_error(&mut self, event: Event) { + if self.0.send(event).await.is_err() { + tracing::info!(target: LOG_TARGET, "Failed to send an event to Overseer"); + } + } } /// An event telling the `Overseer` on the particular block @@ -285,12 +263,9 @@ enum ExternalRequest { /// Glues together the [`Overseer`] and `BlockchainEvents` by forwarding /// import and finality notifications into the [`OverseerHandler`]. -/// -/// [`Overseer`]: struct.Overseer.html -/// [`OverseerHandler`]: struct.OverseerHandler.html pub async fn forward_events>( client: Arc

, - mut handler: OverseerHandler2, + mut handler: Handler, ) { let mut finality = client.finality_notification_stream(); let mut imports = client.import_notification_stream(); @@ -318,15 +293,6 @@ pub async fn forward_events>( } } -impl Debug for ToOverseer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ToOverseer::SpawnJob { .. } => write!(f, "OverseerMessage::Spawn(..)"), - ToOverseer::SpawnBlockingJob { .. } => write!(f, "OverseerMessage::SpawnBlocking(..)") - } - } -} - // async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) // -> SubsystemResult<()> @@ -350,59 +316,60 @@ impl Debug for ToOverseer { /// The `Overseer` itself. #[overlord(gen=AllMessages, event=Event, signal=OverseerSignal, error=SubsystemError)] pub struct Overseer { + #[subsystem(no_dispatch, CandidateValidationMessage)] - candidate_validation_message: CandidateValidation, + candidate_validation: CandidateValidation, #[subsystem(no_dispatch, CandidateBackingMessage)] - candidate_backing_message: CandidateBacking, + candidate_backing: CandidateBacking, #[subsystem(no_dispatch, CandidateSelectionMessage)] candidate_selection_message: CandidateSelection, #[subsystem(StatementDistributionMessage)] - statement_distribution_message: StatementDistribution, + statement_distribution: StatementDistribution, #[subsystem(no_dispatch, AvailabilityDistributionMessage)] - availability_distribution_message: AvailabilityDistribution, + availability_distribution: AvailabilityDistribution, #[subsystem(no_dispatch, AvailabilityRecoveryMessage)] - availability_recovery_message: AvailabilityRecovery, + availability_recovery: AvailabilityRecovery, #[subsystem(blocking, BitfieldSigningMessage)] - bitfield_signing_message: BitfieldSigning, + bitfield_signing: BitfieldSigning, #[subsystem(no_dispatch, BitfieldDistributionMessage)] - bitfield_distribution_message: BitfieldDistribution, + bitfield_distribution: BitfieldDistribution, #[subsystem(no_dispatch, ProvisionerMessage)] - provisioner_message: Provisioner, + provisioner: Provisioner, #[subsystem(no_dispatch, blocking, RuntimeApiMessage)] - runtime_api_message: RuntimeApi, + runtime_api: RuntimeApi, #[subsystem(no_dispatch, blocking, AvailabilityStoreMessage)] - availability_store_message: AvailabilityStore, + availability_store: AvailabilityStore, #[subsystem(no_dispatch, NetworkBridgeMessage)] - network_bridge_message: NetworkBridge, + network_bridge: NetworkBridge, #[subsystem(no_dispatch, blocking, ChainApiMessage)] - chain_api_message: ChainApi, + chain_api: ChainApi, #[subsystem(no_dispatch, CollationGenerationMessage)] - collation_generation_message: CollationGeneration, + collation_generation: CollationGeneration, #[subsystem(no_dispatch, CollatorProtocolMessage)] - collator_protocol_message: CollatorProtocol, + collator_protocol: CollatorProtocol, #[subsystem(ApprovalDistributionMessage)] - approval_distribution_message: ApprovalDistribution, + approval_distribution: ApprovalDistribution, #[subsystem(no_dispatch, ApprovalVotingMessage)] - approval_voting_message: ApprovalVoting, + approval_voting: ApprovalVoting, #[subsystem(no_dispatch, GossipSupportMessage)] - gossip_support_message: GossipSupport, + gossip_support: GossipSupport, /// External listeners waiting for a hash to be in the active-leave set. activation_external_listeners: HashMap>>>, @@ -526,7 +493,7 @@ where prometheus_registry: Option<&prometheus::Registry>, supports_parachains: SupportsParachains, mut s: S, - ) -> ::polkadot_overseer_gen::SubsystemResult<(Self, OverseerHandler)> + ) -> ::polkadot_overseer_gen::SubsystemResult<(Self, Handler)> where CV: Subsystem, E> + Send, CB: Subsystem, E> + Send, @@ -615,7 +582,7 @@ where .spawner(s) .build(); - Ok((overseer, OverseerHandler { handler })) + Ok((overseer, Handler(handler))) } // Stop the overseer. @@ -828,11 +795,11 @@ where } fn spawn_job(&mut self, name: &'static str, j: BoxFuture<'static, ()>) { - self.s.spawn(name, j); + self.spawner.spawn(name, j); } fn spawn_blocking_job(&mut self, name: &'static str, j: BoxFuture<'static, ()>) { - self.s.spawn_blocking(name, j); + self.spawner.spawn_blocking(name, j); } } diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index d6b910078ccd..64f0c2c3eaf5 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -2,6 +2,7 @@ use ::polkadot_overseer_gen::{ MapSubsystem, SubsystemContext, Subsystem, + SubsystemError, SpawnedSubsystem, FromOverseer, }; @@ -13,11 +14,10 @@ use crate::OverseerSignal; #[derive(Clone, Copy, Debug)] pub struct DummySubsystem; -impl Subsystem for DummySubsystem +impl Subsystem for DummySubsystem where Context: SubsystemContext, ::Message: std::fmt::Debug + Send + 'static, - E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::SubsystemError>, { fn start(self, mut ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { diff --git a/node/overseer/src/tests.rs b/node/overseer/src/tests.rs index 66d27d189574..0fb65958c078 100644 --- a/node/overseer/src/tests.rs +++ b/node/overseer/src/tests.rs @@ -46,7 +46,7 @@ struct TestSubsystem2(metered::MeteredSender); impl Subsystem for TestSubsystem2 where C: SubsystemContext { - fn start(self, mut ctx: C) -> SpawnedSubsystem<::::polkadot_overseer_gen::SubsystemError> { + fn start(self, mut ctx: C) -> SpawnedSubsystem { let sender = self.0.clone(); SpawnedSubsystem { name: "test-subsystem-2", From 06c969fca0985bf26ba5da480edd2fac6fcf51e2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 27 May 2021 19:16:11 +0200 Subject: [PATCH 052/161] foo --- node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index af98cfdf0ca2..e8e850c59f06 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -174,7 +174,7 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result SubsystemResult<()> { + pub async fn send_signal(&mut self, signal: #signal) -> ::polkadot_overseer_gen::SubsystemResult<()> { const SIGNAL_TIMEOUT: ::std::time::Duration = ::std::time::Duration::from_secs(10); if let Some(ref mut instance) = self.instance { From eddda167cb60fbf6eea37c6c7fb4c3a33dd2eeff Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 1 Jun 2021 16:04:02 +0200 Subject: [PATCH 053/161] fixins --- node/overseer/Cargo.toml | 3 +- node/overseer/overseer-gen/examples/dummy.rs | 1 - .../proc-macro/src/impl_builder.rs | 14 +++- .../proc-macro/src/impl_dispatch.rs | 4 +- .../overseer-gen/proc-macro/src/impl_misc.rs | 2 +- .../proc-macro/src/impl_overseer.rs | 19 ++++- node/overseer/overseer-gen/src/lib.rs | 12 +-- node/overseer/src/lib.rs | 78 +++++++++++-------- node/subsystem-util/src/lib.rs | 17 +++- node/subsystem/src/errors.rs | 7 ++ node/subsystem/src/lib.rs | 56 ------------- 11 files changed, 107 insertions(+), 106 deletions(-) diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml index d714224f013a..78fa1bc9124d 100644 --- a/node/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -10,11 +10,12 @@ client = { package = "sc-client-api", git = "https://github.com/paritytech/subst sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" } futures = "0.3.12" futures-timer = "3.0.2" +polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-node-primitives = { package = "polkadot-node-primitives", path = "../primitives" } polkadot-node-subsystem-util = { path = "../subsystem-util" } polkadot-primitives = { path = "../../primitives" } polkadot-overseer-gen = { package = "polkadot-overseer-gen", path = "./overseer-gen" } -polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../subsystem" } +polkadot-node-subsystem = { package = "polkadot-node-subsystem", path = "../subsystem" } tracing = "0.1.25" [dev-dependencies] diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index 5f9bcccc91ee..f52a4750dca1 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -65,7 +65,6 @@ pub struct MsgStrukt(u8); #[derive(Debug, Clone, Copy)] pub struct Plinko; - impl From for MsgStrukt { fn from(_event: EvX) -> Self { MsgStrukt(1u8) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index afeb99a88dd3..ad466c3cde7d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -64,13 +64,14 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result, #error_ty>, )* + #( + #builder_generic_ty : Subsystem<#subsyste_ctx_name< #consumes >, #error_ty>, + )* }; let event = &info.extern_event_ty; let mut ts = quote! { - impl #generics #overseer_name #generics #where_clause { pub fn builder #builder_additional_generics () -> #builder #builder_generics #builder_where_clause @@ -94,6 +95,13 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Self { + // explicitly assure the required traits are implemented + fn trait_from_must_be_implemented() + where E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::SubsystemError> + {} + + trait_from_must_be_implemented::< #error_ty >(); + Self { #( #subsystem_name: None, @@ -271,7 +279,7 @@ pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result SubsystemResult> where S: ::polkadot_overseer_gen::SpawnNamed, - M: Send + 'static, + M: std::fmt::Debug + Send + 'static, TK: TaskKind, Ctx: ::polkadot_overseer_gen::SubsystemContext, E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::SubsystemError>, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index be8ff74cfa53..1758ecac81cf 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -49,7 +49,9 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { let mut iter = iter.chain( ::std::iter::once( - #dispatchable :: try_from( event ).ok().map(|event| { + // alt: + // #dispatchable :: try_from( event ) + event.focus().ok().map(|event| { #message_wrapper :: #dispatchable ( event ) }) ) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index fc14e7a42003..3502ffe67a1f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -84,7 +84,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result } #[::polkadot_overseer_gen::async_trait] - impl SubsystemContext for #subsystem_ctx_name + impl SubsystemContext for #subsystem_ctx_name where #subsystem_sender_name: polkadot_overseer_gen::SubsystemSender, { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index e8e850c59f06..a04b876175bb 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -27,6 +27,8 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result Result> + BoxFuture<'static, ::std::result::Result<(), #error_ty>> >, /// Gather running subsystems' outbound streams into one. @@ -124,6 +126,21 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result(&self, mapper: Mapper) + -> Vec + where + #( + Mapper: for<'a> MapSubsystem<&'a OverseenSubsystem< #consumes >, Output=Output>, + )* + { + vec![ + #( + mapper.map_subsystem( & self. #subsystem_name ), + )* + ] + } } }; diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index dd02e01a110e..62ff798fb2c2 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -220,13 +220,13 @@ pub trait AnnotateErrorOrigin: 'static + Send + Sync + std::error::Error { /// An asynchronous subsystem task.. /// /// In essence it's just a newtype wrapping a `BoxFuture`. -pub struct SpawnedSubsystem +pub struct SpawnedSubsystem where E: std::error::Error + Send + Sync + 'static - + From + + From, { /// Name of the subsystem being spawned. pub name: &'static str, @@ -274,7 +274,7 @@ pub enum SubsystemError { } /// Alias for a result with error type `SubsystemError`. -pub type SubsystemResult = std::result::Result; +pub type SubsystemResult = std::result::Result; /// Collection of meters related to a subsystem. #[derive(Clone)] @@ -360,13 +360,13 @@ pub trait SubsystemContext: Send + 'static { /// And the same for signals. type Signal: std::fmt::Debug + Send + 'static; /// The sender type as provided by `sender()` and underlying. - type Sender: SubsystemSender + std::fmt::Debug + Clone + Send; + type Sender: SubsystemSender + std::fmt::Debug + Send; /// Try to asynchronously receive a message. /// /// This has to be used with caution, if you loop over this without /// using `pending!()` macro you will end up with a busy loop! - async fn try_recv(&mut self) -> SubsystemResult>>; + async fn try_recv(&mut self) -> Result>, ()>; /// Receive a message. async fn recv(&mut self) -> SubsystemResult>; @@ -408,7 +408,7 @@ pub trait SubsystemContext: Send + 'static { pub trait Subsystem where Ctx: SubsystemContext, - E: std::error::Error + Send + Sync + 'static + From, + E: std::error::Error + Send + Sync + 'static + From, { /// Start this `Subsystem` and return `SpawnedSubsystem`. fn start(self, ctx: Ctx) -> SpawnedSubsystem < E >; diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 3ca0532f9635..3583014f035c 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -80,7 +80,7 @@ use polkadot_primitives::v1::{Block, BlockId,BlockNumber, Hash, ParachainHost}; use client::{BlockImportNotification, BlockchainEvents, FinalityNotification}; use sp_api::{ApiExt, ProvideRuntimeApi}; -use polkadot_subsystem::messages::{ +use polkadot_node_subsystem::messages::{ CandidateValidationMessage, CandidateBackingMessage, CandidateSelectionMessage, ChainApiMessage, StatementDistributionMessage, AvailabilityDistributionMessage, BitfieldSigningMessage, BitfieldDistributionMessage, @@ -88,10 +88,15 @@ use polkadot_subsystem::messages::{ AvailabilityStoreMessage, NetworkBridgeMessage, CollationGenerationMessage, CollatorProtocolMessage, AvailabilityRecoveryMessage, ApprovalDistributionMessage, ApprovalVotingMessage, GossipSupportMessage, + NetworkBridgeEvent, }; -pub use polkadot_subsystem::{ +pub use polkadot_node_subsystem::{ OverseerSignal, - SubsystemResult, ActiveLeavesUpdate, ActivatedLeaf, jaeger, + errors::{SubsystemResult, SubsystemError,}, + ActiveLeavesUpdate, ActivatedLeaf, jaeger, +}; +use polkadot_node_network_protocol::{ + PeerId, View, v1 as protocol_v1, UnifiedReputationChange as Rep, }; /// TODO legacy, to be deleted, left for easier integration @@ -107,7 +112,6 @@ use polkadot_overseer_gen::{ SpawnNamed, SpawnedSubsystem, Subsystem, - SubsystemError, SubsystemMeterReadouts, SubsystemMeters, SubsystemIncomingMessages, @@ -192,7 +196,7 @@ impl Handler { self.send_and_log_error(Event::ExternalRequest(ExternalRequest::WaitForActivation { hash, response_channel - })).await + })).await; } /// Tell `Overseer` to shutdown. @@ -312,9 +316,15 @@ pub async fn forward_events>( // }).await.map_err(Into::into) // } +// NetworkBridgeEvent, /// The `Overseer` itself. -#[overlord(gen=AllMessages, event=Event, signal=OverseerSignal, error=SubsystemError)] +#[overlord( + gen=AllMessages, + event=Event, + signal=OverseerSignal, + error=SubsystemError +)] pub struct Overseer { #[subsystem(no_dispatch, CandidateValidationMessage)] @@ -433,7 +443,7 @@ where /// # use futures_timer::Delay; /// # use polkadot_overseer::{Overseer, HeadSupportsParachains, AllSubsystems}; /// # use polkadot_primitives::v1::Hash; - /// # use polkadot_subsystem::{ + /// # use polkadot_node_subsystem::{ /// # Subsystem, DummySubsystem, SpawnedSubsystem, SubsystemContext, /// # messages::CandidateValidationMessage, /// # }; @@ -487,33 +497,32 @@ where /// # /// # }); } /// ``` - pub fn new( + pub fn new( leaves: impl IntoIterator, all_subsystems: AllSubsystems, prometheus_registry: Option<&prometheus::Registry>, supports_parachains: SupportsParachains, mut s: S, - ) -> ::polkadot_overseer_gen::SubsystemResult<(Self, Handler)> + ) -> SubsystemResult<(Self, Handler)> where - CV: Subsystem, E> + Send, - CB: Subsystem, E> + Send, - CS: Subsystem, E> + Send, - SD: Subsystem, E> + Send, - AD: Subsystem, E> + Send, - AR: Subsystem, E> + Send, - BS: Subsystem, E> + Send, - BD: Subsystem, E> + Send, - P: Subsystem, E> + Send, - RA: Subsystem, E> + Send, - AS: Subsystem, E> + Send, - NB: Subsystem, E> + Send, - CA: Subsystem, E> + Send, - CG: Subsystem, E> + Send, - CP: Subsystem, E> + Send, - ApD: Subsystem, E> + Send, - ApV: Subsystem, E> + Send, - GS: Subsystem, E> + Send, - E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::SubsystemError>, + CV: Subsystem, SubsystemError> + Send, + CB: Subsystem, SubsystemError> + Send, + CS: Subsystem, SubsystemError> + Send, + SD: Subsystem, SubsystemError> + Send, + AD: Subsystem, SubsystemError> + Send, + AR: Subsystem, SubsystemError> + Send, + BS: Subsystem, SubsystemError> + Send, + BD: Subsystem, SubsystemError> + Send, + P: Subsystem, SubsystemError> + Send, + RA: Subsystem, SubsystemError> + Send, + AS: Subsystem, SubsystemError> + Send, + NB: Subsystem, SubsystemError> + Send, + CA: Subsystem, SubsystemError> + Send, + CG: Subsystem, SubsystemError> + Send, + CP: Subsystem, SubsystemError> + Send, + ApD: Subsystem, SubsystemError> + Send, + ApV: Subsystem, SubsystemError> + Send, + GS: Subsystem, SubsystemError> + Send, S: SpawnNamed, { @@ -521,6 +530,7 @@ where // spawn the metrics metronome task { struct ExtractNameAndMeters; + impl<'a, T: 'a> MapSubsystem<&'a OverseenSubsystem> for ExtractNameAndMeters { type Output = (&'static str, SubsystemMeters); @@ -536,7 +546,10 @@ where } } - let subsystem_meters = all_subsystems.as_ref().map_subsystems(ExtractNameAndMeters); + let subsystem_meters = all_subsystems + .as_ref() + .map_subsystems(ExtractNameAndMeters); + let metronome_metrics = metrics.clone(); let metronome = Metronome::new(std::time::Duration::from_millis(950)) .for_each(move |_| { @@ -666,7 +679,7 @@ where tracing::error!(target: LOG_TARGET, subsystem = ?finished, "subsystem finished unexpectedly"); self.stop().await; - return finished; + return Ok(finished?); }, } } @@ -700,10 +713,9 @@ where self.clean_up_external_listeners(); if !update.is_empty() { - self.broadcast_signal(OverseerSignal::ActiveLeaves(update)).await - } else { - Ok(()) + self.broadcast_signal(OverseerSignal::ActiveLeaves(update)).await?; } + Ok(()) } #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 029335bfa7a0..25305d1e26e4 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -407,13 +407,24 @@ pub enum FromJobCommand { } /// A sender for messages from jobs, as well as commands to the overseer. -#[derive(Clone)] pub struct JobSender, M> { sender: S, from_job: mpsc::Sender, _phantom: std::marker::PhantomData, } +// A custom clone impl, since M does not need to impl `Clone` +// which `#[derive(Clone)]` requires. +impl, M> Clone for JobSender { + fn clone(&self) -> Self { + Self { + sender: self.sender.clone(), + from_job: self.from_job.clone(), + _phantom: Default::default(), + } + } +} + impl, M> JobSender { /// Get access to the underlying subsystem sender. pub fn subsystem_sender(&mut self) -> &mut S { @@ -451,8 +462,8 @@ impl, M> JobSender { #[async_trait::async_trait] impl SubsystemSender for JobSender where - M: Clone + Send + 'static, - S: SubsystemSender + M: Send + 'static, + S: SubsystemSender + Clone, { async fn send_message(&mut self, msg: M) { self.sender.send_message(msg).await diff --git a/node/subsystem/src/errors.rs b/node/subsystem/src/errors.rs index 764c366c87eb..353c0342ec63 100644 --- a/node/subsystem/src/errors.rs +++ b/node/subsystem/src/errors.rs @@ -118,6 +118,10 @@ pub enum SubsystemError { #[error("Subsystem stalled: {0}")] SubsystemStalled(&'static str), + /// Generated by the `#[overseer(..)]` proc-macro + #[error(transparent)] + Generated(#[from] polkadot_overseer_gen::SubsystemError), + /// Per origin (or subsystem) annotations to wrap an error. #[error("Error originated in {origin}")] FromOrigin { @@ -143,3 +147,6 @@ impl SubsystemError { Self::FromOrigin { origin, source: Box::new(err) } } } + +/// Ease the use of subsystem errors. +pub type SubsystemResult = Result; diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 7f8aae8f0aaf..da4f1cdf1003 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -123,59 +123,3 @@ pub enum OverseerSignal { /// Conclude the work of the `Overseer` and all `Subsystem`s. Conclude, } - -/// An error type that describes faults that may happen -/// -/// These are: -/// * Channels being closed -/// * Subsystems dying when they are not expected to -/// * Subsystems not dying when they are told to die -/// * etc. -#[derive(thiserror::Error, Debug)] -#[allow(missing_docs)] -pub enum SubsystemError { - #[error(transparent)] - NotifyCancellation(#[from] oneshot::Canceled), - - #[error(transparent)] - QueueError(#[from] mpsc::SendError), - - #[error(transparent)] - TaskSpawn(#[from] futures::task::SpawnError), - - #[error(transparent)] - Infallible(#[from] std::convert::Infallible), - - #[error(transparent)] - Prometheus(#[from] substrate_prometheus_endpoint::PrometheusError), - - #[error(transparent)] - Jaeger(#[from] jaeger::JaegerError), - - #[error("Failed to {0}")] - Context(String), - - #[error("Subsystem stalled: {0}")] - SubsystemStalled(&'static str), - - /// Per origin (or subsystem) annotations to wrap an error. - #[error("Error originated in {origin}")] - FromOrigin { - /// An additional annotation tag for the origin of `source`. - origin: &'static str, - /// The wrapped error. Marked as source for tracking the error chain. - #[source] source: Box - }, -} - -impl SubsystemError { - /// Adds a `str` as `origin` to the given error `err`. - pub fn with_origin(origin: &'static str, err: E) -> Self { - Self::FromOrigin { origin, source: Box::new(err) } - } -} - -/// A `Result` type that wraps [`SubsystemError`]. -/// -/// [`SubsystemError`]: struct.SubsystemError.html -pub type SubsystemResult = Result; From c7f346736f6538588dc5f7093c99b5250cfda24a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 1 Jun 2021 16:15:54 +0200 Subject: [PATCH 054/161] minor --- node/network/availability-recovery/src/lib.rs | 2 +- node/overseer/src/lib.rs | 8 +- node/overseer/src/subsystems.rs | 110 +++++++++--------- node/subsystem/src/errors.rs | 3 - 4 files changed, 61 insertions(+), 62 deletions(-) diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index 79c1238b7bff..213abc330eeb 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -666,7 +666,7 @@ async fn launch_interaction( #[tracing::instrument(level = "trace", skip(ctx, state), fields(subsystem = LOG_TARGET))] async fn handle_recover( state: &mut State, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, receipt: CandidateReceipt, session_index: SessionIndex, backing_group: Option, diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 3583014f035c..6c0d64f9c3f6 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -334,7 +334,7 @@ pub struct Overseer { candidate_backing: CandidateBacking, #[subsystem(no_dispatch, CandidateSelectionMessage)] - candidate_selection_message: CandidateSelection, + candidate_selection: CandidateSelection, #[subsystem(StatementDistributionMessage)] statement_distribution: StatementDistribution, @@ -587,13 +587,15 @@ where .approval_distribution(all_subsystems.approval_distribution) .approval_voting(all_subsystems.approval_voting) .gossip_support(all_subsystems.gossip_support) - .leaves(Vec::from_iter(leaves)) + .leaves(Vec::from_iter( + leaves.into_iter().map(|BlockInfo { hash, parent_hash: _, number }| (hash, number)) + )) .active_leaves(Default::default()) .span_per_active_leaf(Default::default()) .activation_external_listeners(Default::default()) .metrics(metrics) .spawner(s) - .build(); + .build()?; Ok((overseer, Handler(handler))) } diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index 64f0c2c3eaf5..bfb3e866257e 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -177,66 +177,66 @@ impl } } - pub fn map_subsystems(self, m: M) + pub fn map_subsystems(self, mapper: Mapper) -> AllSubsystems< - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, - >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, + >::Output, > where - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem

, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, - M: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem

, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, + Mapper: MapSubsystem, { AllSubsystems { - candidate_validation: m.map_subsystem(self.candidate_validation), - candidate_backing: m.map_subsystem(self.candidate_backing), - candidate_selection: m.map_subsystem(self.candidate_selection), - statement_distribution: m.map_subsystem(self.statement_distribution), - availability_distribution: m.map_subsystem(self.availability_distribution), - availability_recovery: m.map_subsystem(self.availability_recovery), - bitfield_signing: m.map_subsystem(self.bitfield_signing), - bitfield_distribution: m.map_subsystem(self.bitfield_distribution), - provisioner: m.map_subsystem(self.provisioner), - runtime_api: m.map_subsystem(self.runtime_api), - availability_store: m.map_subsystem(self.availability_store), - network_bridge: m.map_subsystem(self.network_bridge), - chain_api: m.map_subsystem(self.chain_api), - collation_generation: m.map_subsystem(self.collation_generation), - collator_protocol: m.map_subsystem(self.collator_protocol), - approval_distribution: m.map_subsystem(self.approval_distribution), - approval_voting: m.map_subsystem(self.approval_voting), - gossip_support: m.map_subsystem(self.gossip_support), + candidate_validation: mapper.map_subsystem(self.candidate_validation), + candidate_backing: mapper.map_subsystem(self.candidate_backing), + candidate_selection: mapper.map_subsystem(self.candidate_selection), + statement_distribution: mapper.map_subsystem(self.statement_distribution), + availability_distribution: mapper.map_subsystem(self.availability_distribution), + availability_recovery: mapper.map_subsystem(self.availability_recovery), + bitfield_signing: mapper.map_subsystem(self.bitfield_signing), + bitfield_distribution: mapper.map_subsystem(self.bitfield_distribution), + provisioner: mapper.map_subsystem(self.provisioner), + runtime_api: mapper.map_subsystem(self.runtime_api), + availability_store: mapper.map_subsystem(self.availability_store), + network_bridge: mapper.map_subsystem(self.network_bridge), + chain_api: mapper.map_subsystem(self.chain_api), + collation_generation: mapper.map_subsystem(self.collation_generation), + collator_protocol: mapper.map_subsystem(self.collator_protocol), + approval_distribution: mapper.map_subsystem(self.approval_distribution), + approval_voting: mapper.map_subsystem(self.approval_voting), + gossip_support: mapper.map_subsystem(self.gossip_support), } } } diff --git a/node/subsystem/src/errors.rs b/node/subsystem/src/errors.rs index 353c0342ec63..976a4e0ed60a 100644 --- a/node/subsystem/src/errors.rs +++ b/node/subsystem/src/errors.rs @@ -81,9 +81,6 @@ impl std::fmt::Display for RecoveryError { impl std::error::Error for RecoveryError {} - - - /// An error type that describes faults that may happen /// /// These are: From ca23d5b2958b3970ad858ce313a3405aeb05d413 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 2 Jun 2021 11:02:09 +0200 Subject: [PATCH 055/161] minor --- .../proc-macro/src/impl_builder.rs | 1 - .../proc-macro/src/impl_overseer.rs | 2 +- node/overseer/src/lib.rs | 19 +++---------------- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index ad466c3cde7d..27d27703b931 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -82,7 +82,6 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result; - #[derive(Debug, Clone)] pub struct #builder #builder_generics { #( #subsystem_name : ::std::option::Option< #builder_generic_ty >, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index a04b876175bb..f6221c50aef9 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -127,7 +127,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result(&self, mapper: Mapper) -> Vec where diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 6c0d64f9c3f6..a906603a15d4 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -124,25 +124,12 @@ use polkadot_overseer_gen::{ SignalsReceived, FromOverseer, ToOverseer, + MapSubsystem, }; // Target for logs. // const LOG_TARGET: &'static str = "parachain::overseer"; -trait MapSubsystem { - type Output; - - fn map_subsystem(&self, sub: T) -> Self::Output; -} - -impl MapSubsystem for F where F: Fn(T) -> U { - type Output = U; - - fn map_subsystem(&self, sub: T) -> U { - (self)(sub) - } -} - /// Whether a header supports parachain consensus or not. pub trait HeadSupportsParachains { /// Return true if the given header supports parachain consensus. Otherwise, false. @@ -548,7 +535,7 @@ where let subsystem_meters = all_subsystems .as_ref() - .map_subsystems(ExtractNameAndMeters); + .map_subsystems::(ExtractNameAndMeters); let metronome_metrics = metrics.clone(); let metronome = Metronome::new(std::time::Duration::from_millis(950)) @@ -558,7 +545,7 @@ where // We combine the amount of messages from subsystems to the overseer // as well as the amount of messages from external sources to the overseer - // into one to_overseer value. + // into one `to_overseer` value. metronome_metrics.channel_fill_level_snapshot(subsystem_meters); async move { From 489f72ff5cb8ba0840f5685b6f8dbe54c102bacd Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 2 Jun 2021 17:06:29 +0200 Subject: [PATCH 056/161] undo some damage --- node/core/candidate-validation/src/lib.rs | 10 +- node/core/chain-api/src/lib.rs | 2 +- node/core/runtime-api/src/lib.rs | 2 +- node/network/approval-distribution/src/lib.rs | 18 +- node/network/availability-recovery/src/lib.rs | 6 +- node/network/bitfield-distribution/src/lib.rs | 713 +-------- .../bitfield-distribution/src/tests.rs | 710 +++++++++ node/network/bridge/src/lib.rs | 1300 +---------------- node/network/bridge/src/tests.rs | 1292 ++++++++++++++++ .../collator-protocol/src/collator_side.rs | 10 +- .../collator-protocol/src/validator_side.rs | 2 +- .../network/statement-distribution/src/lib.rs | 4 +- 12 files changed, 2032 insertions(+), 2037 deletions(-) create mode 100644 node/network/bitfield-distribution/src/tests.rs create mode 100644 node/network/bridge/src/tests.rs diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index ec2ffc3dbc52..11537b2e3e96 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -96,7 +96,7 @@ impl Subsystem for CandidateValidationSubsystem where #[tracing::instrument(skip(ctx, metrics), fields(subsystem = LOG_TARGET))] async fn run( - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, metrics: Metrics, cache_path: PathBuf, program_path: PathBuf, @@ -172,7 +172,7 @@ async fn run( } async fn runtime_api_request( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, relay_parent: Hash, request: RuntimeApiRequest, receiver: oneshot::Receiver>, @@ -196,7 +196,7 @@ enum AssumptionCheckOutcome { #[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn check_assumption_validation_data( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, descriptor: &CandidateDescriptor, assumption: OccupiedCoreAssumption, ) -> SubsystemResult { @@ -247,7 +247,7 @@ async fn check_assumption_validation_data( #[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn find_assumed_validation_data( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, descriptor: &CandidateDescriptor, ) -> SubsystemResult { // The candidate descriptor has a `persisted_validation_data_hash` which corresponds to @@ -283,7 +283,7 @@ async fn find_assumed_validation_data( fields(subsystem = LOG_TARGET), )] async fn spawn_validate_from_chain_state( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, validation_host: &mut ValidationHost, descriptor: CandidateDescriptor, pov: Arc, diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index eb0aa9d19e24..782fcb19ed06 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -79,7 +79,7 @@ impl Subsystem for ChainApiSubsystem where #[tracing::instrument(skip(ctx, subsystem), fields(subsystem = LOG_TARGET))] async fn run( - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, subsystem: ChainApiSubsystem, ) -> SubsystemResult<()> where diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index 25f65bdec521..a2cd09a8f380 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -264,7 +264,7 @@ impl RuntimeApiSubsystem where #[tracing::instrument(skip(ctx, subsystem), fields(subsystem = LOG_TARGET))] async fn run( - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, mut subsystem: RuntimeApiSubsystem, ) -> SubsystemResult<()> where Client: ProvideRuntimeApi + Send + Sync + 'static, diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index caf9e365b56a..7c69ac136389 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -186,7 +186,7 @@ enum PendingMessage { impl State { async fn handle_network_msg( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, event: NetworkBridgeEvent, ) { @@ -247,7 +247,7 @@ impl State { async fn handle_new_blocks( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, metas: Vec, ) { @@ -349,7 +349,7 @@ impl State { async fn process_incoming_peer_message( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, peer_id: PeerId, msg: protocol_v1::ApprovalDistributionMessage, @@ -437,7 +437,7 @@ impl State { async fn handle_peer_view_change( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, peer_id: PeerId, view: View, @@ -506,7 +506,7 @@ impl State { async fn import_and_circulate_assignment( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, source: MessageSource, assignment: IndirectAssignmentCert, @@ -722,7 +722,7 @@ impl State { async fn import_and_circulate_approval( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, source: MessageSource, vote: IndirectSignedApprovalVote, @@ -960,7 +960,7 @@ impl State { } async fn unify_with_peer( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, entries: &mut HashMap, peer_id: PeerId, @@ -1010,7 +1010,7 @@ impl State { async fn send_gossip_messages_to_peer( entries: &HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, peer_id: PeerId, blocks: Vec<(BlockDepth, Hash)>, ) { @@ -1112,7 +1112,7 @@ impl State { /// Modify the reputation of a peer based on its behavior. #[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn modify_reputation( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, peer_id: PeerId, rep: Rep, ) { diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index 213abc330eeb..d6abdb4b6cf6 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -611,7 +611,7 @@ async fn handle_signal( #[tracing::instrument(level = "trace", skip(ctx, state), fields(subsystem = LOG_TARGET))] async fn launch_interaction( state: &mut State, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, session_index: SessionIndex, session_info: SessionInfo, receipt: CandidateReceipt, @@ -729,7 +729,7 @@ async fn handle_recover( /// Queries a chunk from av-store. #[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn query_full_data( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, candidate_hash: CandidateHash, ) -> error::Result> { let (tx, rx) = oneshot::channel(); @@ -753,7 +753,7 @@ impl AvailabilityRecoverySubsystem { async fn run( self, - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, ) -> SubsystemResult<()> { let mut state = State::default(); diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index 848eeb02bb71..c43ce75fe40a 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -822,715 +822,4 @@ impl metrics::Metrics for Metrics { #[cfg(test)] -mod test { - use super::*; - use bitvec::bitvec; - use futures::executor; - use maplit::hashmap; - use polkadot_primitives::v1::{Signed, AvailabilityBitfield, ValidatorIndex}; - use polkadot_node_subsystem_test_helpers::make_subsystem_context; - use polkadot_node_subsystem_util::TimeoutExt; - use sp_keystore::{SyncCryptoStorePtr, SyncCryptoStore}; - use sp_application_crypto::AppKey; - use sp_keystore::testing::KeyStore; - use std::sync::Arc; - use std::time::Duration; - use assert_matches::assert_matches; - use polkadot_node_network_protocol::{view, ObservedRole, our_view}; - use polkadot_subsystem::jaeger; - - macro_rules! launch { - ($fut:expr) => { - $fut - .timeout(Duration::from_millis(10)) - .await - .expect("10ms is more than enough for sending messages.") - }; - } - - /// A very limited state, only interested in the relay parent of the - /// given message, which must be signed by `validator` and a set of peers - /// which are also only interested in that relay parent. - fn prewarmed_state( - validator: ValidatorId, - signing_context: SigningContext, - known_message: BitfieldGossipMessage, - peers: Vec, - ) -> ProtocolState { - let relay_parent = known_message.relay_parent.clone(); - ProtocolState { - per_relay_parent: hashmap! { - relay_parent.clone() => - PerRelayParentData { - signing_context, - validator_set: vec![validator.clone()], - one_per_validator: hashmap! { - validator.clone() => known_message.clone(), - }, - message_received_from_peer: hashmap!{}, - message_sent_to_peer: hashmap!{}, - span: PerLeafSpan::new(Arc::new(jaeger::Span::Disabled), "test"), - }, - }, - peer_views: peers - .into_iter() - .map(|peer| (peer, view!(relay_parent))) - .collect(), - view: our_view!(relay_parent), - } - } - - fn state_with_view( - view: OurView, - relay_parent: Hash, - ) -> (ProtocolState, SigningContext, SyncCryptoStorePtr, ValidatorId) { - let mut state = ProtocolState::default(); - - let signing_context = SigningContext { - session_index: 1, - parent_hash: relay_parent.clone(), - }; - - let keystore : SyncCryptoStorePtr = Arc::new(KeyStore::new()); - let validator = SyncCryptoStore::sr25519_generate_new(&*keystore, ValidatorId::ID, None) - .expect("generating sr25519 key not to fail"); - - state.per_relay_parent = view.iter().map(|relay_parent| {( - relay_parent.clone(), - PerRelayParentData { - signing_context: signing_context.clone(), - validator_set: vec![validator.clone().into()], - one_per_validator: hashmap!{}, - message_received_from_peer: hashmap!{}, - message_sent_to_peer: hashmap!{}, - span: PerLeafSpan::new(Arc::new(jaeger::Span::Disabled), "test"), - }) - }).collect(); - - state.view = view; - - (state, signing_context, keystore, validator.into()) - } - - #[test] - fn receive_invalid_signature() { - let _ = env_logger::builder() - .filter(None, log::LevelFilter::Trace) - .is_test(true) - .try_init(); - - let hash_a: Hash = [0; 32].into(); - - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - assert_ne!(peer_a, peer_b); - - let signing_context = SigningContext { - session_index: 1, - parent_hash: hash_a.clone(), - }; - - // another validator not part of the validatorset - let keystore : SyncCryptoStorePtr = Arc::new(KeyStore::new()); - let malicious = SyncCryptoStore::sr25519_generate_new(&*keystore, ValidatorId::ID, None) - .expect("Malicious key created"); - let validator_0 = SyncCryptoStore::sr25519_generate_new(&*keystore, ValidatorId::ID, None) - .expect("key created"); - let validator_1 = SyncCryptoStore::sr25519_generate_new(&*keystore, ValidatorId::ID, None) - .expect("key created"); - - let payload = AvailabilityBitfield(bitvec![bitvec::order::Lsb0, u8; 1u8; 32]); - let invalid_signed = executor::block_on(Signed::::sign( - &keystore, - payload.clone(), - &signing_context, - ValidatorIndex(0), - &malicious.into(), - )).ok().flatten().expect("should be signed"); - let invalid_signed_2 = executor::block_on(Signed::::sign( - &keystore, - payload.clone(), - &signing_context, - ValidatorIndex(1), - &malicious.into(), - )).ok().flatten().expect("should be signed"); - - let valid_signed = executor::block_on(Signed::::sign( - &keystore, - payload, - &signing_context, - ValidatorIndex(0), - &validator_0.into(), - )).ok().flatten().expect("should be signed"); - - let invalid_msg = BitfieldGossipMessage { - relay_parent: hash_a.clone(), - signed_availability: invalid_signed.clone(), - }; - let invalid_msg_2 = BitfieldGossipMessage { - relay_parent: hash_a.clone(), - signed_availability: invalid_signed_2.clone(), - }; - let valid_msg = BitfieldGossipMessage { - relay_parent: hash_a.clone(), - signed_availability: valid_signed.clone(), - }; - - let pool = sp_core::testing::TaskExecutor::new(); - let (mut ctx, mut handle) = - make_subsystem_context::(pool); - - let mut state = prewarmed_state( - validator_0.into(), - signing_context.clone(), - valid_msg, - vec![peer_b.clone()], - ); - state.per_relay_parent.get_mut(&hash_a) - .unwrap() - .validator_set - .push(validator_1.into()); - - executor::block_on(async move { - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_b.clone(), invalid_msg.into_network_message()), - )); - - // reputation doesn't change due to one_job_per_validator check - assert!(handle.recv().timeout(Duration::from_millis(10)).await.is_none()); - - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_b.clone(), invalid_msg_2.into_network_message()), - )); - // reputation change due to invalid signature - assert_matches!( - handle.recv().await, - AllMessages::NetworkBridge( - NetworkBridgeMessage::ReportPeer(peer, rep) - ) => { - assert_eq!(peer, peer_b); - assert_eq!(rep, COST_SIGNATURE_INVALID) - } - ); - }); - } - - #[test] - fn receive_invalid_validator_index() { - let _ = env_logger::builder() - .filter(None, log::LevelFilter::Trace) - .is_test(true) - .try_init(); - - let hash_a: Hash = [0; 32].into(); - let hash_b: Hash = [1; 32].into(); // other - - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - assert_ne!(peer_a, peer_b); - - // validator 0 key pair - let (mut state, signing_context, keystore, validator) = state_with_view(our_view![hash_a, hash_b], hash_a.clone()); - - state.peer_views.insert(peer_b.clone(), view![hash_a]); - - let payload = AvailabilityBitfield(bitvec![bitvec::order::Lsb0, u8; 1u8; 32]); - let signed = executor::block_on(Signed::::sign( - &keystore, - payload, - &signing_context, - ValidatorIndex(42), - &validator, - )).ok().flatten().expect("should be signed"); - - let msg = BitfieldGossipMessage { - relay_parent: hash_a.clone(), - signed_availability: signed.clone(), - }; - - let pool = sp_core::testing::TaskExecutor::new(); - let (mut ctx, mut handle) = - make_subsystem_context::(pool); - - executor::block_on(async move { - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerMessage(peer_b.clone(), msg.into_network_message()), - )); - - // reputation change due to invalid validator index - assert_matches!( - handle.recv().await, - AllMessages::NetworkBridge( - NetworkBridgeMessage::ReportPeer(peer, rep) - ) => { - assert_eq!(peer, peer_b); - assert_eq!(rep, COST_VALIDATOR_INDEX_INVALID) - } - ); - }); - } - - #[test] - fn receive_duplicate_messages() { - let _ = env_logger::builder() - .filter(None, log::LevelFilter::Trace) - .is_test(true) - .try_init(); - - let hash_a: Hash = [0; 32].into(); - let hash_b: Hash = [1; 32].into(); - - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - assert_ne!(peer_a, peer_b); - - // validator 0 key pair - let (mut state, signing_context, keystore, validator) = state_with_view(our_view![hash_a, hash_b], hash_a.clone()); - - // create a signed message by validator 0 - let payload = AvailabilityBitfield(bitvec![bitvec::order::Lsb0, u8; 1u8; 32]); - let signed_bitfield = executor::block_on(Signed::::sign( - &keystore, - payload, - &signing_context, - ValidatorIndex(0), - &validator, - )).ok().flatten().expect("should be signed"); - - let msg = BitfieldGossipMessage { - relay_parent: hash_a.clone(), - signed_availability: signed_bitfield.clone(), - }; - - let pool = sp_core::testing::TaskExecutor::new(); - let (mut ctx, mut handle) = - make_subsystem_context::(pool); - - executor::block_on(async move { - // send a first message - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerMessage( - peer_b.clone(), - msg.clone().into_network_message(), - ), - )); - - // none of our peers has any interest in any messages - // so we do not receive a network send type message here - // but only the one for the next subsystem - assert_matches!( - handle.recv().await, - AllMessages::Provisioner(ProvisionerMessage::ProvisionableData( - _, - ProvisionableData::Bitfield(hash, signed) - )) => { - assert_eq!(hash, hash_a); - assert_eq!(signed, signed_bitfield) - } - ); - - assert_matches!( - handle.recv().await, - AllMessages::NetworkBridge( - NetworkBridgeMessage::ReportPeer(peer, rep) - ) => { - assert_eq!(peer, peer_b); - assert_eq!(rep, BENEFIT_VALID_MESSAGE_FIRST) - } - ); - - // let peer A send the same message again - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerMessage( - peer_a.clone(), - msg.clone().into_network_message(), - ), - )); - - assert_matches!( - handle.recv().await, - AllMessages::NetworkBridge( - NetworkBridgeMessage::ReportPeer(peer, rep) - ) => { - assert_eq!(peer, peer_a); - assert_eq!(rep, BENEFIT_VALID_MESSAGE) - } - ); - - // let peer B send the initial message again - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerMessage( - peer_b.clone(), - msg.clone().into_network_message(), - ), - )); - - assert_matches!( - handle.recv().await, - AllMessages::NetworkBridge( - NetworkBridgeMessage::ReportPeer(peer, rep) - ) => { - assert_eq!(peer, peer_b); - assert_eq!(rep, COST_PEER_DUPLICATE_MESSAGE) - } - ); - }); - } - - #[test] - fn do_not_relay_message_twice() { - let _ = env_logger::builder() - .filter(None, log::LevelFilter::Trace) - .is_test(true) - .try_init(); - - let hash = Hash::random(); - - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - assert_ne!(peer_a, peer_b); - - // validator 0 key pair - let (mut state, signing_context, keystore, validator) = state_with_view(our_view![hash], hash.clone()); - - // create a signed message by validator 0 - let payload = AvailabilityBitfield(bitvec![bitvec::order::Lsb0, u8; 1u8; 32]); - let signed_bitfield = executor::block_on(Signed::::sign( - &keystore, - payload, - &signing_context, - ValidatorIndex(0), - &validator, - )).ok().flatten().expect("should be signed"); - - state.peer_views.insert(peer_b.clone(), view![hash]); - state.peer_views.insert(peer_a.clone(), view![hash]); - - let msg = BitfieldGossipMessage { - relay_parent: hash.clone(), - signed_availability: signed_bitfield.clone(), - }; - - let pool = sp_core::testing::TaskExecutor::new(); - let (mut ctx, mut handle) = - make_subsystem_context::(pool); - - executor::block_on(async move { - relay_message( - &mut ctx, - state.per_relay_parent.get_mut(&hash).unwrap(), - &mut state.peer_views, - validator.clone(), - msg.clone(), - ).await; - - assert_matches!( - handle.recv().await, - AllMessages::Provisioner(ProvisionerMessage::ProvisionableData( - _, - ProvisionableData::Bitfield(h, signed) - )) => { - assert_eq!(h, hash); - assert_eq!(signed, signed_bitfield) - } - ); - - assert_matches!( - handle.recv().await, - AllMessages::NetworkBridge( - NetworkBridgeMessage::SendValidationMessage(peers, send_msg), - ) => { - assert_eq!(2, peers.len()); - assert!(peers.contains(&peer_a)); - assert!(peers.contains(&peer_b)); - assert_eq!(send_msg, msg.clone().into_validation_protocol()); - } - ); - - // Relaying the message a second time shouldn't work. - relay_message( - &mut ctx, - state.per_relay_parent.get_mut(&hash).unwrap(), - &mut state.peer_views, - validator.clone(), - msg.clone(), - ).await; - - assert_matches!( - handle.recv().await, - AllMessages::Provisioner(ProvisionerMessage::ProvisionableData( - _, - ProvisionableData::Bitfield(h, signed) - )) => { - assert_eq!(h, hash); - assert_eq!(signed, signed_bitfield) - } - ); - - // There shouldn't be any other message - assert!(handle.recv().timeout(Duration::from_millis(10)).await.is_none()); - }); - } - - #[test] - fn changing_view() { - let _ = env_logger::builder() - .filter(None, log::LevelFilter::Trace) - .is_test(true) - .try_init(); - - let hash_a: Hash = [0; 32].into(); - let hash_b: Hash = [1; 32].into(); - - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - assert_ne!(peer_a, peer_b); - - // validator 0 key pair - let (mut state, signing_context, keystore, validator) = state_with_view(our_view![hash_a, hash_b], hash_a.clone()); - - // create a signed message by validator 0 - let payload = AvailabilityBitfield(bitvec![bitvec::order::Lsb0, u8; 1u8; 32]); - let signed_bitfield = executor::block_on(Signed::::sign( - &keystore, - payload, - &signing_context, - ValidatorIndex(0), - &validator, - )).ok().flatten().expect("should be signed"); - - let msg = BitfieldGossipMessage { - relay_parent: hash_a.clone(), - signed_availability: signed_bitfield.clone(), - }; - - let pool = sp_core::testing::TaskExecutor::new(); - let (mut ctx, mut handle) = - make_subsystem_context::(pool); - - executor::block_on(async move { - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerConnected(peer_b.clone(), ObservedRole::Full, None), - )); - - // make peer b interested - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerViewChange(peer_b.clone(), view![hash_a, hash_b]), - )); - - assert!(state.peer_views.contains_key(&peer_b)); - - // recv a first message from the network - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerMessage( - peer_b.clone(), - msg.clone().into_network_message(), - ), - )); - - // gossip to the overseer - assert_matches!( - handle.recv().await, - AllMessages::Provisioner(ProvisionerMessage::ProvisionableData( - _, - ProvisionableData::Bitfield(hash, signed) - )) => { - assert_eq!(hash, hash_a); - assert_eq!(signed, signed_bitfield) - } - ); - - // reputation change for peer B - assert_matches!( - handle.recv().await, - AllMessages::NetworkBridge( - NetworkBridgeMessage::ReportPeer(peer, rep) - ) => { - assert_eq!(peer, peer_b); - assert_eq!(rep, BENEFIT_VALID_MESSAGE_FIRST) - } - ); - - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerViewChange(peer_b.clone(), view![]), - )); - - assert!(state.peer_views.contains_key(&peer_b)); - assert_eq!( - state.peer_views.get(&peer_b).expect("Must contain value for peer B"), - &view![] - ); - - // on rx of the same message, since we are not interested, - // should give penalty - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerMessage( - peer_b.clone(), - msg.clone().into_network_message(), - ), - )); - - // reputation change for peer B - assert_matches!( - handle.recv().await, - AllMessages::NetworkBridge( - NetworkBridgeMessage::ReportPeer(peer, rep) - ) => { - assert_eq!(peer, peer_b); - assert_eq!(rep, COST_PEER_DUPLICATE_MESSAGE) - } - ); - - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerDisconnected(peer_b.clone()), - )); - - // we are not interested in any peers at all anymore - state.view = our_view![]; - - // on rx of the same message, since we are not interested, - // should give penalty - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerMessage( - peer_a.clone(), - msg.clone().into_network_message(), - ), - )); - - // reputation change for peer B - assert_matches!( - handle.recv().await, - AllMessages::NetworkBridge( - NetworkBridgeMessage::ReportPeer(peer, rep) - ) => { - assert_eq!(peer, peer_a); - assert_eq!(rep, COST_NOT_IN_VIEW) - } - ); - - }); - } - - #[test] - fn do_not_send_message_back_to_origin() { - let _ = env_logger::builder() - .filter(None, log::LevelFilter::Trace) - .is_test(true) - .try_init(); - - let hash: Hash = [0; 32].into(); - - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - assert_ne!(peer_a, peer_b); - - // validator 0 key pair - let (mut state, signing_context, keystore, validator) = state_with_view(our_view![hash], hash); - - // create a signed message by validator 0 - let payload = AvailabilityBitfield(bitvec![bitvec::order::Lsb0, u8; 1u8; 32]); - let signed_bitfield = executor::block_on(Signed::::sign( - &keystore, - payload, - &signing_context, - ValidatorIndex(0), - &validator, - )).ok().flatten().expect("should be signed"); - - state.peer_views.insert(peer_b.clone(), view![hash]); - state.peer_views.insert(peer_a.clone(), view![hash]); - - let msg = BitfieldGossipMessage { - relay_parent: hash.clone(), - signed_availability: signed_bitfield.clone(), - }; - - let pool = sp_core::testing::TaskExecutor::new(); - let (mut ctx, mut handle) = - make_subsystem_context::(pool); - - executor::block_on(async move { - // send a first message - launch!(handle_network_msg( - &mut ctx, - &mut state, - &Default::default(), - NetworkBridgeEvent::PeerMessage( - peer_b.clone(), - msg.clone().into_network_message(), - ), - )); - - assert_matches!( - handle.recv().await, - AllMessages::Provisioner(ProvisionerMessage::ProvisionableData( - _, - ProvisionableData::Bitfield(hash, signed) - )) => { - assert_eq!(hash, hash); - assert_eq!(signed, signed_bitfield) - } - ); - - assert_matches!( - handle.recv().await, - AllMessages::NetworkBridge( - NetworkBridgeMessage::SendValidationMessage(peers, send_msg), - ) => { - assert_eq!(1, peers.len()); - assert!(peers.contains(&peer_a)); - assert_eq!(send_msg, msg.clone().into_validation_protocol()); - } - ); - - assert_matches!( - handle.recv().await, - AllMessages::NetworkBridge( - NetworkBridgeMessage::ReportPeer(peer, rep) - ) => { - assert_eq!(peer, peer_b); - assert_eq!(rep, BENEFIT_VALID_MESSAGE_FIRST) - } - ); - }); - } -} +mod tests; diff --git a/node/network/bitfield-distribution/src/tests.rs b/node/network/bitfield-distribution/src/tests.rs new file mode 100644 index 000000000000..afe08bc66f94 --- /dev/null +++ b/node/network/bitfield-distribution/src/tests.rs @@ -0,0 +1,710 @@ +use super::*; +use bitvec::bitvec; +use futures::executor; +use maplit::hashmap; +use polkadot_primitives::v1::{Signed, AvailabilityBitfield, ValidatorIndex}; +use polkadot_node_subsystem_test_helpers::make_subsystem_context; +use polkadot_node_subsystem_util::TimeoutExt; +use sp_keystore::{SyncCryptoStorePtr, SyncCryptoStore}; +use sp_application_crypto::AppKey; +use sp_keystore::testing::KeyStore; +use std::sync::Arc; +use std::time::Duration; +use assert_matches::assert_matches; +use polkadot_node_network_protocol::{view, ObservedRole, our_view}; +use polkadot_subsystem::jaeger; + +macro_rules! launch { + ($fut:expr) => { + $fut + .timeout(Duration::from_millis(10)) + .await + .expect("10ms is more than enough for sending messages.") + }; +} + +/// A very limited state, only interested in the relay parent of the +/// given message, which must be signed by `validator` and a set of peers +/// which are also only interested in that relay parent. +fn prewarmed_state( + validator: ValidatorId, + signing_context: SigningContext, + known_message: BitfieldGossipMessage, + peers: Vec, +) -> ProtocolState { + let relay_parent = known_message.relay_parent.clone(); + ProtocolState { + per_relay_parent: hashmap! { + relay_parent.clone() => + PerRelayParentData { + signing_context, + validator_set: vec![validator.clone()], + one_per_validator: hashmap! { + validator.clone() => known_message.clone(), + }, + message_received_from_peer: hashmap!{}, + message_sent_to_peer: hashmap!{}, + span: PerLeafSpan::new(Arc::new(jaeger::Span::Disabled), "test"), + }, + }, + peer_views: peers + .into_iter() + .map(|peer| (peer, view!(relay_parent))) + .collect(), + view: our_view!(relay_parent), + } +} + +fn state_with_view( + view: OurView, + relay_parent: Hash, +) -> (ProtocolState, SigningContext, SyncCryptoStorePtr, ValidatorId) { + let mut state = ProtocolState::default(); + + let signing_context = SigningContext { + session_index: 1, + parent_hash: relay_parent.clone(), + }; + + let keystore : SyncCryptoStorePtr = Arc::new(KeyStore::new()); + let validator = SyncCryptoStore::sr25519_generate_new(&*keystore, ValidatorId::ID, None) + .expect("generating sr25519 key not to fail"); + + state.per_relay_parent = view.iter().map(|relay_parent| {( + relay_parent.clone(), + PerRelayParentData { + signing_context: signing_context.clone(), + validator_set: vec![validator.clone().into()], + one_per_validator: hashmap!{}, + message_received_from_peer: hashmap!{}, + message_sent_to_peer: hashmap!{}, + span: PerLeafSpan::new(Arc::new(jaeger::Span::Disabled), "test"), + }) + }).collect(); + + state.view = view; + + (state, signing_context, keystore, validator.into()) +} + +#[test] +fn receive_invalid_signature() { + let _ = env_logger::builder() + .filter(None, log::LevelFilter::Trace) + .is_test(true) + .try_init(); + + let hash_a: Hash = [0; 32].into(); + + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + assert_ne!(peer_a, peer_b); + + let signing_context = SigningContext { + session_index: 1, + parent_hash: hash_a.clone(), + }; + + // another validator not part of the validatorset + let keystore : SyncCryptoStorePtr = Arc::new(KeyStore::new()); + let malicious = SyncCryptoStore::sr25519_generate_new(&*keystore, ValidatorId::ID, None) + .expect("Malicious key created"); + let validator_0 = SyncCryptoStore::sr25519_generate_new(&*keystore, ValidatorId::ID, None) + .expect("key created"); + let validator_1 = SyncCryptoStore::sr25519_generate_new(&*keystore, ValidatorId::ID, None) + .expect("key created"); + + let payload = AvailabilityBitfield(bitvec![bitvec::order::Lsb0, u8; 1u8; 32]); + let invalid_signed = executor::block_on(Signed::::sign( + &keystore, + payload.clone(), + &signing_context, + ValidatorIndex(0), + &malicious.into(), + )).ok().flatten().expect("should be signed"); + let invalid_signed_2 = executor::block_on(Signed::::sign( + &keystore, + payload.clone(), + &signing_context, + ValidatorIndex(1), + &malicious.into(), + )).ok().flatten().expect("should be signed"); + + let valid_signed = executor::block_on(Signed::::sign( + &keystore, + payload, + &signing_context, + ValidatorIndex(0), + &validator_0.into(), + )).ok().flatten().expect("should be signed"); + + let invalid_msg = BitfieldGossipMessage { + relay_parent: hash_a.clone(), + signed_availability: invalid_signed.clone(), + }; + let invalid_msg_2 = BitfieldGossipMessage { + relay_parent: hash_a.clone(), + signed_availability: invalid_signed_2.clone(), + }; + let valid_msg = BitfieldGossipMessage { + relay_parent: hash_a.clone(), + signed_availability: valid_signed.clone(), + }; + + let pool = sp_core::testing::TaskExecutor::new(); + let (mut ctx, mut handle) = + make_subsystem_context::(pool); + + let mut state = prewarmed_state( + validator_0.into(), + signing_context.clone(), + valid_msg, + vec![peer_b.clone()], + ); + state.per_relay_parent.get_mut(&hash_a) + .unwrap() + .validator_set + .push(validator_1.into()); + + executor::block_on(async move { + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerMessage(peer_b.clone(), invalid_msg.into_network_message()), + )); + + // reputation doesn't change due to one_job_per_validator check + assert!(handle.recv().timeout(Duration::from_millis(10)).await.is_none()); + + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerMessage(peer_b.clone(), invalid_msg_2.into_network_message()), + )); + // reputation change due to invalid signature + assert_matches!( + handle.recv().await, + AllMessages::NetworkBridge( + NetworkBridgeMessage::ReportPeer(peer, rep) + ) => { + assert_eq!(peer, peer_b); + assert_eq!(rep, COST_SIGNATURE_INVALID) + } + ); + }); +} + +#[test] +fn receive_invalid_validator_index() { + let _ = env_logger::builder() + .filter(None, log::LevelFilter::Trace) + .is_test(true) + .try_init(); + + let hash_a: Hash = [0; 32].into(); + let hash_b: Hash = [1; 32].into(); // other + + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + assert_ne!(peer_a, peer_b); + + // validator 0 key pair + let (mut state, signing_context, keystore, validator) = state_with_view(our_view![hash_a, hash_b], hash_a.clone()); + + state.peer_views.insert(peer_b.clone(), view![hash_a]); + + let payload = AvailabilityBitfield(bitvec![bitvec::order::Lsb0, u8; 1u8; 32]); + let signed = executor::block_on(Signed::::sign( + &keystore, + payload, + &signing_context, + ValidatorIndex(42), + &validator, + )).ok().flatten().expect("should be signed"); + + let msg = BitfieldGossipMessage { + relay_parent: hash_a.clone(), + signed_availability: signed.clone(), + }; + + let pool = sp_core::testing::TaskExecutor::new(); + let (mut ctx, mut handle) = + make_subsystem_context::(pool); + + executor::block_on(async move { + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerMessage(peer_b.clone(), msg.into_network_message()), + )); + + // reputation change due to invalid validator index + assert_matches!( + handle.recv().await, + AllMessages::NetworkBridge( + NetworkBridgeMessage::ReportPeer(peer, rep) + ) => { + assert_eq!(peer, peer_b); + assert_eq!(rep, COST_VALIDATOR_INDEX_INVALID) + } + ); + }); +} + +#[test] +fn receive_duplicate_messages() { + let _ = env_logger::builder() + .filter(None, log::LevelFilter::Trace) + .is_test(true) + .try_init(); + + let hash_a: Hash = [0; 32].into(); + let hash_b: Hash = [1; 32].into(); + + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + assert_ne!(peer_a, peer_b); + + // validator 0 key pair + let (mut state, signing_context, keystore, validator) = state_with_view(our_view![hash_a, hash_b], hash_a.clone()); + + // create a signed message by validator 0 + let payload = AvailabilityBitfield(bitvec![bitvec::order::Lsb0, u8; 1u8; 32]); + let signed_bitfield = executor::block_on(Signed::::sign( + &keystore, + payload, + &signing_context, + ValidatorIndex(0), + &validator, + )).ok().flatten().expect("should be signed"); + + let msg = BitfieldGossipMessage { + relay_parent: hash_a.clone(), + signed_availability: signed_bitfield.clone(), + }; + + let pool = sp_core::testing::TaskExecutor::new(); + let (mut ctx, mut handle) = + make_subsystem_context::(pool); + + executor::block_on(async move { + // send a first message + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + msg.clone().into_network_message(), + ), + )); + + // none of our peers has any interest in any messages + // so we do not receive a network send type message here + // but only the one for the next subsystem + assert_matches!( + handle.recv().await, + AllMessages::Provisioner(ProvisionerMessage::ProvisionableData( + _, + ProvisionableData::Bitfield(hash, signed) + )) => { + assert_eq!(hash, hash_a); + assert_eq!(signed, signed_bitfield) + } + ); + + assert_matches!( + handle.recv().await, + AllMessages::NetworkBridge( + NetworkBridgeMessage::ReportPeer(peer, rep) + ) => { + assert_eq!(peer, peer_b); + assert_eq!(rep, BENEFIT_VALID_MESSAGE_FIRST) + } + ); + + // let peer A send the same message again + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerMessage( + peer_a.clone(), + msg.clone().into_network_message(), + ), + )); + + assert_matches!( + handle.recv().await, + AllMessages::NetworkBridge( + NetworkBridgeMessage::ReportPeer(peer, rep) + ) => { + assert_eq!(peer, peer_a); + assert_eq!(rep, BENEFIT_VALID_MESSAGE) + } + ); + + // let peer B send the initial message again + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + msg.clone().into_network_message(), + ), + )); + + assert_matches!( + handle.recv().await, + AllMessages::NetworkBridge( + NetworkBridgeMessage::ReportPeer(peer, rep) + ) => { + assert_eq!(peer, peer_b); + assert_eq!(rep, COST_PEER_DUPLICATE_MESSAGE) + } + ); + }); +} + +#[test] +fn do_not_relay_message_twice() { + let _ = env_logger::builder() + .filter(None, log::LevelFilter::Trace) + .is_test(true) + .try_init(); + + let hash = Hash::random(); + + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + assert_ne!(peer_a, peer_b); + + // validator 0 key pair + let (mut state, signing_context, keystore, validator) = state_with_view(our_view![hash], hash.clone()); + + // create a signed message by validator 0 + let payload = AvailabilityBitfield(bitvec![bitvec::order::Lsb0, u8; 1u8; 32]); + let signed_bitfield = executor::block_on(Signed::::sign( + &keystore, + payload, + &signing_context, + ValidatorIndex(0), + &validator, + )).ok().flatten().expect("should be signed"); + + state.peer_views.insert(peer_b.clone(), view![hash]); + state.peer_views.insert(peer_a.clone(), view![hash]); + + let msg = BitfieldGossipMessage { + relay_parent: hash.clone(), + signed_availability: signed_bitfield.clone(), + }; + + let pool = sp_core::testing::TaskExecutor::new(); + let (mut ctx, mut handle) = + make_subsystem_context::(pool); + + executor::block_on(async move { + relay_message( + &mut ctx, + state.per_relay_parent.get_mut(&hash).unwrap(), + &mut state.peer_views, + validator.clone(), + msg.clone(), + ).await; + + assert_matches!( + handle.recv().await, + AllMessages::Provisioner(ProvisionerMessage::ProvisionableData( + _, + ProvisionableData::Bitfield(h, signed) + )) => { + assert_eq!(h, hash); + assert_eq!(signed, signed_bitfield) + } + ); + + assert_matches!( + handle.recv().await, + AllMessages::NetworkBridge( + NetworkBridgeMessage::SendValidationMessage(peers, send_msg), + ) => { + assert_eq!(2, peers.len()); + assert!(peers.contains(&peer_a)); + assert!(peers.contains(&peer_b)); + assert_eq!(send_msg, msg.clone().into_validation_protocol()); + } + ); + + // Relaying the message a second time shouldn't work. + relay_message( + &mut ctx, + state.per_relay_parent.get_mut(&hash).unwrap(), + &mut state.peer_views, + validator.clone(), + msg.clone(), + ).await; + + assert_matches!( + handle.recv().await, + AllMessages::Provisioner(ProvisionerMessage::ProvisionableData( + _, + ProvisionableData::Bitfield(h, signed) + )) => { + assert_eq!(h, hash); + assert_eq!(signed, signed_bitfield) + } + ); + + // There shouldn't be any other message + assert!(handle.recv().timeout(Duration::from_millis(10)).await.is_none()); + }); +} + +#[test] +fn changing_view() { + let _ = env_logger::builder() + .filter(None, log::LevelFilter::Trace) + .is_test(true) + .try_init(); + + let hash_a: Hash = [0; 32].into(); + let hash_b: Hash = [1; 32].into(); + + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + assert_ne!(peer_a, peer_b); + + // validator 0 key pair + let (mut state, signing_context, keystore, validator) = state_with_view(our_view![hash_a, hash_b], hash_a.clone()); + + // create a signed message by validator 0 + let payload = AvailabilityBitfield(bitvec![bitvec::order::Lsb0, u8; 1u8; 32]); + let signed_bitfield = executor::block_on(Signed::::sign( + &keystore, + payload, + &signing_context, + ValidatorIndex(0), + &validator, + )).ok().flatten().expect("should be signed"); + + let msg = BitfieldGossipMessage { + relay_parent: hash_a.clone(), + signed_availability: signed_bitfield.clone(), + }; + + let pool = sp_core::testing::TaskExecutor::new(); + let (mut ctx, mut handle) = + make_subsystem_context::(pool); + + executor::block_on(async move { + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerConnected(peer_b.clone(), ObservedRole::Full, None), + )); + + // make peer b interested + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerViewChange(peer_b.clone(), view![hash_a, hash_b]), + )); + + assert!(state.peer_views.contains_key(&peer_b)); + + // recv a first message from the network + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + msg.clone().into_network_message(), + ), + )); + + // gossip to the overseer + assert_matches!( + handle.recv().await, + AllMessages::Provisioner(ProvisionerMessage::ProvisionableData( + _, + ProvisionableData::Bitfield(hash, signed) + )) => { + assert_eq!(hash, hash_a); + assert_eq!(signed, signed_bitfield) + } + ); + + // reputation change for peer B + assert_matches!( + handle.recv().await, + AllMessages::NetworkBridge( + NetworkBridgeMessage::ReportPeer(peer, rep) + ) => { + assert_eq!(peer, peer_b); + assert_eq!(rep, BENEFIT_VALID_MESSAGE_FIRST) + } + ); + + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerViewChange(peer_b.clone(), view![]), + )); + + assert!(state.peer_views.contains_key(&peer_b)); + assert_eq!( + state.peer_views.get(&peer_b).expect("Must contain value for peer B"), + &view![] + ); + + // on rx of the same message, since we are not interested, + // should give penalty + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + msg.clone().into_network_message(), + ), + )); + + // reputation change for peer B + assert_matches!( + handle.recv().await, + AllMessages::NetworkBridge( + NetworkBridgeMessage::ReportPeer(peer, rep) + ) => { + assert_eq!(peer, peer_b); + assert_eq!(rep, COST_PEER_DUPLICATE_MESSAGE) + } + ); + + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerDisconnected(peer_b.clone()), + )); + + // we are not interested in any peers at all anymore + state.view = our_view![]; + + // on rx of the same message, since we are not interested, + // should give penalty + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerMessage( + peer_a.clone(), + msg.clone().into_network_message(), + ), + )); + + // reputation change for peer B + assert_matches!( + handle.recv().await, + AllMessages::NetworkBridge( + NetworkBridgeMessage::ReportPeer(peer, rep) + ) => { + assert_eq!(peer, peer_a); + assert_eq!(rep, COST_NOT_IN_VIEW) + } + ); + + }); +} + +#[test] +fn do_not_send_message_back_to_origin() { + let _ = env_logger::builder() + .filter(None, log::LevelFilter::Trace) + .is_test(true) + .try_init(); + + let hash: Hash = [0; 32].into(); + + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + assert_ne!(peer_a, peer_b); + + // validator 0 key pair + let (mut state, signing_context, keystore, validator) = state_with_view(our_view![hash], hash); + + // create a signed message by validator 0 + let payload = AvailabilityBitfield(bitvec![bitvec::order::Lsb0, u8; 1u8; 32]); + let signed_bitfield = executor::block_on(Signed::::sign( + &keystore, + payload, + &signing_context, + ValidatorIndex(0), + &validator, + )).ok().flatten().expect("should be signed"); + + state.peer_views.insert(peer_b.clone(), view![hash]); + state.peer_views.insert(peer_a.clone(), view![hash]); + + let msg = BitfieldGossipMessage { + relay_parent: hash.clone(), + signed_availability: signed_bitfield.clone(), + }; + + let pool = sp_core::testing::TaskExecutor::new(); + let (mut ctx, mut handle) = + make_subsystem_context::(pool); + + executor::block_on(async move { + // send a first message + launch!(handle_network_msg( + &mut ctx, + &mut state, + &Default::default(), + NetworkBridgeEvent::PeerMessage( + peer_b.clone(), + msg.clone().into_network_message(), + ), + )); + + assert_matches!( + handle.recv().await, + AllMessages::Provisioner(ProvisionerMessage::ProvisionableData( + _, + ProvisionableData::Bitfield(hash, signed) + )) => { + assert_eq!(hash, hash); + assert_eq!(signed, signed_bitfield) + } + ); + + assert_matches!( + handle.recv().await, + AllMessages::NetworkBridge( + NetworkBridgeMessage::SendValidationMessage(peers, send_msg), + ) => { + assert_eq!(1, peers.len()); + assert!(peers.contains(&peer_a)); + assert_eq!(send_msg, msg.clone().into_validation_protocol()); + } + ); + + assert_matches!( + handle.recv().await, + AllMessages::NetworkBridge( + NetworkBridgeMessage::ReportPeer(peer, rep) + ) => { + assert_eq!(peer, peer_b); + assert_eq!(rep, BENEFIT_VALID_MESSAGE_FIRST) + } + ); + }); +} diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index bd4134ecea10..d9f236d5aae3 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -902,7 +902,7 @@ fn construct_view(live_heads: impl DoubleEndedIterator, finalized_n #[tracing::instrument(level = "trace", skip(net, ctx, shared, metrics), fields(subsystem = LOG_TARGET))] async fn update_our_view( net: &mut impl Network, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, live_heads: &[ActivatedLeaf], shared: &Shared, finalized_number: BlockNumber, @@ -1114,1301 +1114,5 @@ async fn dispatch_collation_events_to_all( ctx.send_messages(events.into_iter().flat_map(messages_for)).await } - - - #[cfg(test)] -mod tests { - use super::*; - use futures::executor; - use futures::stream::BoxStream; - use futures::channel::oneshot; - - use std::borrow::Cow; - use std::collections::HashSet; - use std::pin::Pin; - use std::sync::atomic::{AtomicBool, Ordering}; - use async_trait::async_trait; - use parking_lot::Mutex; - use assert_matches::assert_matches; - - use sc_network::{Event as NetworkEvent, IfDisconnected}; - - use polkadot_subsystem::{jaeger, ActiveLeavesUpdate, FromOverseer, OverseerSignal}; - use polkadot_subsystem::messages::{ - ApprovalDistributionMessage, - BitfieldDistributionMessage, - StatementDistributionMessage - }; - use polkadot_node_subsystem_test_helpers::{ - SingleItemSink, SingleItemStream, TestSubsystemContextHandle, - }; - use polkadot_node_subsystem_util::metered; - use polkadot_node_network_protocol::view; - use sc_network::Multiaddr; - use sc_network::config::RequestResponseConfig; - use sp_keyring::Sr25519Keyring; - use polkadot_primitives::v1::AuthorityDiscoveryId; - use polkadot_node_network_protocol::{ObservedRole, request_response::request::Requests}; - - use crate::network::{Network, NetworkAction}; - use crate::validator_discovery::AuthorityDiscovery; - - // The subsystem's view of the network - only supports a single call to `event_stream`. - #[derive(Clone)] - struct TestNetwork { - net_events: Arc>>>, - action_tx: metered::UnboundedMeteredSender, - _req_configs: Vec, - } - - #[derive(Clone)] - struct TestAuthorityDiscovery; - - // The test's view of the network. This receives updates from the subsystem in the form - // of `NetworkAction`s. - struct TestNetworkHandle { - action_rx: metered::UnboundedMeteredReceiver, - net_tx: SingleItemSink, - } - - fn new_test_network(req_configs: Vec) -> ( - TestNetwork, - TestNetworkHandle, - TestAuthorityDiscovery, - ) { - let (net_tx, net_rx) = polkadot_node_subsystem_test_helpers::single_item_sink(); - let (action_tx, action_rx) = metered::unbounded(); - - ( - TestNetwork { - net_events: Arc::new(Mutex::new(Some(net_rx))), - action_tx, - _req_configs: req_configs, - }, - TestNetworkHandle { - action_rx, - net_tx, - }, - TestAuthorityDiscovery, - ) - } - - #[async_trait] - impl Network for TestNetwork { - fn event_stream(&mut self) -> BoxStream<'static, NetworkEvent> { - self.net_events.lock() - .take() - .expect("Subsystem made more than one call to `event_stream`") - .boxed() - } - - async fn add_to_peers_set(&mut self, _protocol: Cow<'static, str>, _: HashSet) -> Result<(), String> { - Ok(()) - } - - async fn remove_from_peers_set(&mut self, _protocol: Cow<'static, str>, _: HashSet) -> Result<(), String> { - Ok(()) - } - - fn action_sink<'a>(&'a mut self) - -> Pin + Send + 'a>> - { - Box::pin((&mut self.action_tx).sink_map_err(Into::into)) - } - - async fn start_request(&self, _: &mut AD, _: Requests, _: IfDisconnected) { - } - } - - #[async_trait] - impl validator_discovery::AuthorityDiscovery for TestAuthorityDiscovery { - async fn get_addresses_by_authority_id(&mut self, _authority: AuthorityDiscoveryId) -> Option> { - None - } - - async fn get_authority_id_by_peer_id(&mut self, _peer_id: PeerId) -> Option { - None - } - } - - impl TestNetworkHandle { - // Get the next network action. - async fn next_network_action(&mut self) -> NetworkAction { - self.action_rx.next().await.expect("subsystem concluded early") - } - - // Wait for the next N network actions. - async fn next_network_actions(&mut self, n: usize) -> Vec { - let mut v = Vec::with_capacity(n); - for _ in 0..n { - v.push(self.next_network_action().await); - } - - v - } - - async fn connect_peer(&mut self, peer: PeerId, peer_set: PeerSet, role: ObservedRole) { - self.send_network_event(NetworkEvent::NotificationStreamOpened { - remote: peer, - protocol: peer_set.into_protocol_name(), - negotiated_fallback: None, - role: role.into(), - }).await; - } - - async fn disconnect_peer(&mut self, peer: PeerId, peer_set: PeerSet) { - self.send_network_event(NetworkEvent::NotificationStreamClosed { - remote: peer, - protocol: peer_set.into_protocol_name(), - }).await; - } - - async fn peer_message(&mut self, peer: PeerId, peer_set: PeerSet, message: Vec) { - self.send_network_event(NetworkEvent::NotificationsReceived { - remote: peer, - messages: vec![(peer_set.into_protocol_name(), message.into())], - }).await; - } - - async fn send_network_event(&mut self, event: NetworkEvent) { - self.net_tx.send(event).await.expect("subsystem concluded early"); - } - } - - /// Assert that the given actions contain the given `action`. - fn assert_network_actions_contains(actions: &[NetworkAction], action: &NetworkAction) { - if !actions.iter().any(|x| x == action) { - panic!("Could not find `{:?}` in `{:?}`", action, actions); - } - } - - #[derive(Clone)] - struct TestSyncOracle { - flag: Arc, - done_syncing_sender: Arc>>>, - } - - struct TestSyncOracleHandle { - done_syncing_receiver: oneshot::Receiver<()>, - flag: Arc, - } - - impl TestSyncOracleHandle { - fn set_done(&self) { - self.flag.store(false, Ordering::SeqCst); - } - - async fn await_mode_switch(self) { - let _ = self.done_syncing_receiver.await; - } - } - - impl SyncOracle for TestSyncOracle { - fn is_major_syncing(&mut self) -> bool { - let is_major_syncing = self.flag.load(Ordering::SeqCst); - - if !is_major_syncing { - if let Some(sender) = self.done_syncing_sender.lock().take() { - let _ = sender.send(()); - } - } - - is_major_syncing - } - - fn is_offline(&mut self) -> bool { - unimplemented!("not used in network bridge") - } - } - - // val - result of `is_major_syncing`. - fn make_sync_oracle(val: bool) -> (TestSyncOracle, TestSyncOracleHandle) { - let (tx, rx) = oneshot::channel(); - let flag = Arc::new(AtomicBool::new(val)); - - ( - TestSyncOracle { - flag: flag.clone(), - done_syncing_sender: Arc::new(Mutex::new(Some(tx))), - }, - TestSyncOracleHandle { - flag, - done_syncing_receiver: rx, - } - ) - } - - fn done_syncing_oracle() -> Box { - let (oracle, _) = make_sync_oracle(false); - Box::new(oracle) - } - - type VirtualOverseer = TestSubsystemContextHandle; - - struct TestHarness { - network_handle: TestNetworkHandle, - virtual_overseer: VirtualOverseer, - } - - fn test_harness>( - sync_oracle: Box, - test: impl FnOnce(TestHarness) -> T, - ) { - let pool = sp_core::testing::TaskExecutor::new(); - let (request_multiplexer, req_configs) = RequestMultiplexer::new(); - let (mut network, network_handle, discovery) = new_test_network(req_configs); - let (context, virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context(pool); - let network_stream = network.event_stream(); - - let bridge = NetworkBridge { - network_service: network, - authority_discovery_service: discovery, - request_multiplexer, - metrics: Metrics(None), - sync_oracle, - }; - - let network_bridge = run_network( - bridge, - context, - network_stream, - ) - .map_err(|_| panic!("subsystem execution failed")) - .map(|_| ()); - - let test_fut = test(TestHarness { - network_handle, - virtual_overseer, - }); - - futures::pin_mut!(test_fut); - futures::pin_mut!(network_bridge); - - let _ = executor::block_on(future::join(async move { - let mut virtual_overseer = test_fut.await; - virtual_overseer.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }, network_bridge)); - } - - async fn assert_sends_validation_event_to_all( - event: NetworkBridgeEvent, - virtual_overseer: &mut TestSubsystemContextHandle, - ) { - // Ordering must match the enum variant order - // in `AllMessages`. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::StatementDistribution( - StatementDistributionMessage::NetworkBridgeUpdateV1(e) - ) if e == event.focus().expect("could not focus message") - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::BitfieldDistribution( - BitfieldDistributionMessage::NetworkBridgeUpdateV1(e) - ) if e == event.focus().expect("could not focus message") - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::ApprovalDistribution( - ApprovalDistributionMessage::NetworkBridgeUpdateV1(e) - ) if e == event.focus().expect("could not focus message") - ); - } - - async fn assert_sends_collation_event_to_all( - event: NetworkBridgeEvent, - virtual_overseer: &mut TestSubsystemContextHandle, - ) { - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CollatorProtocol( - CollatorProtocolMessage::NetworkBridgeUpdateV1(e) - ) if e == event.focus().expect("could not focus message") - ) - } - - #[test] - fn send_our_view_upon_connection() { - let (oracle, handle) = make_sync_oracle(false); - test_harness(Box::new(oracle), |test_harness| async move { - let TestHarness { - mut network_handle, - mut virtual_overseer, - } = test_harness; - - let peer = PeerId::random(); - - let head = Hash::repeat_byte(1); - virtual_overseer.send( - FromOverseer::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: head, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - }) - )) - ).await; - - handle.await_mode_switch().await; - - network_handle.connect_peer(peer.clone(), PeerSet::Validation, ObservedRole::Full).await; - network_handle.connect_peer(peer.clone(), PeerSet::Collation, ObservedRole::Full).await; - - let view = view![head]; - let actions = network_handle.next_network_actions(2).await; - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer.clone(), - PeerSet::Validation, - WireMessage::::ViewUpdate( - view.clone(), - ).encode(), - ), - ); - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer.clone(), - PeerSet::Collation, - WireMessage::::ViewUpdate( - view.clone(), - ).encode(), - ), - ); - virtual_overseer - }); - } - - #[test] - fn sends_view_updates_to_peers() { - let (oracle, handle) = make_sync_oracle(false); - test_harness(Box::new(oracle), |test_harness| async move { - let TestHarness { mut network_handle, mut virtual_overseer } = test_harness; - - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - - virtual_overseer.send( - FromOverseer::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate { - activated: Default::default(), - deactivated: Default::default(), - } - )) - ).await; - - handle.await_mode_switch().await; - - network_handle.connect_peer( - peer_a.clone(), - PeerSet::Validation, - ObservedRole::Full, - ).await; - network_handle.connect_peer( - peer_b.clone(), - PeerSet::Collation, - ObservedRole::Full, - ).await; - - let actions = network_handle.next_network_actions(2).await; - let wire_message = WireMessage::::ViewUpdate( - View::default(), - ).encode(); - - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer_a, - PeerSet::Validation, - wire_message.clone(), - ), - ); - - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer_b, - PeerSet::Collation, - wire_message.clone(), - ), - ); - - let hash_a = Hash::repeat_byte(1); - - virtual_overseer.send( - FromOverseer::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - }) - )) - ).await; - - let actions = network_handle.next_network_actions(2).await; - let wire_message = WireMessage::::ViewUpdate( - view![hash_a] - ).encode(); - - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer_a, - PeerSet::Validation, - wire_message.clone(), - ), - ); - - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer_b, - PeerSet::Collation, - wire_message.clone(), - ), - ); - virtual_overseer - }); - } - - #[test] - fn do_not_send_view_update_until_synced() { - let (oracle, handle) = make_sync_oracle(true); - test_harness(Box::new(oracle), |test_harness| async move { - let TestHarness { mut network_handle, mut virtual_overseer } = test_harness; - - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - - network_handle.connect_peer( - peer_a.clone(), - PeerSet::Validation, - ObservedRole::Full, - ).await; - network_handle.connect_peer( - peer_b.clone(), - PeerSet::Collation, - ObservedRole::Full, - ).await; - - { - let actions = network_handle.next_network_actions(2).await; - let wire_message = WireMessage::::ViewUpdate( - View::default(), - ).encode(); - - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer_a, - PeerSet::Validation, - wire_message.clone(), - ), - ); - - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer_b, - PeerSet::Collation, - wire_message.clone(), - ), - ); - } - - let hash_a = Hash::repeat_byte(1); - let hash_b = Hash::repeat_byte(1); - - virtual_overseer.send( - FromOverseer::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - }) - )) - ).await; - - // delay until the previous update has certainly been processed. - futures_timer::Delay::new(std::time::Duration::from_millis(100)).await; - - handle.set_done(); - - virtual_overseer.send( - FromOverseer::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_b, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - }) - )) - ).await; - - handle.await_mode_switch().await; - - // There should be a mode switch only for the second view update. - { - let actions = network_handle.next_network_actions(2).await; - let wire_message = WireMessage::::ViewUpdate( - view![hash_a, hash_b] - ).encode(); - - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer_a, - PeerSet::Validation, - wire_message.clone(), - ), - ); - - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer_b, - PeerSet::Collation, - wire_message.clone(), - ), - ); - } - virtual_overseer - }); - } - - #[test] - fn do_not_send_view_update_when_only_finalized_block_changed() { - test_harness(done_syncing_oracle(), |test_harness| async move { - let TestHarness { mut network_handle, mut virtual_overseer } = test_harness; - - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - - network_handle.connect_peer( - peer_a.clone(), - PeerSet::Validation, - ObservedRole::Full, - ).await; - network_handle.connect_peer( - peer_b.clone(), - PeerSet::Validation, - ObservedRole::Full, - ).await; - - let hash_a = Hash::repeat_byte(1); - - virtual_overseer.send(FromOverseer::Signal(OverseerSignal::BlockFinalized(Hash::random(), 5))).await; - - // Send some empty active leaves update - // - // This should not trigger a view update to our peers. - virtual_overseer.send( - FromOverseer::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::default())) - ).await; - - // This should trigger the view update to our peers. - virtual_overseer.send( - FromOverseer::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - }) - )) - ).await; - - let actions = network_handle.next_network_actions(4).await; - let wire_message = WireMessage::::ViewUpdate( - View::new(vec![hash_a], 5) - ).encode(); - - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer_a, - PeerSet::Validation, - wire_message.clone(), - ), - ); - - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer_b, - PeerSet::Validation, - wire_message.clone(), - ), - ); - virtual_overseer - }); - } - - #[test] - fn peer_view_updates_sent_via_overseer() { - test_harness(done_syncing_oracle(), |test_harness| async move { - let TestHarness { - mut network_handle, - mut virtual_overseer, - } = test_harness; - - let peer = PeerId::random(); - - network_handle.connect_peer(peer.clone(), PeerSet::Validation, ObservedRole::Full).await; - - let view = view![Hash::repeat_byte(1)]; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::StatementDistribution( - StatementDistributionMessage::StatementFetchingReceiver(_) - ) - ); - - // bridge will inform about all connected peers. - { - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), - &mut virtual_overseer, - ).await; - - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), - &mut virtual_overseer, - ).await; - } - - network_handle.peer_message( - peer.clone(), - PeerSet::Validation, - WireMessage::::ViewUpdate( - view.clone(), - ).encode(), - ).await; - - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer.clone(), view), - &mut virtual_overseer, - ).await; - virtual_overseer - }); - } - - #[test] - fn peer_messages_sent_via_overseer() { - test_harness(done_syncing_oracle(), |test_harness| async move { - let TestHarness { - mut network_handle, - mut virtual_overseer, - } = test_harness; - - let peer = PeerId::random(); - - network_handle.connect_peer( - peer.clone(), - PeerSet::Validation, - ObservedRole::Full, - ).await; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::StatementDistribution( - StatementDistributionMessage::StatementFetchingReceiver(_) - ) - ); - - // bridge will inform about all connected peers. - { - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), - &mut virtual_overseer, - ).await; - - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), - &mut virtual_overseer, - ).await; - } - - let approval_distribution_message = protocol_v1::ApprovalDistributionMessage::Approvals( - Vec::new() - ); - - let message = protocol_v1::ValidationProtocol::ApprovalDistribution( - approval_distribution_message.clone(), - ); - - network_handle.peer_message( - peer.clone(), - PeerSet::Validation, - WireMessage::ProtocolMessage(message.clone()).encode(), - ).await; - - network_handle.disconnect_peer(peer.clone(), PeerSet::Validation).await; - - // Approval distribution message comes first, and the message is only sent to that subsystem. - // then a disconnection event arises that is sent to all validation networking subsystems. - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::ApprovalDistribution( - ApprovalDistributionMessage::NetworkBridgeUpdateV1( - NetworkBridgeEvent::PeerMessage(p, m) - ) - ) => { - assert_eq!(p, peer); - assert_eq!(m, approval_distribution_message); - } - ); - - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerDisconnected(peer), - &mut virtual_overseer, - ).await; - virtual_overseer - }); - } - - #[test] - fn peer_disconnect_from_just_one_peerset() { - test_harness(done_syncing_oracle(), |test_harness| async move { - let TestHarness { - mut network_handle, - mut virtual_overseer, - } = test_harness; - - let peer = PeerId::random(); - - network_handle.connect_peer(peer.clone(), PeerSet::Validation, ObservedRole::Full).await; - network_handle.connect_peer(peer.clone(), PeerSet::Collation, ObservedRole::Full).await; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::StatementDistribution( - StatementDistributionMessage::StatementFetchingReceiver(_) - ) - ); - - // bridge will inform about all connected peers. - { - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), - &mut virtual_overseer, - ).await; - - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), - &mut virtual_overseer, - ).await; - } - - { - assert_sends_collation_event_to_all( - NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), - &mut virtual_overseer, - ).await; - - assert_sends_collation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), - &mut virtual_overseer, - ).await; - } - - network_handle.disconnect_peer(peer.clone(), PeerSet::Validation).await; - - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerDisconnected(peer.clone()), - &mut virtual_overseer, - ).await; - - // to show that we're still connected on the collation protocol, send a view update. - - let hash_a = Hash::repeat_byte(1); - - virtual_overseer.send( - FromOverseer::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - }) - )) - ).await; - - let actions = network_handle.next_network_actions(3).await; - let wire_message = WireMessage::::ViewUpdate( - view![hash_a] - ).encode(); - - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer.clone(), - PeerSet::Collation, - wire_message.clone(), - ), - ); - virtual_overseer - }); - } - - #[test] - fn relays_collation_protocol_messages() { - test_harness(done_syncing_oracle(), |test_harness| async move { - let TestHarness { - mut network_handle, - mut virtual_overseer, - } = test_harness; - - let peer_a = PeerId::random(); - let peer_b = PeerId::random(); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::StatementDistribution( - StatementDistributionMessage::StatementFetchingReceiver(_) - ) - ); - - network_handle.connect_peer(peer_a.clone(), PeerSet::Validation, ObservedRole::Full).await; - network_handle.connect_peer(peer_b.clone(), PeerSet::Collation, ObservedRole::Full).await; - - // bridge will inform about all connected peers. - { - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerConnected(peer_a.clone(), ObservedRole::Full, None), - &mut virtual_overseer, - ).await; - - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer_a.clone(), View::default()), - &mut virtual_overseer, - ).await; - } - - { - assert_sends_collation_event_to_all( - NetworkBridgeEvent::PeerConnected(peer_b.clone(), ObservedRole::Full, None), - &mut virtual_overseer, - ).await; - - assert_sends_collation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer_b.clone(), View::default()), - &mut virtual_overseer, - ).await; - } - - // peer A gets reported for sending a collation message. - - let collator_protocol_message = protocol_v1::CollatorProtocolMessage::Declare( - Sr25519Keyring::Alice.public().into(), - Default::default(), - Default::default(), - ); - - let message = protocol_v1::CollationProtocol::CollatorProtocol( - collator_protocol_message.clone() - ); - - network_handle.peer_message( - peer_a.clone(), - PeerSet::Collation, - WireMessage::ProtocolMessage(message.clone()).encode(), - ).await; - - let actions = network_handle.next_network_actions(3).await; - assert_network_actions_contains( - &actions, - &NetworkAction::ReputationChange( - peer_a.clone(), - UNCONNECTED_PEERSET_COST, - ), - ); - - // peer B has the message relayed. - - network_handle.peer_message( - peer_b.clone(), - PeerSet::Collation, - WireMessage::ProtocolMessage(message.clone()).encode(), - ).await; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CollatorProtocol( - CollatorProtocolMessage::NetworkBridgeUpdateV1( - NetworkBridgeEvent::PeerMessage(p, m) - ) - ) => { - assert_eq!(p, peer_b); - assert_eq!(m, collator_protocol_message); - } - ); - virtual_overseer - }); - } - - #[test] - fn different_views_on_different_peer_sets() { - test_harness(done_syncing_oracle(), |test_harness| async move { - let TestHarness { - mut network_handle, - mut virtual_overseer, - } = test_harness; - - let peer = PeerId::random(); - - network_handle.connect_peer(peer.clone(), PeerSet::Validation, ObservedRole::Full).await; - network_handle.connect_peer(peer.clone(), PeerSet::Collation, ObservedRole::Full).await; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::StatementDistribution( - StatementDistributionMessage::StatementFetchingReceiver(_) - ) - ); - - // bridge will inform about all connected peers. - { - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), - &mut virtual_overseer, - ).await; - - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), - &mut virtual_overseer, - ).await; - } - - { - assert_sends_collation_event_to_all( - NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), - &mut virtual_overseer, - ).await; - - assert_sends_collation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), - &mut virtual_overseer, - ).await; - } - - let view_a = view![Hash::repeat_byte(1)]; - let view_b = view![Hash::repeat_byte(2)]; - - network_handle.peer_message( - peer.clone(), - PeerSet::Validation, - WireMessage::::ViewUpdate(view_a.clone()).encode(), - ).await; - - network_handle.peer_message( - peer.clone(), - PeerSet::Collation, - WireMessage::::ViewUpdate(view_b.clone()).encode(), - ).await; - - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer.clone(), view_a.clone()), - &mut virtual_overseer, - ).await; - - assert_sends_collation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer.clone(), view_b.clone()), - &mut virtual_overseer, - ).await; - virtual_overseer - }); - } - - #[test] - fn sent_views_include_finalized_number_update() { - test_harness(done_syncing_oracle(), |test_harness| async move { - let TestHarness { mut network_handle, mut virtual_overseer } = test_harness; - - let peer_a = PeerId::random(); - - network_handle.connect_peer( - peer_a.clone(), - PeerSet::Validation, - ObservedRole::Full, - ).await; - - let hash_a = Hash::repeat_byte(1); - let hash_b = Hash::repeat_byte(2); - - virtual_overseer.send( - FromOverseer::Signal(OverseerSignal::BlockFinalized(hash_a, 1)) - ).await; - virtual_overseer.send( - FromOverseer::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_b, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - }) - )) - ).await; - - let actions = network_handle.next_network_actions(2).await; - let wire_message = WireMessage::::ViewUpdate( - View::new(vec![hash_b], 1) - ).encode(); - - assert_network_actions_contains( - &actions, - &NetworkAction::WriteNotification( - peer_a.clone(), - PeerSet::Validation, - wire_message.clone(), - ), - ); - virtual_overseer - }); - } - - #[test] - fn view_finalized_number_can_not_go_down() { - test_harness(done_syncing_oracle(), |test_harness| async move { - let TestHarness { mut network_handle, virtual_overseer } = test_harness; - - let peer_a = PeerId::random(); - - network_handle.connect_peer( - peer_a.clone(), - PeerSet::Validation, - ObservedRole::Full, - ).await; - - network_handle.peer_message( - peer_a.clone(), - PeerSet::Validation, - WireMessage::::ViewUpdate( - View::new(vec![Hash::repeat_byte(0x01)], 1), - ).encode(), - ).await; - - network_handle.peer_message( - peer_a.clone(), - PeerSet::Validation, - WireMessage::::ViewUpdate( - View::new(vec![], 0), - ).encode(), - ).await; - - let actions = network_handle.next_network_actions(2).await; - assert_network_actions_contains( - &actions, - &NetworkAction::ReputationChange( - peer_a.clone(), - MALFORMED_VIEW_COST, - ), - ); - virtual_overseer - }); - } - - #[test] - fn send_messages_to_peers() { - test_harness(done_syncing_oracle(), |test_harness| async move { - let TestHarness { - mut network_handle, - mut virtual_overseer, - } = test_harness; - - let peer = PeerId::random(); - - network_handle.connect_peer(peer.clone(), PeerSet::Validation, ObservedRole::Full).await; - network_handle.connect_peer(peer.clone(), PeerSet::Collation, ObservedRole::Full).await; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::StatementDistribution( - StatementDistributionMessage::StatementFetchingReceiver(_) - ) - ); - - // bridge will inform about all connected peers. - { - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), - &mut virtual_overseer, - ).await; - - assert_sends_validation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), - &mut virtual_overseer, - ).await; - } - - { - assert_sends_collation_event_to_all( - NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), - &mut virtual_overseer, - ).await; - - assert_sends_collation_event_to_all( - NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), - &mut virtual_overseer, - ).await; - } - - // consume peer view changes - { - let _peer_view_changes = network_handle.next_network_actions(2).await; - } - - // send a validation protocol message. - - { - let approval_distribution_message = protocol_v1::ApprovalDistributionMessage::Approvals( - Vec::new() - ); - - let message = protocol_v1::ValidationProtocol::ApprovalDistribution( - approval_distribution_message.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication { - msg: NetworkBridgeMessage::SendValidationMessage( - vec![peer.clone()], - message.clone(), - ) - }).await; - - assert_eq!( - network_handle.next_network_action().await, - NetworkAction::WriteNotification( - peer.clone(), - PeerSet::Validation, - WireMessage::ProtocolMessage(message).encode(), - ) - ); - } - - // send a collation protocol message. - - { - let collator_protocol_message = protocol_v1::CollatorProtocolMessage::Declare( - Sr25519Keyring::Alice.public().into(), - Default::default(), - Default::default(), - ); - - let message = protocol_v1::CollationProtocol::CollatorProtocol( - collator_protocol_message.clone() - ); - - virtual_overseer.send(FromOverseer::Communication { - msg: NetworkBridgeMessage::SendCollationMessage( - vec![peer.clone()], - message.clone(), - ) - }).await; - - assert_eq!( - network_handle.next_network_action().await, - NetworkAction::WriteNotification( - peer.clone(), - PeerSet::Collation, - WireMessage::ProtocolMessage(message).encode(), - ) - ); - } - virtual_overseer - }); - } - - #[test] - fn spread_event_to_subsystems_is_up_to_date() { - // Number of subsystems expected to be interested in a network event, - // and hence the network event broadcasted to. - const EXPECTED_COUNT: usize = 3; - - let mut cnt = 0_usize; - for msg in AllMessages::dispatch_iter(NetworkBridgeEvent::PeerDisconnected(PeerId::random())) { - match msg { - AllMessages::CandidateValidation(_) => unreachable!("Not interested in network events"), - AllMessages::CandidateBacking(_) => unreachable!("Not interested in network events"), - AllMessages::CandidateSelection(_) => unreachable!("Not interested in network events"), - AllMessages::ChainApi(_) => unreachable!("Not interested in network events"), - AllMessages::CollatorProtocol(_) => unreachable!("Not interested in network events"), - AllMessages::StatementDistribution(_) => { cnt += 1; } - AllMessages::AvailabilityDistribution(_) => unreachable!("Not interested in network events"), - AllMessages::AvailabilityRecovery(_) => unreachable!("Not interested in network events"), - AllMessages::BitfieldDistribution(_) => { cnt += 1; } - AllMessages::BitfieldSigning(_) => unreachable!("Not interested in network events"), - AllMessages::Provisioner(_) => unreachable!("Not interested in network events"), - AllMessages::RuntimeApi(_) => unreachable!("Not interested in network events"), - AllMessages::AvailabilityStore(_) => unreachable!("Not interested in network events"), - AllMessages::NetworkBridge(_) => unreachable!("Not interested in network events"), - AllMessages::CollationGeneration(_) => unreachable!("Not interested in network events"), - AllMessages::ApprovalVoting(_) => unreachable!("Not interested in network events"), - AllMessages::ApprovalDistribution(_) => { cnt += 1; } - AllMessages::GossipSupport(_) => unreachable!("Not interested in network events"), - // Add variants here as needed, `{ cnt += 1; }` for those that need to be - // notified, `unreachable!()` for those that should not. - } - } - assert_eq!(cnt, EXPECTED_COUNT); - } - - #[test] - fn our_view_updates_decreasing_order_and_limited_to_max() { - test_harness(done_syncing_oracle(), |test_harness| async move { - let TestHarness { - mut virtual_overseer, - .. - } = test_harness; - - - // to show that we're still connected on the collation protocol, send a view update. - - let hashes = (0..MAX_VIEW_HEADS * 3).map(|i| Hash::repeat_byte(i as u8)); - - virtual_overseer.send( - FromOverseer::Signal(OverseerSignal::ActiveLeaves( - // These are in reverse order, so the subsystem must sort internally to - // get the correct view. - ActiveLeavesUpdate { - activated: hashes.enumerate().map(|(i, h)| ActivatedLeaf { - hash: h, - number: i as _, - span: Arc::new(jaeger::Span::Disabled), - }).rev().collect(), - deactivated: Default::default(), - } - )) - ).await; - - let view_heads = (MAX_VIEW_HEADS * 2 .. MAX_VIEW_HEADS * 3).rev() - .map(|i| (Hash::repeat_byte(i as u8), Arc::new(jaeger::Span::Disabled)) ); - - let our_view = OurView::new( - view_heads, - 0, - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::StatementDistribution( - StatementDistributionMessage::StatementFetchingReceiver(_) - ) - ); - - assert_sends_validation_event_to_all( - NetworkBridgeEvent::OurViewChange(our_view.clone()), - &mut virtual_overseer, - ).await; - - assert_sends_collation_event_to_all( - NetworkBridgeEvent::OurViewChange(our_view), - &mut virtual_overseer, - ).await; - virtual_overseer - }); - } -} +mod tests; diff --git a/node/network/bridge/src/tests.rs b/node/network/bridge/src/tests.rs new file mode 100644 index 000000000000..048d431355f4 --- /dev/null +++ b/node/network/bridge/src/tests.rs @@ -0,0 +1,1292 @@ +use super::*; +use futures::executor; +use futures::stream::BoxStream; +use futures::channel::oneshot; + +use std::borrow::Cow; +use std::collections::HashSet; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, Ordering}; +use async_trait::async_trait; +use parking_lot::Mutex; +use assert_matches::assert_matches; + +use sc_network::{Event as NetworkEvent, IfDisconnected}; + +use polkadot_subsystem::{jaeger, ActiveLeavesUpdate, FromOverseer, OverseerSignal}; +use polkadot_subsystem::messages::{ + ApprovalDistributionMessage, + BitfieldDistributionMessage, + StatementDistributionMessage +}; +use polkadot_node_subsystem_test_helpers::{ + SingleItemSink, SingleItemStream, TestSubsystemContextHandle, +}; +use polkadot_node_subsystem_util::metered; +use polkadot_node_network_protocol::view; +use sc_network::Multiaddr; +use sc_network::config::RequestResponseConfig; +use sp_keyring::Sr25519Keyring; +use polkadot_primitives::v1::AuthorityDiscoveryId; +use polkadot_node_network_protocol::{ObservedRole, request_response::request::Requests}; + +use crate::network::{Network, NetworkAction}; +use crate::validator_discovery::AuthorityDiscovery; + +// The subsystem's view of the network - only supports a single call to `event_stream`. +#[derive(Clone)] +struct TestNetwork { + net_events: Arc>>>, + action_tx: metered::UnboundedMeteredSender, + _req_configs: Vec, +} + +#[derive(Clone)] +struct TestAuthorityDiscovery; + +// The test's view of the network. This receives updates from the subsystem in the form +// of `NetworkAction`s. +struct TestNetworkHandle { + action_rx: metered::UnboundedMeteredReceiver, + net_tx: SingleItemSink, +} + +fn new_test_network(req_configs: Vec) -> ( + TestNetwork, + TestNetworkHandle, + TestAuthorityDiscovery, +) { + let (net_tx, net_rx) = polkadot_node_subsystem_test_helpers::single_item_sink(); + let (action_tx, action_rx) = metered::unbounded(); + + ( + TestNetwork { + net_events: Arc::new(Mutex::new(Some(net_rx))), + action_tx, + _req_configs: req_configs, + }, + TestNetworkHandle { + action_rx, + net_tx, + }, + TestAuthorityDiscovery, + ) +} + +#[async_trait] +impl Network for TestNetwork { + fn event_stream(&mut self) -> BoxStream<'static, NetworkEvent> { + self.net_events.lock() + .take() + .expect("Subsystem made more than one call to `event_stream`") + .boxed() + } + + async fn add_to_peers_set(&mut self, _protocol: Cow<'static, str>, _: HashSet) -> Result<(), String> { + Ok(()) + } + + async fn remove_from_peers_set(&mut self, _protocol: Cow<'static, str>, _: HashSet) -> Result<(), String> { + Ok(()) + } + + fn action_sink<'a>(&'a mut self) + -> Pin + Send + 'a>> + { + Box::pin((&mut self.action_tx).sink_map_err(Into::into)) + } + + async fn start_request(&self, _: &mut AD, _: Requests, _: IfDisconnected) { + } +} + +#[async_trait] +impl validator_discovery::AuthorityDiscovery for TestAuthorityDiscovery { + async fn get_addresses_by_authority_id(&mut self, _authority: AuthorityDiscoveryId) -> Option> { + None + } + + async fn get_authority_id_by_peer_id(&mut self, _peer_id: PeerId) -> Option { + None + } +} + +impl TestNetworkHandle { + // Get the next network action. + async fn next_network_action(&mut self) -> NetworkAction { + self.action_rx.next().await.expect("subsystem concluded early") + } + + // Wait for the next N network actions. + async fn next_network_actions(&mut self, n: usize) -> Vec { + let mut v = Vec::with_capacity(n); + for _ in 0..n { + v.push(self.next_network_action().await); + } + + v + } + + async fn connect_peer(&mut self, peer: PeerId, peer_set: PeerSet, role: ObservedRole) { + self.send_network_event(NetworkEvent::NotificationStreamOpened { + remote: peer, + protocol: peer_set.into_protocol_name(), + negotiated_fallback: None, + role: role.into(), + }).await; + } + + async fn disconnect_peer(&mut self, peer: PeerId, peer_set: PeerSet) { + self.send_network_event(NetworkEvent::NotificationStreamClosed { + remote: peer, + protocol: peer_set.into_protocol_name(), + }).await; + } + + async fn peer_message(&mut self, peer: PeerId, peer_set: PeerSet, message: Vec) { + self.send_network_event(NetworkEvent::NotificationsReceived { + remote: peer, + messages: vec![(peer_set.into_protocol_name(), message.into())], + }).await; + } + + async fn send_network_event(&mut self, event: NetworkEvent) { + self.net_tx.send(event).await.expect("subsystem concluded early"); + } +} + +/// Assert that the given actions contain the given `action`. +fn assert_network_actions_contains(actions: &[NetworkAction], action: &NetworkAction) { + if !actions.iter().any(|x| x == action) { + panic!("Could not find `{:?}` in `{:?}`", action, actions); + } +} + +#[derive(Clone)] +struct TestSyncOracle { + flag: Arc, + done_syncing_sender: Arc>>>, +} + +struct TestSyncOracleHandle { + done_syncing_receiver: oneshot::Receiver<()>, + flag: Arc, +} + +impl TestSyncOracleHandle { + fn set_done(&self) { + self.flag.store(false, Ordering::SeqCst); + } + + async fn await_mode_switch(self) { + let _ = self.done_syncing_receiver.await; + } +} + +impl SyncOracle for TestSyncOracle { + fn is_major_syncing(&mut self) -> bool { + let is_major_syncing = self.flag.load(Ordering::SeqCst); + + if !is_major_syncing { + if let Some(sender) = self.done_syncing_sender.lock().take() { + let _ = sender.send(()); + } + } + + is_major_syncing + } + + fn is_offline(&mut self) -> bool { + unimplemented!("not used in network bridge") + } +} + +// val - result of `is_major_syncing`. +fn make_sync_oracle(val: bool) -> (TestSyncOracle, TestSyncOracleHandle) { + let (tx, rx) = oneshot::channel(); + let flag = Arc::new(AtomicBool::new(val)); + + ( + TestSyncOracle { + flag: flag.clone(), + done_syncing_sender: Arc::new(Mutex::new(Some(tx))), + }, + TestSyncOracleHandle { + flag, + done_syncing_receiver: rx, + } + ) +} + +fn done_syncing_oracle() -> Box { + let (oracle, _) = make_sync_oracle(false); + Box::new(oracle) +} + +type VirtualOverseer = TestSubsystemContextHandle; + +struct TestHarness { + network_handle: TestNetworkHandle, + virtual_overseer: VirtualOverseer, +} + +fn test_harness>( + sync_oracle: Box, + test: impl FnOnce(TestHarness) -> T, +) { + let pool = sp_core::testing::TaskExecutor::new(); + let (request_multiplexer, req_configs) = RequestMultiplexer::new(); + let (mut network, network_handle, discovery) = new_test_network(req_configs); + let (context, virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context(pool); + let network_stream = network.event_stream(); + + let bridge = NetworkBridge { + network_service: network, + authority_discovery_service: discovery, + request_multiplexer, + metrics: Metrics(None), + sync_oracle, + }; + + let network_bridge = run_network( + bridge, + context, + network_stream, + ) + .map_err(|_| panic!("subsystem execution failed")) + .map(|_| ()); + + let test_fut = test(TestHarness { + network_handle, + virtual_overseer, + }); + + futures::pin_mut!(test_fut); + futures::pin_mut!(network_bridge); + + let _ = executor::block_on(future::join(async move { + let mut virtual_overseer = test_fut.await; + virtual_overseer.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }, network_bridge)); +} + +async fn assert_sends_validation_event_to_all( + event: NetworkBridgeEvent, + virtual_overseer: &mut TestSubsystemContextHandle, +) { + // Ordering must match the enum variant order + // in `AllMessages`. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::NetworkBridgeUpdateV1(e) + ) if e == event.focus().expect("could not focus message") + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::BitfieldDistribution( + BitfieldDistributionMessage::NetworkBridgeUpdateV1(e) + ) if e == event.focus().expect("could not focus message") + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ApprovalDistribution( + ApprovalDistributionMessage::NetworkBridgeUpdateV1(e) + ) if e == event.focus().expect("could not focus message") + ); +} + +async fn assert_sends_collation_event_to_all( + event: NetworkBridgeEvent, + virtual_overseer: &mut TestSubsystemContextHandle, +) { + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CollatorProtocol( + CollatorProtocolMessage::NetworkBridgeUpdateV1(e) + ) if e == event.focus().expect("could not focus message") + ) +} + +#[test] +fn send_our_view_upon_connection() { + let (oracle, handle) = make_sync_oracle(false); + test_harness(Box::new(oracle), |test_harness| async move { + let TestHarness { + mut network_handle, + mut virtual_overseer, + } = test_harness; + + let peer = PeerId::random(); + + let head = Hash::repeat_byte(1); + virtual_overseer.send( + FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: head, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + }) + )) + ).await; + + handle.await_mode_switch().await; + + network_handle.connect_peer(peer.clone(), PeerSet::Validation, ObservedRole::Full).await; + network_handle.connect_peer(peer.clone(), PeerSet::Collation, ObservedRole::Full).await; + + let view = view![head]; + let actions = network_handle.next_network_actions(2).await; + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer.clone(), + PeerSet::Validation, + WireMessage::::ViewUpdate( + view.clone(), + ).encode(), + ), + ); + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer.clone(), + PeerSet::Collation, + WireMessage::::ViewUpdate( + view.clone(), + ).encode(), + ), + ); + virtual_overseer + }); +} + +#[test] +fn sends_view_updates_to_peers() { + let (oracle, handle) = make_sync_oracle(false); + test_harness(Box::new(oracle), |test_harness| async move { + let TestHarness { mut network_handle, mut virtual_overseer } = test_harness; + + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + + virtual_overseer.send( + FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate { + activated: Default::default(), + deactivated: Default::default(), + } + )) + ).await; + + handle.await_mode_switch().await; + + network_handle.connect_peer( + peer_a.clone(), + PeerSet::Validation, + ObservedRole::Full, + ).await; + network_handle.connect_peer( + peer_b.clone(), + PeerSet::Collation, + ObservedRole::Full, + ).await; + + let actions = network_handle.next_network_actions(2).await; + let wire_message = WireMessage::::ViewUpdate( + View::default(), + ).encode(); + + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer_a, + PeerSet::Validation, + wire_message.clone(), + ), + ); + + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer_b, + PeerSet::Collation, + wire_message.clone(), + ), + ); + + let hash_a = Hash::repeat_byte(1); + + virtual_overseer.send( + FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: hash_a, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + }) + )) + ).await; + + let actions = network_handle.next_network_actions(2).await; + let wire_message = WireMessage::::ViewUpdate( + view![hash_a] + ).encode(); + + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer_a, + PeerSet::Validation, + wire_message.clone(), + ), + ); + + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer_b, + PeerSet::Collation, + wire_message.clone(), + ), + ); + virtual_overseer + }); +} + +#[test] +fn do_not_send_view_update_until_synced() { + let (oracle, handle) = make_sync_oracle(true); + test_harness(Box::new(oracle), |test_harness| async move { + let TestHarness { mut network_handle, mut virtual_overseer } = test_harness; + + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + + network_handle.connect_peer( + peer_a.clone(), + PeerSet::Validation, + ObservedRole::Full, + ).await; + network_handle.connect_peer( + peer_b.clone(), + PeerSet::Collation, + ObservedRole::Full, + ).await; + + { + let actions = network_handle.next_network_actions(2).await; + let wire_message = WireMessage::::ViewUpdate( + View::default(), + ).encode(); + + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer_a, + PeerSet::Validation, + wire_message.clone(), + ), + ); + + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer_b, + PeerSet::Collation, + wire_message.clone(), + ), + ); + } + + let hash_a = Hash::repeat_byte(1); + let hash_b = Hash::repeat_byte(1); + + virtual_overseer.send( + FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: hash_a, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + }) + )) + ).await; + + // delay until the previous update has certainly been processed. + futures_timer::Delay::new(std::time::Duration::from_millis(100)).await; + + handle.set_done(); + + virtual_overseer.send( + FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: hash_b, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + }) + )) + ).await; + + handle.await_mode_switch().await; + + // There should be a mode switch only for the second view update. + { + let actions = network_handle.next_network_actions(2).await; + let wire_message = WireMessage::::ViewUpdate( + view![hash_a, hash_b] + ).encode(); + + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer_a, + PeerSet::Validation, + wire_message.clone(), + ), + ); + + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer_b, + PeerSet::Collation, + wire_message.clone(), + ), + ); + } + virtual_overseer + }); +} + +#[test] +fn do_not_send_view_update_when_only_finalized_block_changed() { + test_harness(done_syncing_oracle(), |test_harness| async move { + let TestHarness { mut network_handle, mut virtual_overseer } = test_harness; + + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + + network_handle.connect_peer( + peer_a.clone(), + PeerSet::Validation, + ObservedRole::Full, + ).await; + network_handle.connect_peer( + peer_b.clone(), + PeerSet::Validation, + ObservedRole::Full, + ).await; + + let hash_a = Hash::repeat_byte(1); + + virtual_overseer.send(FromOverseer::Signal(OverseerSignal::BlockFinalized(Hash::random(), 5))).await; + + // Send some empty active leaves update + // + // This should not trigger a view update to our peers. + virtual_overseer.send( + FromOverseer::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::default())) + ).await; + + // This should trigger the view update to our peers. + virtual_overseer.send( + FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: hash_a, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + }) + )) + ).await; + + let actions = network_handle.next_network_actions(4).await; + let wire_message = WireMessage::::ViewUpdate( + View::new(vec![hash_a], 5) + ).encode(); + + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer_a, + PeerSet::Validation, + wire_message.clone(), + ), + ); + + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer_b, + PeerSet::Validation, + wire_message.clone(), + ), + ); + virtual_overseer + }); +} + +#[test] +fn peer_view_updates_sent_via_overseer() { + test_harness(done_syncing_oracle(), |test_harness| async move { + let TestHarness { + mut network_handle, + mut virtual_overseer, + } = test_harness; + + let peer = PeerId::random(); + + network_handle.connect_peer(peer.clone(), PeerSet::Validation, ObservedRole::Full).await; + + let view = view![Hash::repeat_byte(1)]; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::StatementFetchingReceiver(_) + ) + ); + + // bridge will inform about all connected peers. + { + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), + &mut virtual_overseer, + ).await; + + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), + &mut virtual_overseer, + ).await; + } + + network_handle.peer_message( + peer.clone(), + PeerSet::Validation, + WireMessage::::ViewUpdate( + view.clone(), + ).encode(), + ).await; + + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer.clone(), view), + &mut virtual_overseer, + ).await; + virtual_overseer + }); +} + +#[test] +fn peer_messages_sent_via_overseer() { + test_harness(done_syncing_oracle(), |test_harness| async move { + let TestHarness { + mut network_handle, + mut virtual_overseer, + } = test_harness; + + let peer = PeerId::random(); + + network_handle.connect_peer( + peer.clone(), + PeerSet::Validation, + ObservedRole::Full, + ).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::StatementFetchingReceiver(_) + ) + ); + + // bridge will inform about all connected peers. + { + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), + &mut virtual_overseer, + ).await; + + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), + &mut virtual_overseer, + ).await; + } + + let approval_distribution_message = protocol_v1::ApprovalDistributionMessage::Approvals( + Vec::new() + ); + + let message = protocol_v1::ValidationProtocol::ApprovalDistribution( + approval_distribution_message.clone(), + ); + + network_handle.peer_message( + peer.clone(), + PeerSet::Validation, + WireMessage::ProtocolMessage(message.clone()).encode(), + ).await; + + network_handle.disconnect_peer(peer.clone(), PeerSet::Validation).await; + + // Approval distribution message comes first, and the message is only sent to that subsystem. + // then a disconnection event arises that is sent to all validation networking subsystems. + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ApprovalDistribution( + ApprovalDistributionMessage::NetworkBridgeUpdateV1( + NetworkBridgeEvent::PeerMessage(p, m) + ) + ) => { + assert_eq!(p, peer); + assert_eq!(m, approval_distribution_message); + } + ); + + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerDisconnected(peer), + &mut virtual_overseer, + ).await; + virtual_overseer + }); +} + +#[test] +fn peer_disconnect_from_just_one_peerset() { + test_harness(done_syncing_oracle(), |test_harness| async move { + let TestHarness { + mut network_handle, + mut virtual_overseer, + } = test_harness; + + let peer = PeerId::random(); + + network_handle.connect_peer(peer.clone(), PeerSet::Validation, ObservedRole::Full).await; + network_handle.connect_peer(peer.clone(), PeerSet::Collation, ObservedRole::Full).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::StatementFetchingReceiver(_) + ) + ); + + // bridge will inform about all connected peers. + { + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), + &mut virtual_overseer, + ).await; + + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), + &mut virtual_overseer, + ).await; + } + + { + assert_sends_collation_event_to_all( + NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), + &mut virtual_overseer, + ).await; + + assert_sends_collation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), + &mut virtual_overseer, + ).await; + } + + network_handle.disconnect_peer(peer.clone(), PeerSet::Validation).await; + + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerDisconnected(peer.clone()), + &mut virtual_overseer, + ).await; + + // to show that we're still connected on the collation protocol, send a view update. + + let hash_a = Hash::repeat_byte(1); + + virtual_overseer.send( + FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: hash_a, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + }) + )) + ).await; + + let actions = network_handle.next_network_actions(3).await; + let wire_message = WireMessage::::ViewUpdate( + view![hash_a] + ).encode(); + + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer.clone(), + PeerSet::Collation, + wire_message.clone(), + ), + ); + virtual_overseer + }); +} + +#[test] +fn relays_collation_protocol_messages() { + test_harness(done_syncing_oracle(), |test_harness| async move { + let TestHarness { + mut network_handle, + mut virtual_overseer, + } = test_harness; + + let peer_a = PeerId::random(); + let peer_b = PeerId::random(); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::StatementFetchingReceiver(_) + ) + ); + + network_handle.connect_peer(peer_a.clone(), PeerSet::Validation, ObservedRole::Full).await; + network_handle.connect_peer(peer_b.clone(), PeerSet::Collation, ObservedRole::Full).await; + + // bridge will inform about all connected peers. + { + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerConnected(peer_a.clone(), ObservedRole::Full, None), + &mut virtual_overseer, + ).await; + + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer_a.clone(), View::default()), + &mut virtual_overseer, + ).await; + } + + { + assert_sends_collation_event_to_all( + NetworkBridgeEvent::PeerConnected(peer_b.clone(), ObservedRole::Full, None), + &mut virtual_overseer, + ).await; + + assert_sends_collation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer_b.clone(), View::default()), + &mut virtual_overseer, + ).await; + } + + // peer A gets reported for sending a collation message. + + let collator_protocol_message = protocol_v1::CollatorProtocolMessage::Declare( + Sr25519Keyring::Alice.public().into(), + Default::default(), + Default::default(), + ); + + let message = protocol_v1::CollationProtocol::CollatorProtocol( + collator_protocol_message.clone() + ); + + network_handle.peer_message( + peer_a.clone(), + PeerSet::Collation, + WireMessage::ProtocolMessage(message.clone()).encode(), + ).await; + + let actions = network_handle.next_network_actions(3).await; + assert_network_actions_contains( + &actions, + &NetworkAction::ReputationChange( + peer_a.clone(), + UNCONNECTED_PEERSET_COST, + ), + ); + + // peer B has the message relayed. + + network_handle.peer_message( + peer_b.clone(), + PeerSet::Collation, + WireMessage::ProtocolMessage(message.clone()).encode(), + ).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CollatorProtocol( + CollatorProtocolMessage::NetworkBridgeUpdateV1( + NetworkBridgeEvent::PeerMessage(p, m) + ) + ) => { + assert_eq!(p, peer_b); + assert_eq!(m, collator_protocol_message); + } + ); + virtual_overseer + }); +} + +#[test] +fn different_views_on_different_peer_sets() { + test_harness(done_syncing_oracle(), |test_harness| async move { + let TestHarness { + mut network_handle, + mut virtual_overseer, + } = test_harness; + + let peer = PeerId::random(); + + network_handle.connect_peer(peer.clone(), PeerSet::Validation, ObservedRole::Full).await; + network_handle.connect_peer(peer.clone(), PeerSet::Collation, ObservedRole::Full).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::StatementFetchingReceiver(_) + ) + ); + + // bridge will inform about all connected peers. + { + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), + &mut virtual_overseer, + ).await; + + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), + &mut virtual_overseer, + ).await; + } + + { + assert_sends_collation_event_to_all( + NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), + &mut virtual_overseer, + ).await; + + assert_sends_collation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), + &mut virtual_overseer, + ).await; + } + + let view_a = view![Hash::repeat_byte(1)]; + let view_b = view![Hash::repeat_byte(2)]; + + network_handle.peer_message( + peer.clone(), + PeerSet::Validation, + WireMessage::::ViewUpdate(view_a.clone()).encode(), + ).await; + + network_handle.peer_message( + peer.clone(), + PeerSet::Collation, + WireMessage::::ViewUpdate(view_b.clone()).encode(), + ).await; + + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer.clone(), view_a.clone()), + &mut virtual_overseer, + ).await; + + assert_sends_collation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer.clone(), view_b.clone()), + &mut virtual_overseer, + ).await; + virtual_overseer + }); +} + +#[test] +fn sent_views_include_finalized_number_update() { + test_harness(done_syncing_oracle(), |test_harness| async move { + let TestHarness { mut network_handle, mut virtual_overseer } = test_harness; + + let peer_a = PeerId::random(); + + network_handle.connect_peer( + peer_a.clone(), + PeerSet::Validation, + ObservedRole::Full, + ).await; + + let hash_a = Hash::repeat_byte(1); + let hash_b = Hash::repeat_byte(2); + + virtual_overseer.send( + FromOverseer::Signal(OverseerSignal::BlockFinalized(hash_a, 1)) + ).await; + virtual_overseer.send( + FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: hash_b, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + }) + )) + ).await; + + let actions = network_handle.next_network_actions(2).await; + let wire_message = WireMessage::::ViewUpdate( + View::new(vec![hash_b], 1) + ).encode(); + + assert_network_actions_contains( + &actions, + &NetworkAction::WriteNotification( + peer_a.clone(), + PeerSet::Validation, + wire_message.clone(), + ), + ); + virtual_overseer + }); +} + +#[test] +fn view_finalized_number_can_not_go_down() { + test_harness(done_syncing_oracle(), |test_harness| async move { + let TestHarness { mut network_handle, virtual_overseer } = test_harness; + + let peer_a = PeerId::random(); + + network_handle.connect_peer( + peer_a.clone(), + PeerSet::Validation, + ObservedRole::Full, + ).await; + + network_handle.peer_message( + peer_a.clone(), + PeerSet::Validation, + WireMessage::::ViewUpdate( + View::new(vec![Hash::repeat_byte(0x01)], 1), + ).encode(), + ).await; + + network_handle.peer_message( + peer_a.clone(), + PeerSet::Validation, + WireMessage::::ViewUpdate( + View::new(vec![], 0), + ).encode(), + ).await; + + let actions = network_handle.next_network_actions(2).await; + assert_network_actions_contains( + &actions, + &NetworkAction::ReputationChange( + peer_a.clone(), + MALFORMED_VIEW_COST, + ), + ); + virtual_overseer + }); +} + +#[test] +fn send_messages_to_peers() { + test_harness(done_syncing_oracle(), |test_harness| async move { + let TestHarness { + mut network_handle, + mut virtual_overseer, + } = test_harness; + + let peer = PeerId::random(); + + network_handle.connect_peer(peer.clone(), PeerSet::Validation, ObservedRole::Full).await; + network_handle.connect_peer(peer.clone(), PeerSet::Collation, ObservedRole::Full).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::StatementFetchingReceiver(_) + ) + ); + + // bridge will inform about all connected peers. + { + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), + &mut virtual_overseer, + ).await; + + assert_sends_validation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), + &mut virtual_overseer, + ).await; + } + + { + assert_sends_collation_event_to_all( + NetworkBridgeEvent::PeerConnected(peer.clone(), ObservedRole::Full, None), + &mut virtual_overseer, + ).await; + + assert_sends_collation_event_to_all( + NetworkBridgeEvent::PeerViewChange(peer.clone(), View::default()), + &mut virtual_overseer, + ).await; + } + + // consume peer view changes + { + let _peer_view_changes = network_handle.next_network_actions(2).await; + } + + // send a validation protocol message. + + { + let approval_distribution_message = protocol_v1::ApprovalDistributionMessage::Approvals( + Vec::new() + ); + + let message = protocol_v1::ValidationProtocol::ApprovalDistribution( + approval_distribution_message.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication { + msg: NetworkBridgeMessage::SendValidationMessage( + vec![peer.clone()], + message.clone(), + ) + }).await; + + assert_eq!( + network_handle.next_network_action().await, + NetworkAction::WriteNotification( + peer.clone(), + PeerSet::Validation, + WireMessage::ProtocolMessage(message).encode(), + ) + ); + } + + // send a collation protocol message. + + { + let collator_protocol_message = protocol_v1::CollatorProtocolMessage::Declare( + Sr25519Keyring::Alice.public().into(), + Default::default(), + Default::default(), + ); + + let message = protocol_v1::CollationProtocol::CollatorProtocol( + collator_protocol_message.clone() + ); + + virtual_overseer.send(FromOverseer::Communication { + msg: NetworkBridgeMessage::SendCollationMessage( + vec![peer.clone()], + message.clone(), + ) + }).await; + + assert_eq!( + network_handle.next_network_action().await, + NetworkAction::WriteNotification( + peer.clone(), + PeerSet::Collation, + WireMessage::ProtocolMessage(message).encode(), + ) + ); + } + virtual_overseer + }); +} + +#[test] +fn spread_event_to_subsystems_is_up_to_date() { + // Number of subsystems expected to be interested in a network event, + // and hence the network event broadcasted to. + const EXPECTED_COUNT: usize = 3; + + let mut cnt = 0_usize; + for msg in AllMessages::dispatch_iter(NetworkBridgeEvent::PeerDisconnected(PeerId::random())) { + match msg { + AllMessages::CandidateValidation(_) => unreachable!("Not interested in network events"), + AllMessages::CandidateBacking(_) => unreachable!("Not interested in network events"), + AllMessages::CandidateSelection(_) => unreachable!("Not interested in network events"), + AllMessages::ChainApi(_) => unreachable!("Not interested in network events"), + AllMessages::CollatorProtocol(_) => unreachable!("Not interested in network events"), + AllMessages::StatementDistribution(_) => { cnt += 1; } + AllMessages::AvailabilityDistribution(_) => unreachable!("Not interested in network events"), + AllMessages::AvailabilityRecovery(_) => unreachable!("Not interested in network events"), + AllMessages::BitfieldDistribution(_) => { cnt += 1; } + AllMessages::BitfieldSigning(_) => unreachable!("Not interested in network events"), + AllMessages::Provisioner(_) => unreachable!("Not interested in network events"), + AllMessages::RuntimeApi(_) => unreachable!("Not interested in network events"), + AllMessages::AvailabilityStore(_) => unreachable!("Not interested in network events"), + AllMessages::NetworkBridge(_) => unreachable!("Not interested in network events"), + AllMessages::CollationGeneration(_) => unreachable!("Not interested in network events"), + AllMessages::ApprovalVoting(_) => unreachable!("Not interested in network events"), + AllMessages::ApprovalDistribution(_) => { cnt += 1; } + AllMessages::GossipSupport(_) => unreachable!("Not interested in network events"), + // Add variants here as needed, `{ cnt += 1; }` for those that need to be + // notified, `unreachable!()` for those that should not. + } + } + assert_eq!(cnt, EXPECTED_COUNT); +} + +#[test] +fn our_view_updates_decreasing_order_and_limited_to_max() { + test_harness(done_syncing_oracle(), |test_harness| async move { + let TestHarness { + mut virtual_overseer, + .. + } = test_harness; + + + // to show that we're still connected on the collation protocol, send a view update. + + let hashes = (0..MAX_VIEW_HEADS * 3).map(|i| Hash::repeat_byte(i as u8)); + + virtual_overseer.send( + FromOverseer::Signal(OverseerSignal::ActiveLeaves( + // These are in reverse order, so the subsystem must sort internally to + // get the correct view. + ActiveLeavesUpdate { + activated: hashes.enumerate().map(|(i, h)| ActivatedLeaf { + hash: h, + number: i as _, + span: Arc::new(jaeger::Span::Disabled), + }).rev().collect(), + deactivated: Default::default(), + } + )) + ).await; + + let view_heads = (MAX_VIEW_HEADS * 2 .. MAX_VIEW_HEADS * 3).rev() + .map(|i| (Hash::repeat_byte(i as u8), Arc::new(jaeger::Span::Disabled)) ); + + let our_view = OurView::new( + view_heads, + 0, + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::StatementFetchingReceiver(_) + ) + ); + + assert_sends_validation_event_to_all( + NetworkBridgeEvent::OurViewChange(our_view.clone()), + &mut virtual_overseer, + ).await; + + assert_sends_collation_event_to_all( + NetworkBridgeEvent::OurViewChange(our_view), + &mut virtual_overseer, + ).await; + virtual_overseer + }); +} diff --git a/node/network/collator-protocol/src/collator_side.rs b/node/network/collator-protocol/src/collator_side.rs index b51ca89dcb5e..0eb2eae9925a 100644 --- a/node/network/collator-protocol/src/collator_side.rs +++ b/node/network/collator-protocol/src/collator_side.rs @@ -437,7 +437,7 @@ async fn determine_our_validators( /// Issue a `Declare` collation message to the given `peer`. #[tracing::instrument(level = "trace", skip(ctx, state), fields(subsystem = LOG_TARGET))] async fn declare( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, peer: PeerId, ) { @@ -543,7 +543,7 @@ async fn advertise_collation( /// The main incoming message dispatching switch. #[tracing::instrument(level = "trace", skip(ctx, runtime, state), fields(subsystem = LOG_TARGET))] async fn process_msg( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, runtime: &mut RuntimeInfo, state: &mut State, msg: CollatorProtocolMessage, @@ -756,7 +756,7 @@ async fn handle_incoming_peer_message( /// Our view has changed. #[tracing::instrument(level = "trace", skip(ctx, state), fields(subsystem = LOG_TARGET))] async fn handle_peer_view_change( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, peer_id: PeerId, view: View, @@ -775,7 +775,7 @@ async fn handle_peer_view_change( /// Bridge messages switch. #[tracing::instrument(level = "trace", skip(ctx, runtime, state), fields(subsystem = LOG_TARGET))] async fn handle_network_msg( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, runtime: &mut RuntimeInfo, state: &mut State, bridge_message: NetworkBridgeEvent, @@ -883,7 +883,7 @@ async fn handle_our_view_change( /// The collator protocol collator side main loop. #[tracing::instrument(skip(ctx, collator_pair, metrics), fields(subsystem = LOG_TARGET))] pub(crate) async fn run( - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, local_peer_id: PeerId, collator_pair: CollatorPair, metrics: Metrics, diff --git a/node/network/collator-protocol/src/validator_side.rs b/node/network/collator-protocol/src/validator_side.rs index 7584303c22d0..e0a378a20e13 100644 --- a/node/network/collator-protocol/src/validator_side.rs +++ b/node/network/collator-protocol/src/validator_side.rs @@ -539,7 +539,7 @@ where /// Notify a collator that its collation got seconded. #[tracing::instrument(level = "trace", skip(ctx, peer_data), fields(subsystem = LOG_TARGET))] async fn notify_collation_seconded( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, peer_data: &HashMap, id: CollatorId, relay_parent: Hash, diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index fc1af1d39ea7..ef43e7b0eb72 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -589,7 +589,7 @@ enum Message { impl Message { async fn receive( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, from_requester: &mut mpsc::Receiver, from_responder: &mut mpsc::Receiver, ) -> Message { @@ -1563,7 +1563,7 @@ impl StatementDistribution { #[tracing::instrument(skip(self, ctx), fields(subsystem = LOG_TARGET))] async fn run( self, - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, ) -> std::result::Result<(), Fatal> { let mut peers: HashMap = HashMap::new(); let mut authorities: HashMap = HashMap::new(); From 67e7dde69bcbbe2602a9da7831b1a56cf184a774 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 7 Jun 2021 16:43:00 +0200 Subject: [PATCH 057/161] =?UTF-8?q?=F0=9F=8C=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- node/overseer/overseer-gen/examples/dummy.rs | 17 +++++- .../proc-macro/src/impl_dispatch.rs | 55 ++++++++++--------- .../overseer-gen/proc-macro/src/lib.rs | 1 + .../overseer-gen/proc-macro/src/parse_attr.rs | 18 +++++- .../proc-macro/src/parse_struct.rs | 5 +- node/overseer/src/lib.rs | 4 +- node/overseer/src/metrics.rs | 2 +- 7 files changed, 67 insertions(+), 35 deletions(-) diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index f52a4750dca1..ea5f5ed6ef2d 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -65,13 +65,24 @@ pub struct MsgStrukt(u8); #[derive(Debug, Clone, Copy)] pub struct Plinko; -impl From for MsgStrukt { - fn from(_event: EvX) -> Self { +impl From for MsgStrukt { + fn from(_event: NetworkMsg) -> Self { MsgStrukt(1u8) } } -#[overlord(signal=SigSigSig, event=EvX, error=Yikes, gen=AllMessages)] + +#[derive(Debug, Clone, Copy)] +enum NetworkMsg { + A, + B, + C, +} + + + + +#[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] struct Xxx { #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index 1758ecac81cf..a95e891d341f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -29,37 +29,38 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { .map(|ssf| ssf.consumes.clone()) .collect::>(); - let extern_event_ty = &info.extern_event_ty.clone(); + let mut ts = TokenStream::new(); + if let Some(extern_network_ty) = &info.extern_network_ty.clone() { + ts.extend(quote! { + impl #message_wrapper { + /// Generated dispatch iterator generator. + pub fn dispatch_iter(event: #extern_network_ty) -> impl Iterator + Send { + let mut iter = None.into_iter(); - let ts = quote! { - impl #message_wrapper { - /// Generated dispatch iterator generator. - pub fn dispatch_iter(event: #extern_event_ty) -> impl Iterator + Send { - let mut iter = None.into_iter(); + use ::std::convert::TryFrom; - use ::std::convert::TryFrom; + // creates pretty errors when the inner variant + // does not impl `TryFrom< #extern_network_ty >` + fn dispatchable_message_impls_try_from_extern_event>() { + } - // creates pretty errors when the inner variant - // does not impl `TryFrom< #extern_event_ty >` - fn dispatchable_message_impls_try_from_extern_event>() { - } - - #( - dispatchable_message_impls_try_from_extern_event::< #dispatchable >(); + #( + dispatchable_message_impls_try_from_extern_event::< #dispatchable >(); - let mut iter = iter.chain( - ::std::iter::once( - // alt: - // #dispatchable :: try_from( event ) - event.focus().ok().map(|event| { - #message_wrapper :: #dispatchable ( event ) - }) - ) - ); - )* - iter.filter_map(|x: Option<_>| x) + let mut iter = iter.chain( + ::std::iter::once( + // alt: + // #dispatchable :: try_from( event ) + event.focus().ok().map(|event| { + #message_wrapper :: #dispatchable ( event ) + }) + ) + ); + )* + iter.filter_map(|x: Option<_>| x) + } } - } - }; + }); + } Ok(ts) } diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 426530c289dc..67c8ceaeea8c 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -59,6 +59,7 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< extern_event_ty: args.extern_event_ty, extern_signal_ty: args.extern_signal_ty, extern_error_ty: args.extern_error_ty, + extern_network_ty: args.extern_network_ty, }; let mut additive = impl_overseer_struct(&info)?; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs index 0f5cc750cd51..6ddb75dbe26c 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs @@ -14,6 +14,7 @@ use syn::Token; #[derive(Clone, Debug)] enum AttrItem { ExternEventType(Path), + ExternNetworkType(Path), ExternOverseerSignalType(Path), ExternErrorType(Path), MessageWrapperName(Ident), @@ -25,6 +26,7 @@ impl Spanned for AttrItem { fn span(&self) -> Span { match self { AttrItem::ExternEventType(x) => x.span(), + AttrItem::ExternNetworkType(x) => x.span(), AttrItem::ExternOverseerSignalType(x) => x.span(), AttrItem::ExternErrorType(x) => x.span(), AttrItem::MessageWrapperName(x) => x.span(), @@ -35,6 +37,7 @@ impl Spanned for AttrItem { } const TAG_EXT_EVENT_TY: &str = "event"; +const TAG_EXT_NETWORK_TY: &str = "network"; const TAG_EXT_SIGNAL_TY: &str = "signal"; const TAG_EXT_ERROR_TY: &str = "error"; const TAG_GEN_TY: &str = "gen"; @@ -45,6 +48,7 @@ impl AttrItem { fn key(&self) -> &'static str { match self { AttrItem::ExternEventType(_) => TAG_EXT_EVENT_TY, + AttrItem::ExternNetworkType(_) => TAG_EXT_NETWORK_TY, AttrItem::ExternOverseerSignalType(_) => TAG_EXT_SIGNAL_TY, AttrItem::ExternErrorType(_) => TAG_EXT_ERROR_TY, AttrItem::MessageWrapperName(_) => TAG_GEN_TY, @@ -62,6 +66,9 @@ impl Parse for AttrItem { Ok(if key == TAG_EXT_SIGNAL_TY { let path = input.parse::()?; AttrItem::ExternOverseerSignalType(path) + } else if key == TAG_EXT_NETWORK_TY { + let path = input.parse::()?; + AttrItem::ExternNetworkType(path) } else if key == TAG_EXT_EVENT_TY { let path = input.parse::()?; AttrItem::ExternEventType(path) @@ -90,6 +97,10 @@ pub(crate) struct AttrArgs { pub(crate) extern_event_ty: Path, pub(crate) extern_signal_ty: Path, pub(crate) extern_error_ty: Path, + /// A external subsystem that both consumes and produces messages + /// but is not part of the band of subsystems, it's a mere proxy + /// to another entity that consumes/produces messages. + pub(crate) extern_network_ty: Option, pub(crate) signal_channel_capacity: usize, pub(crate) message_channel_capacity: usize, } @@ -145,9 +156,13 @@ impl Parse for AttrArgs { .remove(TAG_EXT_EVENT_TY) .map(|x| if let AttrItem::ExternEventType(x) = x { x.clone() } else { unreachable!() }) .ok_or_else(|| { - Error::new(span, format!("Must declare the external event type via `{}=..`.", TAG_EXT_EVENT_TY)) + Error::new(span, format!("Must declare the external network type via `{}=..`.", TAG_EXT_NETWORK_TY)) })?; + let extern_network_ty = unique + .remove(TAG_EXT_NETWORK_TY) + .map(|x| if let AttrItem::ExternNetworkType(x) = x { x.clone() } else { unreachable!() }); + let message_wrapper = unique .remove(TAG_GEN_TY) .map(|x| if let AttrItem::MessageWrapperName(x) = x { x.clone() } else { unreachable!() }) @@ -166,6 +181,7 @@ impl Parse for AttrArgs { extern_event_ty, extern_signal_ty, extern_error_ty, + extern_network_ty, message_wrapper, }) } diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 2e6ed5bcf1b6..f4338641e02f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -113,9 +113,12 @@ pub(crate) struct OverseerInfo { /// Signals to be sent, sparse information that is used intermittently. pub(crate) extern_signal_ty: Path, - /// Incoming event type from the outer world, commonly from the network. + /// Incoming event type from the outer world, usually an external framework of some sort. pub(crate) extern_event_ty: Path, + /// Incoming event type from an external entity, commonly from the network. + pub(crate) extern_network_ty: Option, + /// Incoming event type from the outer world, commonly from the network. pub(crate) extern_error_ty: Path, } diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index a906603a15d4..578fcb318f3e 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -236,7 +236,7 @@ impl From> for BlockInfo { } /// Some event from the outer world. -enum Event { +pub enum Event { BlockImported(BlockInfo), BlockFinalized(BlockInfo), MsgToSubsystem(AllMessages), @@ -245,7 +245,7 @@ enum Event { } /// Some request from outer world. -enum ExternalRequest { +pub enum ExternalRequest { WaitForActivation { hash: Hash, response_channel: oneshot::Sender>, diff --git a/node/overseer/src/metrics.rs b/node/overseer/src/metrics.rs index 96fccd789aaf..85c9563a7d49 100644 --- a/node/overseer/src/metrics.rs +++ b/node/overseer/src/metrics.rs @@ -34,7 +34,7 @@ struct MetricsInner { } #[derive(Default, Clone)] -pub(crate) struct Metrics(Option); +pub struct Metrics(Option); impl Metrics { pub(crate) fn on_head_activated(&self) { From 6beb6566521dcc44ef097eb199a962b39d5aa011 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 7 Jun 2021 19:41:52 +0200 Subject: [PATCH 058/161] remove dispatch-gen --- Cargo.lock | 12 - node/subsystem/dispatch-gen/Cargo.toml | 18 -- node/subsystem/dispatch-gen/src/lib.rs | 208 ------------------ .../tests/ui/err-01-missing-skip.rs | 37 ---- .../tests/ui/err-01-missing-skip.stderr | 14 -- .../tests/ui/err-02-missing-from.rs | 41 ---- .../tests/ui/err-02-missing-from.stderr | 10 - .../tests/ui/ok-01-with-intermediate.rs | 48 ---- 8 files changed, 388 deletions(-) delete mode 100644 node/subsystem/dispatch-gen/Cargo.toml delete mode 100644 node/subsystem/dispatch-gen/src/lib.rs delete mode 100644 node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.rs delete mode 100644 node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.stderr delete mode 100644 node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.rs delete mode 100644 node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.stderr delete mode 100644 node/subsystem/dispatch-gen/tests/ui/ok-01-with-intermediate.rs diff --git a/Cargo.lock b/Cargo.lock index f52f4ad6156b..6f084e5b587e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6174,7 +6174,6 @@ dependencies = [ "polkadot-node-subsystem-test-helpers", "polkadot-overseer-gen", "polkadot-primitives", - "polkadot-procmacro-subsystem-dispatch-gen", "polkadot-statement-table", "sc-network", "smallvec 1.6.1", @@ -6336,17 +6335,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "polkadot-procmacro-subsystem-dispatch-gen" -version = "0.1.0" -dependencies = [ - "assert_matches", - "proc-macro2", - "quote", - "syn", - "trybuild", -] - [[package]] name = "polkadot-rpc" version = "0.9.1" diff --git a/node/subsystem/dispatch-gen/Cargo.toml b/node/subsystem/dispatch-gen/Cargo.toml deleted file mode 100644 index 09de1362c92c..000000000000 --- a/node/subsystem/dispatch-gen/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "polkadot-procmacro-subsystem-dispatch-gen" -version = "0.1.0" -authors = ["Parity Technologies "] -edition = "2018" -description = "Small proc macro to create the distribution code for network events" - -[lib] -proc-macro = true - -[dependencies] -syn = { version = "1.0.60", features = ["full"] } -quote = "1.0.9" -proc-macro2 = "1.0.24" -assert_matches = "1.5.0" - -[dev-dependencies] -trybuild = "1.0.41" diff --git a/node/subsystem/dispatch-gen/src/lib.rs b/node/subsystem/dispatch-gen/src/lib.rs deleted file mode 100644 index 737712639cff..000000000000 --- a/node/subsystem/dispatch-gen/src/lib.rs +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use proc_macro2::TokenStream; -use quote::{quote, ToTokens}; -use std::fmt; -use syn::{parse2, Error, Fields, FieldsNamed, FieldsUnnamed, Ident, ItemEnum, Path, Result, Type, Variant}; - -#[proc_macro_attribute] -pub fn subsystem_dispatch_gen(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream { - let attr: TokenStream = attr.into(); - let item: TokenStream = item.into(); - let mut backup = item.clone(); - impl_subsystem_dispatch_gen(attr.into(), item).unwrap_or_else(|err| { - backup.extend(err.to_compile_error()); - backup - }).into() -} - -/// An enum variant without base type. -#[derive(Clone)] -struct EnumVariantDispatchWithTy { - // enum ty name - ty: Ident, - // variant - variant: EnumVariantDispatch, -} - -impl fmt::Debug for EnumVariantDispatchWithTy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}::{:?}", self.ty, self.variant) - } -} - -impl ToTokens for EnumVariantDispatchWithTy { - fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { - if let Some(inner) = &self.variant.inner { - let enum_name = &self.ty; - let variant_name = &self.variant.name; - - let quoted = quote! { - #enum_name::#variant_name(#inner::from(event)) - }; - quoted.to_tokens(tokens); - } - } -} - -/// An enum variant without the base type, contains the relevant inner type. -#[derive(Clone)] -struct EnumVariantDispatch { - /// variant name - name: Ident, - /// The inner type for which a `From::from` impl is anticipated from the input type. - /// No code will be generated for this enum variant if `inner` is `None`. - inner: Option, -} - -impl fmt::Debug for EnumVariantDispatch { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}(..)", self.name) - } -} - -fn prepare_enum_variant(variant: &mut Variant) -> Result { - let skip = variant.attrs.iter().find(|attr| attr.path.is_ident("skip")).is_some(); - variant.attrs = variant.attrs.iter().filter(|attr| !attr.path.is_ident("skip")).cloned().collect::>(); - - let variant = variant.clone(); - let span = variant.ident.span(); - let inner = match variant.fields.clone() { - // look for one called inner - Fields::Named(FieldsNamed { brace_token: _, named }) if !skip => named - .iter() - .find_map( - |field| { - if let Some(ident) = &field.ident { - if ident == "inner" { - return Some(Some(field.ty.clone())) - } - } - None - }, - ) - .ok_or_else(|| { - Error::new(span, "To dispatch with struct enum variant, one element must named `inner`") - })?, - - // technically, if it has no inner types we cound not require the #[skip] annotation, but better make it consistent - Fields::Unnamed(FieldsUnnamed { paren_token: _, unnamed }) if !skip => unnamed - .first() - .map(|field| Some(field.ty.clone())) - .ok_or_else(|| Error::new(span, "Must be annotated with skip, even if no inner types exist."))?, - _ if skip => None, - Fields::Unit => { - return Err(Error::new( - span, - "Must be annotated with #[skip].", - )) - } - Fields::Unnamed(_) => { - return Err(Error::new( - span, - "Must be annotated with #[skip] or have in `inner` element which impls `From<_>`.", - )) - } - Fields::Named(_) => { - return Err(Error::new( - span, - "Must be annotated with #[skip] or the first wrapped type must impl `From<_>`.", - )) - } - }; - - Ok(EnumVariantDispatch { name: variant.ident, inner }) -} - -fn impl_subsystem_dispatch_gen(attr: TokenStream, item: TokenStream) -> Result { - let event_ty = parse2::(attr)?; - - let mut ie = parse2::(item)?; - - let message_enum = ie.ident.clone(); - let variants = ie.variants.iter_mut().try_fold(Vec::::new(), |mut acc, variant| { - let variant = prepare_enum_variant(variant)?; - if variant.inner.is_some() { - acc.push(EnumVariantDispatchWithTy { ty: message_enum.clone(), variant }) - } - Ok::<_, syn::Error>(acc) - })?; - - let mut orig = ie.to_token_stream(); - - let msg = "Generated by #[subsystem_dispatch_gen] proc-macro."; - - orig.extend(quote! { - impl #message_enum { - #[doc = #msg] - pub fn dispatch_iter(event: #event_ty) -> impl Iterator + Send { - let mut iter = None.into_iter(); - - #( - let mut iter = iter.chain(std::iter::once(event.focus().ok().map(|event| { - #variants - }))); - )* - iter.filter_map(|x| x) - } - } - }); - Ok(orig) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn basic() { - let attr = quote! { - NetEvent - }; - - let item = quote! { - /// Documentation. - #[derive(Clone)] - enum AllMessages { - - Sub1(Inner1), - - #[skip] - /// D3 - Sub3, - - /// D4 - #[skip] - Sub4(Inner2), - - /// D2 - Sub2(Inner2), - } - }; - - let output = impl_subsystem_dispatch_gen(attr, item).expect("Simple example always works. qed"); - println!("//generated:"); - println!("{}", output); - } - - #[test] - fn ui() { - let t = trybuild::TestCases::new(); - t.compile_fail("tests/ui/err-*.rs"); - t.pass("tests/ui/ok-*.rs"); - } -} diff --git a/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.rs b/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.rs deleted file mode 100644 index 7248a7181e49..000000000000 --- a/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.rs +++ /dev/null @@ -1,37 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_subsystem_dispatch_gen::subsystem_dispatch_gen; - -/// The event type in question. -#[derive(Clone, Copy)] -enum Event { - Smth, - Else, -} - -impl Event { - fn focus(&self) -> std::result::Result { - unimplemented!("foo") - } -} - -/// This should have a `From` impl but does not. -#[derive(Clone)] -enum Inner { - Foo, - Bar(Event), -} - -#[subsystem_dispatch_gen(Event)] -#[derive(Clone)] -enum AllMessages { - /// Foo - Vvvvvv(Inner), - - /// Missing a `#[skip]` annotation - Uuuuu, -} - -fn main() { - let _x = AllMessages::dispatch_iter(Event::Else); -} diff --git a/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.stderr b/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.stderr deleted file mode 100644 index 855521d2c4ef..000000000000 --- a/node/subsystem/dispatch-gen/tests/ui/err-01-missing-skip.stderr +++ /dev/null @@ -1,14 +0,0 @@ -error: Must be annotated with #[skip]. - --> $DIR/err-01-missing-skip.rs:32:5 - | -32 | Uuuuu, - | ^^^^^ - -error[E0599]: no variant or associated item named `dispatch_iter` found for enum `AllMessages` in the current scope - --> $DIR/err-01-missing-skip.rs:36:27 - | -27 | enum AllMessages { - | ---------------- variant or associated item `dispatch_iter` not found here -... -36 | let _x = AllMessages::dispatch_iter(Event::Else); - | ^^^^^^^^^^^^^ variant or associated item not found in `AllMessages` diff --git a/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.rs b/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.rs deleted file mode 100644 index a7abef2c8709..000000000000 --- a/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.rs +++ /dev/null @@ -1,41 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_subsystem_dispatch_gen::subsystem_dispatch_gen; - -/// The event type in question. -#[derive(Clone, Copy, Debug)] -enum Event { - Smth, - Else, -} - -impl Event { - fn focus(&self) -> std::result::Result { - Ok(Intermediate(self.clone())) - } -} - -#[derive(Debug, Clone)] -struct Intermediate(Event); - - -/// This should have a `From` impl but does not. -#[derive(Debug, Clone)] -enum Inner { - Foo, - Bar(Intermediate), -} - -#[subsystem_dispatch_gen(Event)] -#[derive(Clone)] -enum AllMessages { - /// Foo - Vvvvvv(Inner), - - #[skip] - Uuuuu, -} - -fn main() { - let _x = AllMessages::dispatch_iter(Event::Else); -} diff --git a/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.stderr b/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.stderr deleted file mode 100644 index bf82201a7e40..000000000000 --- a/node/subsystem/dispatch-gen/tests/ui/err-02-missing-from.stderr +++ /dev/null @@ -1,10 +0,0 @@ -error[E0308]: mismatched types - --> $DIR/err-02-missing-from.rs:29:1 - | -29 | #[subsystem_dispatch_gen(Event)] - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - | | - | expected enum `Inner`, found struct `Intermediate` - | help: try using a variant of the expected enum: `Inner::Bar(#[subsystem_dispatch_gen(Event)])` - | - = note: this error originates in an attribute macro (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/node/subsystem/dispatch-gen/tests/ui/ok-01-with-intermediate.rs b/node/subsystem/dispatch-gen/tests/ui/ok-01-with-intermediate.rs deleted file mode 100644 index b160bf9ce1c1..000000000000 --- a/node/subsystem/dispatch-gen/tests/ui/ok-01-with-intermediate.rs +++ /dev/null @@ -1,48 +0,0 @@ -#![allow(dead_code)] - -use polkadot_procmacro_subsystem_dispatch_gen::subsystem_dispatch_gen; - -/// The event type in question. -#[derive(Clone, Copy, Debug)] -enum Event { - Smth, - Else, -} - -impl Event { - fn focus(&self) -> std::result::Result { - Ok(Intermediate(self.clone())) - } -} - - -#[derive(Debug, Clone)] -struct Intermediate(Event); - - -/// This should have a `From` impl but does not. -#[derive(Clone, Debug)] -enum Inner { - Foo, - Bar(Intermediate), -} - -impl From for Inner { - fn from(src: Intermediate) -> Self { - Inner::Bar(src) - } -} - -#[subsystem_dispatch_gen(Event)] -#[derive(Clone)] -enum AllMessages { - /// Foo - Vvvvvv(Inner), - - #[skip] - Uuuuu, -} - -fn main() { - let _x = AllMessages::dispatch_iter(Event::Else); -} From 60e46abe6a996045747c44e4de242b832e2d3be4 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 7 Jun 2021 19:43:18 +0200 Subject: [PATCH 059/161] =?UTF-8?q?=F0=9F=8C=882?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 1 + node/core/backing/src/lib.rs | 2 +- node/network/bridge/src/lib.rs | 2 +- node/overseer/examples/minimal-example.rs | 4 +- node/overseer/overseer-gen/Cargo.toml | 2 + .../proc-macro/src/impl_message_wrapper.rs | 6 +- .../proc-macro/src/impl_overseer.rs | 9 +- node/overseer/overseer-gen/src/lib.rs | 3 + node/overseer/src/lib.rs | 128 +++++++----------- node/overseer/src/metrics.rs | 39 +++--- node/overseer/src/subsystems.rs | 36 ++--- node/subsystem/Cargo.toml | 1 - node/subsystem/src/lib.rs | 7 +- node/subsystem/src/messages.rs | 1 - .../src/messages/network_bridge_event.rs | 4 +- 15 files changed, 112 insertions(+), 133 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f084e5b587e..92e5ab2ced57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6270,6 +6270,7 @@ dependencies = [ "futures-util", "metered-channel", "pin-project 1.0.4", + "polkadot-node-network-protocol", "polkadot-node-subsystem", "polkadot-overseer-gen-proc-macro", "sp-core", diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index 90d5da069ead..b913b2615183 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -650,7 +650,7 @@ impl CandidateBackingJob { #[tracing::instrument(level = "trace", skip(self, sender, params), fields(subsystem = LOG_TARGET))] async fn background_validate_and_make_available( &mut self, - sender: &mut JobSender<>>, + sender: &mut JobSender>, params: BackgroundValidationParams< impl SubsystemSender , impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Send + 'static + Sync diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index d9f236d5aae3..420abd3ef7d4 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -805,7 +805,7 @@ async fn handle_network_messages( #[tracing::instrument(skip(bridge, ctx, network_stream), fields(subsystem = LOG_TARGET))] async fn run_network( bridge: NetworkBridge, - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, network_stream: BoxStream<'static, NetworkEvent>, ) -> SubsystemResult<()> where diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index e8d45fcc6389..c4b8dc6c56a8 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -44,7 +44,7 @@ impl HeadSupportsParachains for AlwaysSupportsParachains { struct Subsystem1; impl Subsystem1 { - async fn run(mut ctx: impl SubsystemContext) { + async fn run(mut ctx: impl SubsystemContext) { loop { match ctx.try_recv().await { Ok(Some(msg)) => { @@ -95,7 +95,7 @@ impl Subsystem for Subsystem1 struct Subsystem2; impl Subsystem2 { - async fn run(mut ctx: impl SubsystemContext) { + async fn run(mut ctx: impl SubsystemContext) { ctx.spawn( "subsystem-2-job", Box::pin(async { diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index 1f5adc0c1583..82339ceb63a3 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -12,6 +12,8 @@ async-trait = "0.1" thiserror = "1" overseer-gen-proc-macro = { package = "polkadot-overseer-gen-proc-macro", path = "./proc-macro" } metered = { package = "metered-channel", path = "../../metered-channel" } +polkadot-node-network-protocol = { path = "../../network/protocol"} + # trait SpawnNamed sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } futures-timer = "3.0.2" diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 38e342ce298d..e16086e6c80c 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -19,9 +19,9 @@ pub(crate) fn impl_message_wrapper_enum(info: &OverseerInfo) -> Result for #message_wrapper { - fn from(inner: #consumes) -> Self { - #message_wrapper :: #consumes ( inner ) + impl ::std::convert::From< #consumes > for #message_wrapper { + fn from(message: #consumes) -> Self { + #message_wrapper :: #consumes ( message ) } } )* diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index f6221c50aef9..6b7ca5d869a2 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -128,11 +128,11 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result(&self, mapper: Mapper) + pub fn map_subsystems<'a, Mapper, Output>(&'a self, mapper: Mapper) -> Vec where #( - Mapper: for<'a> MapSubsystem<&'a OverseenSubsystem< #consumes >, Output=Output>, + Mapper: MapSubsystem<&'a OverseenSubsystem< #consumes >, Output=Output>, )* { vec![ @@ -141,7 +141,12 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result (&'a mut self) -> &'a mut S { + &mut self.spawner + } } + }; Ok(ts) diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 62ff798fb2c2..28fd491ee3ce 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -100,6 +100,9 @@ pub use futures_util::stream::StreamExt; #[doc(hidden)] pub use futures_util::future::FutureExt; + +pub use polkadot_node_network_protocol::WrongVariant; + /// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. /// /// Used to launch jobs. diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 578fcb318f3e..aa29c6a11fb4 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -61,25 +61,25 @@ use std::fmt::{self, Debug}; use std::pin::Pin; -use std::sync::{atomic::{self, AtomicUsize}, Arc}; -use std::task::Poll; +use std::sync::Arc; use std::time::Duration; use std::collections::{hash_map, HashMap}; use std::iter::FromIterator; use futures::channel::oneshot; use futures::{ - poll, select, + select, future::BoxFuture, - stream::{self, FuturesUnordered, Fuse}, Future, FutureExt, StreamExt, }; -use futures_timer::Delay; use polkadot_primitives::v1::{Block, BlockId,BlockNumber, Hash, ParachainHost}; use client::{BlockImportNotification, BlockchainEvents, FinalityNotification}; use sp_api::{ApiExt, ProvideRuntimeApi}; +use polkadot_node_network_protocol::{ + v1 as protocol_v1, +}; use polkadot_node_subsystem::messages::{ CandidateValidationMessage, CandidateBackingMessage, CandidateSelectionMessage, ChainApiMessage, StatementDistributionMessage, @@ -95,22 +95,18 @@ pub use polkadot_node_subsystem::{ errors::{SubsystemResult, SubsystemError,}, ActiveLeavesUpdate, ActivatedLeaf, jaeger, }; -use polkadot_node_network_protocol::{ - PeerId, View, v1 as protocol_v1, UnifiedReputationChange as Rep, -}; /// TODO legacy, to be deleted, left for easier integration mod subsystems; -use self::subsystems::{AllSubsystems, AllSubsystemsSame}; +use self::subsystems::AllSubsystems; mod metrics; use self::metrics::Metrics; -use polkadot_node_subsystem_util::{metrics::{prometheus, Metrics as MetricsTrait}, metered, Metronome}; +use polkadot_node_subsystem_util::{metrics::{prometheus, Metrics as MetricsTrait}, Metronome}; use polkadot_overseer_gen::{ TimeoutExt, SpawnNamed, - SpawnedSubsystem, Subsystem, SubsystemMeterReadouts, SubsystemMeters, @@ -284,33 +280,13 @@ pub async fn forward_events>( } } - -// async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) -// -> SubsystemResult<()> -// { -// self.sender().send(ToOverseer::SpawnJob { -// name, -// s, -// }).await.map_err(Into::into) -// } - -// async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) -// -> SubsystemResult<()> -// { -// self.sender().send(ToOverseer::SpawnBlockingJob { -// name, -// s, -// }).await.map_err(Into::into) -// } - -// NetworkBridgeEvent, - /// The `Overseer` itself. #[overlord( gen=AllMessages, event=Event, signal=OverseerSignal, - error=SubsystemError + error=SubsystemError, + network=NetworkBridgeEvent, )] pub struct Overseer { @@ -489,7 +465,7 @@ where all_subsystems: AllSubsystems, prometheus_registry: Option<&prometheus::Registry>, supports_parachains: SupportsParachains, - mut s: S, + s: S, ) -> SubsystemResult<(Self, Handler)> where CV: Subsystem, SubsystemError> + Send, @@ -512,50 +488,9 @@ where GS: Subsystem, SubsystemError> + Send, S: SpawnNamed, { - let metrics: Metrics = ::register(prometheus_registry)?; - // spawn the metrics metronome task - { - struct ExtractNameAndMeters; - impl<'a, T: 'a> MapSubsystem<&'a OverseenSubsystem> for ExtractNameAndMeters { - type Output = (&'static str, SubsystemMeters); - - fn map_subsystem(&self, subsystem: &'a OverseenSubsystem) -> Self::Output { - let instance = subsystem.instance.as_ref() - .expect("Extraction is done directly after spawning when subsystems\ - have not concluded; qed"); - - ( - instance.name, - instance.meters.clone(), - ) - } - } - - let subsystem_meters = all_subsystems - .as_ref() - .map_subsystems::(ExtractNameAndMeters); - - let metronome_metrics = metrics.clone(); - let metronome = Metronome::new(std::time::Duration::from_millis(950)) - .for_each(move |_| { - let subsystem_meters = subsystem_meters.as_ref() - .map_subsystems(|&(name, ref meters): &(_, SubsystemMeters)| (name, meters.read())); - - // We combine the amount of messages from subsystems to the overseer - // as well as the amount of messages from external sources to the overseer - // into one `to_overseer` value. - metronome_metrics.channel_fill_level_snapshot(subsystem_meters); - - async move { - () - } - }); - s.spawn("metrics_metronome", Box::pin(metronome)); - } - - let (overseer, handler) = Self::builder() + let (mut overseer, handler) = Self::builder() .candidate_validation(all_subsystems.candidate_validation) .candidate_backing(all_subsystems.candidate_backing) .candidate_selection(all_subsystems.candidate_selection) @@ -580,10 +515,49 @@ where .active_leaves(Default::default()) .span_per_active_leaf(Default::default()) .activation_external_listeners(Default::default()) - .metrics(metrics) + .metrics(metrics.clone()) .spawner(s) .build()?; + // spawn the metrics metronome task + { + struct ExtractNameAndMeters; + + impl<'a, T: 'a> MapSubsystem<&'a OverseenSubsystem> for ExtractNameAndMeters { + type Output = Option<(&'static str, SubsystemMeters)>; + + fn map_subsystem(&self, subsystem: &'a OverseenSubsystem) -> Self::Output { + subsystem.instance.as_ref().map(|instance| { + ( + instance.name, + instance.meters.clone(), + ) + }) + } + } + let subsystem_meters = overseer.map_subsystems(ExtractNameAndMeters); + + let metronome_metrics = metrics.clone(); + let metronome = Metronome::new(std::time::Duration::from_millis(950)) + .for_each(move |_| { + + // We combine the amount of messages from subsystems to the overseer + // as well as the amount of messages from external sources to the overseer + // into one `to_overseer` value. + metronome_metrics.channel_fill_level_snapshot( + subsystem_meters.iter() + .cloned() + .filter_map(|x| x) + .map(|(name, ref meters)| (name, meters.read())) + ); + + async move { + () + } + }); + overseer.spawner().spawn("metrics_metronome", Box::pin(metronome)); + } + Ok((overseer, Handler(handler))) } diff --git a/node/overseer/src/metrics.rs b/node/overseer/src/metrics.rs index 85c9563a7d49..7a07051dbb24 100644 --- a/node/overseer/src/metrics.rs +++ b/node/overseer/src/metrics.rs @@ -17,7 +17,7 @@ //! Prometheus metrics related to the overseer and its channels. use super::*; -use polkadot_node_subsystem_util::{TimeoutExt, metrics::{self, prometheus}, metered, Metronome}; +use polkadot_node_subsystem_util::metrics::{self, prometheus}; /// Overseer Prometheus metrics. #[derive(Clone)] @@ -57,30 +57,31 @@ impl Metrics { pub(crate) fn channel_fill_level_snapshot( &self, - to_subsystem: AllSubsystemsSame<(&'static str, SubsystemMeterReadouts)>, + collection: impl IntoIterator, ) { - self.0.as_ref().map(|metrics| { - to_subsystem.map_subsystems( - |(name, readouts): (_, SubsystemMeterReadouts)| { - metrics.to_subsystem_bounded_sent.with_label_values(&[name]) - .set(readouts.bounded.sent as u64); + if let Some(metrics) = &self.0 { + collection.into_iter().for_each( + |(name, readouts): (_, SubsystemMeterReadouts)| { + metrics.to_subsystem_bounded_sent.with_label_values(&[name]) + .set(readouts.bounded.sent as u64); - metrics.to_subsystem_bounded_received.with_label_values(&[name]) - .set(readouts.bounded.received as u64); + metrics.to_subsystem_bounded_received.with_label_values(&[name]) + .set(readouts.bounded.received as u64); - metrics.to_subsystem_unbounded_sent.with_label_values(&[name]) - .set(readouts.unbounded.sent as u64); + metrics.to_subsystem_unbounded_sent.with_label_values(&[name]) + .set(readouts.unbounded.sent as u64); - metrics.to_subsystem_unbounded_received.with_label_values(&[name]) - .set(readouts.unbounded.received as u64); + metrics.to_subsystem_unbounded_received.with_label_values(&[name]) + .set(readouts.unbounded.received as u64); - metrics.signals_sent.with_label_values(&[name]) - .set(readouts.signals.sent as u64); + metrics.signals_sent.with_label_values(&[name]) + .set(readouts.signals.sent as u64); - metrics.signals_received.with_label_values(&[name]) - .set(readouts.signals.received as u64); - }); - }); + metrics.signals_received.with_label_values(&[name]) + .set(readouts.signals.received as u64); + } + ); + } } } diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index bfb3e866257e..a5a9002584f6 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -219,24 +219,24 @@ impl Mapper: MapSubsystem, { AllSubsystems { - candidate_validation: mapper.map_subsystem(self.candidate_validation), - candidate_backing: mapper.map_subsystem(self.candidate_backing), - candidate_selection: mapper.map_subsystem(self.candidate_selection), - statement_distribution: mapper.map_subsystem(self.statement_distribution), - availability_distribution: mapper.map_subsystem(self.availability_distribution), - availability_recovery: mapper.map_subsystem(self.availability_recovery), - bitfield_signing: mapper.map_subsystem(self.bitfield_signing), - bitfield_distribution: mapper.map_subsystem(self.bitfield_distribution), - provisioner: mapper.map_subsystem(self.provisioner), - runtime_api: mapper.map_subsystem(self.runtime_api), - availability_store: mapper.map_subsystem(self.availability_store), - network_bridge: mapper.map_subsystem(self.network_bridge), - chain_api: mapper.map_subsystem(self.chain_api), - collation_generation: mapper.map_subsystem(self.collation_generation), - collator_protocol: mapper.map_subsystem(self.collator_protocol), - approval_distribution: mapper.map_subsystem(self.approval_distribution), - approval_voting: mapper.map_subsystem(self.approval_voting), - gossip_support: mapper.map_subsystem(self.gossip_support), + candidate_validation: >::map_subsystem(&mapper, self.candidate_validation), + candidate_backing: >::map_subsystem(&mapper, self.candidate_backing), + candidate_selection: >::map_subsystem(&mapper, self.candidate_selection), + statement_distribution: >::map_subsystem(&mapper, self.statement_distribution), + availability_distribution: >::map_subsystem(&mapper, self.availability_distribution), + availability_recovery: >::map_subsystem(&mapper, self.availability_recovery), + bitfield_signing: >::map_subsystem(&mapper, self.bitfield_signing), + bitfield_distribution: >::map_subsystem(&mapper, self.bitfield_distribution), + provisioner: >::map_subsystem(&mapper, self.provisioner), + runtime_api: >::map_subsystem(&mapper, self.runtime_api), + availability_store: >::map_subsystem(&mapper, self.availability_store), + network_bridge: >::map_subsystem(&mapper, self.network_bridge), + chain_api: >::map_subsystem(&mapper, self.chain_api), + collation_generation: >::map_subsystem(&mapper, self.collation_generation), + collator_protocol: >::map_subsystem(&mapper, self.collator_protocol), + approval_distribution: >::map_subsystem(&mapper, self.approval_distribution), + approval_voting: >::map_subsystem(&mapper, self.approval_voting), + gossip_support: >::map_subsystem(&mapper, self.gossip_support), } } } diff --git a/node/subsystem/Cargo.toml b/node/subsystem/Cargo.toml index f2d0ebd8778a..8c34730b8809 100644 --- a/node/subsystem/Cargo.toml +++ b/node/subsystem/Cargo.toml @@ -23,7 +23,6 @@ polkadot-primitives = { path = "../../primitives" } polkadot-statement-table = { path = "../../statement-table" } polkadot-node-jaeger = { path = "../jaeger" } polkadot-overseer-gen = { path = "../overseer/overseer-gen" } -polkadot-procmacro-subsystem-dispatch-gen = { path = "dispatch-gen" } sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } smallvec = "1.6.1" sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index da4f1cdf1003..69cd30b2b5ef 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -22,14 +22,9 @@ #![warn(missing_docs)] -use std::{pin::Pin, sync::Arc, fmt}; - -use futures::prelude::*; -use futures::channel::{mpsc, oneshot}; -use futures::future::BoxFuture; +use std::{sync::Arc, fmt}; use polkadot_primitives::v1::{Hash, BlockNumber}; -use async_trait::async_trait; use smallvec::SmallVec; pub mod errors; diff --git a/node/subsystem/src/messages.rs b/node/subsystem/src/messages.rs index 87a68dafa95d..80fea73511d0 100644 --- a/node/subsystem/src/messages.rs +++ b/node/subsystem/src/messages.rs @@ -50,7 +50,6 @@ use polkadot_primitives::v1::{ CandidateIndex, GroupIndex, MultiDisputeStatementSet, SignedAvailabilityBitfields, }; use polkadot_statement_table::v1::Misbehavior; -use polkadot_procmacro_subsystem_dispatch_gen::subsystem_dispatch_gen; use std::{sync::Arc, collections::btree_map::BTreeMap}; diff --git a/node/subsystem/src/messages/network_bridge_event.rs b/node/subsystem/src/messages/network_bridge_event.rs index 8d7c2379858a..9eef3ad256cc 100644 --- a/node/subsystem/src/messages/network_bridge_event.rs +++ b/node/subsystem/src/messages/network_bridge_event.rs @@ -60,12 +60,12 @@ impl NetworkBridgeEvent { where T: 'a + Clone, &'a T: TryFrom<&'a M, Error = WrongVariant> { Ok(match *self { + NetworkBridgeEvent::PeerMessage(ref peer, ref msg) + => NetworkBridgeEvent::PeerMessage(peer.clone(), <&'a T>::try_from(msg)?.clone()), NetworkBridgeEvent::PeerConnected(ref peer, ref role, ref authority_id) => NetworkBridgeEvent::PeerConnected(peer.clone(), role.clone(), authority_id.clone()), NetworkBridgeEvent::PeerDisconnected(ref peer) => NetworkBridgeEvent::PeerDisconnected(peer.clone()), - NetworkBridgeEvent::PeerMessage(ref peer, ref msg) - => NetworkBridgeEvent::PeerMessage(peer.clone(), <&'a T>::try_from(msg)?.clone()), NetworkBridgeEvent::PeerViewChange(ref peer, ref view) => NetworkBridgeEvent::PeerViewChange(peer.clone(), view.clone()), NetworkBridgeEvent::OurViewChange(ref view) From f2b3395d2f811d5596dd06f6abe0b999dc7704ac Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 7 Jun 2021 22:36:55 +0200 Subject: [PATCH 060/161] partial fixin --- .../proc-macro/src/impl_dispatch.rs | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index a95e891d341f..7754423417ce 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -36,23 +36,17 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { /// Generated dispatch iterator generator. pub fn dispatch_iter(event: #extern_network_ty) -> impl Iterator + Send { let mut iter = None.into_iter(); - - use ::std::convert::TryFrom; - - // creates pretty errors when the inner variant - // does not impl `TryFrom< #extern_network_ty >` - fn dispatchable_message_impls_try_from_extern_event>() { - } - #( - dispatchable_message_impls_try_from_extern_event::< #dispatchable >(); - let mut iter = iter.chain( ::std::iter::once( - // alt: - // #dispatchable :: try_from( event ) - event.focus().ok().map(|event| { - #message_wrapper :: #dispatchable ( event ) + event.focus::<'_, NetworkBridgeEvent < #dispatchable >>() + .ok().map(|event: NetworkBridgeEvent < #dispatchable >| -> #message_wrapper { + #message_wrapper :: #dispatchable ( + // the inner type of the enum variant + #message_wrapper :: #dispatchable ( + #dispatchable :: from( event ) + ) + ) }) ) ); From ba19adbb707c9ba75dbc87d4d42ff9f04dae0e24 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 8 Jun 2021 09:14:38 +0200 Subject: [PATCH 061/161] foo --- node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index 7754423417ce..8c78f4d21d5e 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -40,12 +40,10 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { let mut iter = iter.chain( ::std::iter::once( event.focus::<'_, NetworkBridgeEvent < #dispatchable >>() - .ok().map(|event: NetworkBridgeEvent < #dispatchable >| -> #message_wrapper { + .ok().map(|event| -> #message_wrapper { #message_wrapper :: #dispatchable ( // the inner type of the enum variant - #message_wrapper :: #dispatchable ( - #dispatchable :: from( event ) - ) + #dispatchable :: from( event ) ) }) ) From 9098e8133be667b84666ba6aea277443edc87ca8 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 8 Jun 2021 10:31:45 +0200 Subject: [PATCH 062/161] fixins --- node/network/protocol/src/lib.rs | 6 ++-- .../proc-macro/src/impl_dispatch.rs | 30 +++++++++++-------- node/overseer/src/lib.rs | 14 +++++++-- node/overseer/src/subsystems.rs | 7 ----- node/subsystem-util/src/lib.rs | 1 + 5 files changed, 34 insertions(+), 24 deletions(-) diff --git a/node/network/protocol/src/lib.rs b/node/network/protocol/src/lib.rs index 75e9a8805245..6463882e8e39 100644 --- a/node/network/protocol/src/lib.rs +++ b/node/network/protocol/src/lib.rs @@ -87,13 +87,15 @@ impl Into for ObservedRole { } } +/// Implement `TryFrom` for one enum variant into the inner type. +/// `$m_ty::$variant(inner) -> Ok(inner)` macro_rules! impl_try_from { ($m_ty:ident, $variant:ident, $out:ty) => { impl TryFrom<$m_ty> for $out { type Error = crate::WrongVariant; - #[allow(unreachable_patterns)] // when there is only one variant fn try_from(x: $m_ty) -> Result<$out, Self::Error> { + #[allow(unreachable_patterns)] // when there is only one variant match x { $m_ty::$variant(y) => Ok(y), _ => Err(crate::WrongVariant), @@ -301,7 +303,7 @@ pub mod v1 { UncheckedSignedFullStatement, }; - + /// Network messages used by the bitfield distribution subsystem. #[derive(Debug, Clone, Encode, Decode, PartialEq, Eq)] pub enum BitfieldDistributionMessage { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index 8c78f4d21d5e..f8a7545d8c1f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -34,22 +34,28 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { ts.extend(quote! { impl #message_wrapper { /// Generated dispatch iterator generator. - pub fn dispatch_iter(event: #extern_network_ty) -> impl Iterator + Send { - let mut iter = None.into_iter(); + pub fn dispatch_iter(extern_msg: #extern_network_ty) -> impl Iterator + Send { + None.into_iter() #( - let mut iter = iter.chain( + .chain( ::std::iter::once( - event.focus::<'_, NetworkBridgeEvent < #dispatchable >>() - .ok().map(|event| -> #message_wrapper { - #message_wrapper :: #dispatchable ( - // the inner type of the enum variant - #dispatchable :: from( event ) - ) - }) + extern_msg + // focuses on a `NetworkBridgeEvent< protocol_v1::*Message >` + .focus() + .ok() + // TODO introduce a `net_protocol_ty` to clean this up. + // TODO avoid hardcoding the `NetworkBridgeEvent` type. + .map(|event: NetworkBridgeEvent< protocol_v1:: #dispatchable >| { + #message_wrapper :: #dispatchable ( + // the inner type of the enum variant + #dispatchable :: from( event ) + ) + }) ) - ); + ) )* - iter.filter_map(|x: Option<_>| x) + + .filter_map(|x: Option<_>| x) } } }); diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index aa29c6a11fb4..917a39012fca 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -231,17 +231,25 @@ impl From> for BlockInfo { } } -/// Some event from the outer world. +/// An event from outside the overseer scope, such +/// as the substrate framework or user interaction. pub enum Event { + /// A new block was imported. BlockImported(BlockInfo), + /// A block was finalized with i.e. babe or another consensus algorithm. BlockFinalized(BlockInfo), + /// An explicit message to the subsystem. MsgToSubsystem(AllMessages), + /// An external request, see the inner type for details. ExternalRequest(ExternalRequest), + /// Stop the overseer on i.e. a UNIX signal. Stop, } /// Some request from outer world. pub enum ExternalRequest { + /// Wait for the activation of a particular hash + /// and be notified by means of the return channel. WaitForActivation { hash: Hash, response_channel: oneshot::Sender>, @@ -308,10 +316,10 @@ pub struct Overseer { #[subsystem(no_dispatch, AvailabilityRecoveryMessage)] availability_recovery: AvailabilityRecovery, - #[subsystem(blocking, BitfieldSigningMessage)] + #[subsystem(blocking, no_dispatch, BitfieldSigningMessage)] bitfield_signing: BitfieldSigning, - #[subsystem(no_dispatch, BitfieldDistributionMessage)] + #[subsystem(BitfieldDistributionMessage)] bitfield_distribution: BitfieldDistribution, #[subsystem(no_dispatch, ProvisionerMessage)] diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index a5a9002584f6..0c927775d0b8 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -240,10 +240,3 @@ impl } } } - -pub type AllSubsystemsSame = AllSubsystems< - T, T, T, T, T, - T, T, T, T, T, - T, T, T, T, T, - T, T, T, ->; diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 25305d1e26e4..666404ec8451 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -706,6 +706,7 @@ impl JobSubsystem { where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext::ToJob, Signal=OverseerSignal>, + ::Sender: Clone, Job: 'static + JobTrait + Send, Job::RunArgs: Clone + Sync, Job::ToJob: From, From 59584b9286cfe2bfbfb87e0be268e6792b638ce1 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 8 Jun 2021 14:25:36 +0200 Subject: [PATCH 063/161] fixins again --- .../proc-macro/src/impl_builder.rs | 11 ++++++++- .../proc-macro/src/impl_channels_out.rs | 5 ++++ .../proc-macro/src/impl_dispatch.rs | 10 ++++---- .../proc-macro/src/impl_message_wrapper.rs | 1 + .../overseer-gen/proc-macro/src/impl_misc.rs | 5 ++++ .../proc-macro/src/impl_overseer.rs | 16 ++++++++----- .../proc-macro/src/parse_struct.rs | 23 +++++++++++++++---- node/overseer/overseer-gen/src/lib.rs | 7 +++--- node/overseer/src/lib.rs | 10 ++++---- node/overseer/src/metrics.rs | 2 ++ 10 files changed, 67 insertions(+), 23 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 27d27703b931..deaf09b8e585 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -73,6 +73,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result #builder #builder_generics #builder_where_clause { @@ -80,8 +81,10 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result; + #[allow(missing_docs)] pub struct #builder #builder_generics { #( #subsystem_name : ::std::option::Option< #builder_generic_ty >, @@ -121,6 +124,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Self { self. #subsystem_name = Some( subsystem ); self @@ -128,12 +132,14 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Self { self. #baggage_name = Some( baggage ); self } )* + /// Complete the construction and create the overseer type. pub fn build(mut self) -> SubsystemResult<(#overseer_name #generics, #handler)> { let (events_tx, events_rx) = ::polkadot_overseer_gen::metered::channel::< @@ -178,7 +184,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result::new(); #( - // FIXME generate a builder pattern that ensures this + // TODO generate a builder pattern that ensures this let #subsystem_name = self. #subsystem_name .expect("All subsystem must exist with the builder pattern."); let unbounded_meter = #channel_name_unbounded_rx.meter().clone(); @@ -246,9 +252,11 @@ pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result(spawner: &mut S, name: &'static str, future: BoxFuture<'static, ()>); } + #[allow(missing_docs)] struct Regular; impl TaskKind for Regular { fn launch_task(spawner: &mut S, name: &'static str, future: BoxFuture<'static, ()>) { @@ -256,6 +264,7 @@ pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result(spawner: &mut S, name: &'static str, future: BoxFuture<'static, ()>) { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index ad719ac1392d..50ad8386247f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -13,9 +13,13 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result @@ -23,6 +27,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index f8a7545d8c1f..c27a77e321b6 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -35,17 +35,17 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { impl #message_wrapper { /// Generated dispatch iterator generator. pub fn dispatch_iter(extern_msg: #extern_network_ty) -> impl Iterator + Send { - None.into_iter() + None + .into_iter() + #( .chain( ::std::iter::once( extern_msg - // focuses on a `NetworkBridgeEvent< protocol_v1::*Message >` + // focuses on a `NetworkBridgeEvent< protocol_v1::* >` .focus() .ok() - // TODO introduce a `net_protocol_ty` to clean this up. - // TODO avoid hardcoding the `NetworkBridgeEvent` type. - .map(|event: NetworkBridgeEvent< protocol_v1:: #dispatchable >| { + .map(|event: NetworkBridgeEvent< _ >| { #message_wrapper :: #dispatchable ( // the inner type of the enum variant #dispatchable :: from( event ) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index e16086e6c80c..8f5e7ee55247 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -11,6 +11,7 @@ pub(crate) fn impl_message_wrapper_enum(info: &OverseerInfo) -> Result Result let signal = &info.extern_signal_ty; let ts = quote! { + /// Connector to send messages towards all subsystems, + /// while tracking the which signals where already received. #[derive(Debug, Clone)] pub struct #subsystem_sender_name { + /// Collection of channels to all subsystems. channels: ChannelsOut, + /// Systemwide tick for which signals were received by all subsystems. signals_received: SignalsReceived, } @@ -49,6 +53,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result /// [`Subsystem`]: trait.Subsystem.html /// [`SubsystemJob`]: trait.SubsystemJob.html #[derive(Debug)] + #[allow(missing_docs)] pub struct #subsystem_ctx_name{ signals: ::polkadot_overseer_gen::metered::MeteredReceiver< #signal >, messages: SubsystemIncomingMessages, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 6b7ca5d869a2..3774b7147dc7 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -8,8 +8,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result Result Result, )* - // Non-subsystem members. #( - #baggage_name: #baggage_ty, + /// A user specified addendum field. + #baggage_decl , )* /// Responsible for driving the subsystem futures. @@ -109,6 +109,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result ::polkadot_overseer_gen::SubsystemResult<()> { #( self. #subsystem_name .send_signal(signal.clone()).await; @@ -118,6 +119,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result ::polkadot_overseer_gen::SubsystemResult<()> { match message { #( @@ -142,6 +144,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result (&'a mut self) -> &'a mut S { &mut self.spawner } @@ -165,6 +168,7 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { + /// The instance. pub instance: std::option::Option< ::polkadot_overseer_gen::SubsystemInstance >, diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index f4338641e02f..81a26b84bb09 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -1,4 +1,6 @@ -use proc_macro2::Span; +use proc_macro2::{Span, TokenStream}; +use quote::quote; +use syn::Visibility; use std::collections::{hash_map::RandomState, HashSet}; use syn::parse::Parse; use syn::parse::ParseStream; @@ -90,6 +92,7 @@ pub(crate) struct BaggageField { pub(crate) field_name: Ident, pub(crate) field_ty: Path, pub(crate) generic: bool, + pub(crate) vis: Visibility, } #[derive(Clone, Debug)] @@ -133,7 +136,8 @@ impl OverseerInfo { } #[allow(dead_code)] - // FIXME use as the defaults + // TODO use as the defaults, if no subsystem is specified + // TODO or drop the type argument. pub(crate) fn subsystem_types(&self) -> Vec { self.subsystems.iter().map(|ssf| ssf.ty.clone()).collect::>() } @@ -144,6 +148,17 @@ impl OverseerInfo { pub(crate) fn baggage_types(&self) -> Vec { self.baggage.iter().map(|bag| bag.field_ty.clone()).collect::>() } + pub(crate) fn baggage_decl(&self) -> Vec { + self.baggage.iter().map(|bag| { + let BaggageField { + vis, + field_ty, + field_name, + .. + } = bag; + quote!{ #vis #field_name: #field_ty } + }).collect::>() + } /// Generic types per subsystem, in the form `Sub#N`. pub(crate) fn builder_generic_types(&self) -> Vec { @@ -183,7 +198,7 @@ impl OverseerGuts { let n = fields.named.len(); let mut subsystems = Vec::with_capacity(n); let mut baggage = Vec::with_capacity(n); - for (idx, Field { attrs, vis: _, ident, ty, .. }) in fields.named.into_iter().enumerate() { + for (idx, Field { attrs, vis, ident, ty, .. }) in fields.named.into_iter().enumerate() { let mut consumes = attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| { let span = attr.path.span(); attr.path.get_ident().filter(|ident| *ident == "subsystem").map(move |_ident| { @@ -225,7 +240,7 @@ impl OverseerGuts { let field_ty: Path = try_type_to_path(ty, ident.span())?; let generic: bool = if let Some(ident) = field_ty.get_ident() { baggage_generics.contains(ident) } else { false }; - baggage.push(BaggageField { field_name: ident, generic, field_ty }); + baggage.push(BaggageField { field_name: ident, generic, field_ty, vis }); } } Ok(Self { name, subsystems, baggage }) diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 28fd491ee3ce..2cdaf87e446c 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -159,7 +159,9 @@ impl MapSubsystem for F where F: Fn(T) -> U { } /// A wrapping type for messages. -// FIXME XXX elaborate the purpose of this. +/// +/// Includes a counter to synchronize signals with messages, +/// such that no inconsistent message sequences are prevented. #[derive(Debug)] pub struct MessagePacket { /// Signal level at the point of reception. @@ -187,8 +189,7 @@ pub type SubsystemIncomingMessages = self::stream::Select< >; -/// Meter to count the received signals in total. -// XXX FIXME is there a necessity for this? Seems redundant to `ReadOuts` +/// Watermark to track the received signals. #[derive(Debug, Default, Clone)] pub struct SignalsReceived(Arc); diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 917a39012fca..2af4beeaf913 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -251,7 +251,9 @@ pub enum ExternalRequest { /// Wait for the activation of a particular hash /// and be notified by means of the return channel. WaitForActivation { + /// The relay parent for which activation to wait for. hash: Hash, + /// Response channel to await on. response_channel: oneshot::Sender>, }, } @@ -361,16 +363,16 @@ pub struct Overseer { /// A set of leaves that `Overseer` starts working with. /// /// Drained at the beginning of `run` and never used again. - leaves: Vec<(Hash, BlockNumber)>, + pub leaves: Vec<(Hash, BlockNumber)>, /// The set of the "active leaves". - active_leaves: HashMap, + pub active_leaves: HashMap, /// An implementation for checking whether a header supports parachain consensus. - supports_parachains: SupportsParachains, + pub supports_parachains: SupportsParachains, /// Various Prometheus metrics. - metrics: Metrics, + pub metrics: Metrics, } impl Overseer diff --git a/node/overseer/src/metrics.rs b/node/overseer/src/metrics.rs index 7a07051dbb24..750d9d7e9800 100644 --- a/node/overseer/src/metrics.rs +++ b/node/overseer/src/metrics.rs @@ -33,6 +33,8 @@ struct MetricsInner { signals_received: prometheus::GaugeVec, } + +/// A sharable metrics type for usage with the overseer. #[derive(Default, Clone)] pub struct Metrics(Option); From f5abc5d9026da27eb605e2422e609d81251973b5 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 8 Jun 2021 15:49:45 +0200 Subject: [PATCH 064/161] silence warnings, fixins --- node/overseer/overseer-gen/proc-macro/src/impl_builder.rs | 7 ++++++- .../overseer-gen/proc-macro/src/impl_channels_out.rs | 2 +- node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs | 4 ++-- node/overseer/overseer-gen/src/lib.rs | 6 ------ node/overseer/src/lib.rs | 7 ++++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index deaf09b8e585..fe2d11145f7b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -308,7 +308,12 @@ pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result::launch_task(spawner, name, fut); - futures.push(Box::pin(rx.map(|e| { tracing::warn!(err = ?e, "dropping error"); Ok(()) }))); + futures.push(Box::pin( + rx.map(|e| { + tracing::warn!(err = ?e, "dropping error"); + Ok(()) + }) + )); let instance = Some(SubsystemInstance { meters: ::polkadot_overseer_gen::SubsystemMeters { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 50ad8386247f..9ea2b24fa97b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -72,7 +72,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result { self. #channel_name_unbounded .unbounded_send( - make_packet(signals_received, inner) + ::polkadot_overseer_gen::make_packet(signals_received, inner) ) .map_err(|e| e.into_send_error()) }, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 3774b7147dc7..2b3fd295475f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -86,7 +86,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result ::polkadot_overseer_gen::SubsystemResult<()> { #( - self. #subsystem_name .send_signal(signal.clone()).await; + let _ = self. #subsystem_name .send_signal(signal.clone()).await; )* let _ = signal; @@ -112,7 +112,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result ::polkadot_overseer_gen::SubsystemResult<()> { #( - self. #subsystem_name .send_signal(signal.clone()).await; + let _ = self. #subsystem_name .send_signal(signal.clone()).await; )* let _ = signal; diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 2cdaf87e446c..97efc040bab7 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -481,11 +481,5 @@ impl Future for Timeout where F: Future { } } - - - - - - #[cfg(test)] mod tests; diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 2af4beeaf913..11b46e91aeb0 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -116,7 +116,6 @@ use polkadot_overseer_gen::{ SubsystemContext, overlord, MessagePacket, - make_packet, SignalsReceived, FromOverseer, ToOverseer, @@ -355,10 +354,10 @@ pub struct Overseer { gossip_support: GossipSupport, /// External listeners waiting for a hash to be in the active-leave set. - activation_external_listeners: HashMap>>>, + pub activation_external_listeners: HashMap>>>, /// Stores the [`jaeger::Span`] per active leaf. - span_per_active_leaf: HashMap>, + pub span_per_active_leaf: HashMap>, /// A set of leaves that `Overseer` starts working with. /// @@ -525,6 +524,7 @@ where .active_leaves(Default::default()) .span_per_active_leaf(Default::default()) .activation_external_listeners(Default::default()) + .supports_parachains(supports_parachains) .metrics(metrics.clone()) .spawner(s) .build()?; @@ -607,6 +607,7 @@ where match msg { Event::MsgToSubsystem(msg) => { + self.metrics.on_message_relayed(); self.route_message(msg.into()).await?; } Event::Stop => { From 2fe03129ddc2f37866774babfd18e2b7347ac427 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 8 Jun 2021 16:26:16 +0200 Subject: [PATCH 065/161] fix/spellcheck: improve whitelist --- .config/lingua.dic | 32 ++++++++++++++++++++------------ .config/spellcheck.toml | 2 +- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.config/lingua.dic b/.config/lingua.dic index 7460cc55d315..189f2a9e2d5c 100644 --- a/.config/lingua.dic +++ b/.config/lingua.dic @@ -1,26 +1,33 @@ -90 +150 +adversary/SM annualised/MS Apache-2.0/M api/SM API/SM APIs +assignee/SM async BFT/M bitfield/MS blake2/MS blockchain/MS borked +BTC BTC/S CLI/MS +codec/SM config/MS crypto/MS customizable/B debian/M decodable/MS +dispatchable/SM +DMP/SM DOT/S ed25519 enum/MS ERC-20 +ETH ethereum/MS externality/MS extrinsic/MS @@ -30,13 +37,16 @@ GiB/S GPL/M GPLv3/M Handler/MS +HMP/SM https +include/BG inherent/MS initialize/RG instantiate/B intrinsic/MS intrinsics io +isolate/BG jaeger/MS js keccak256/M @@ -47,12 +57,15 @@ merkle/MS misbehavior/SM misbehaviors MIT/M +MQC/SM multivalidator/SM +NFT/SM oneshot/MS others' parablock/MS parachain/MS parameterize/D +picosecond/SM polkadot/MS pov-block/MS PoV/MS @@ -77,25 +90,20 @@ teleporters testnet/MS trie/MS trustless/Y +tuple/SM ubuntu/M UDP UI +unfinalize/B union/MSG unservable/B -unfinalize/B validator/SM +VMP/SM +VRF/SM w3f/MS wasm/M WND/S XCM/S XCMP/M -include/BG -isolate/BG -dispatchable/SM -VMP/SM -HMP/SM -DMP/SM -MQC/SM -VRF/SM -assignee/SM -adversary/SM \ No newline at end of file +instantiation/SM +NFA diff --git a/.config/spellcheck.toml b/.config/spellcheck.toml index 7601ef82a33e..0b66d54e2413 100644 --- a/.config/spellcheck.toml +++ b/.config/spellcheck.toml @@ -8,6 +8,6 @@ extra_dictionaries = ["lingua.dic"] # 5x # He tagged it as 'TheGreatestOfAllTimes' # Transforms' -transform_regex = ["^'([^\\s])'$", "^[0-9]+(?:\\.[0-9]*)?x$", "^(.*)'$", "^\\+$"] +transform_regex = ["^'([^\\s])'$", "^[0-9]+(?:\\.[0-9]*)?(x|%)$", "^(.*)'$", "^\\+$"] allow_concatenation = true allow_dashes = true From d3598f3232ef587b7036af7b35a3fa1af5796a3b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 8 Jun 2021 19:33:02 +0200 Subject: [PATCH 066/161] fallout (quite a bit) --- node/core/approval-voting/src/import.rs | 10 +++--- node/core/approval-voting/src/lib.rs | 12 +++---- node/core/av-store/src/lib.rs | 4 +-- node/core/chain-api/src/lib.rs | 6 ++-- node/core/runtime-api/Cargo.toml | 1 + node/core/runtime-api/src/lib.rs | 22 ++++++------- node/network/collator-protocol/Cargo.toml | 1 + .../collator-protocol/src/collator_side.rs | 33 ++++++++++++------- node/network/collator-protocol/src/error.rs | 7 +++- node/network/collator-protocol/src/lib.rs | 24 +++++++++++--- .../collator-protocol/src/validator_side.rs | 26 +++++++++++---- node/network/gossip-support/src/lib.rs | 14 ++++---- .../network/statement-distribution/src/lib.rs | 26 +++++++-------- .../overseer-gen/proc-macro/src/lib.rs | 1 + .../overseer-gen/proc-macro/src/parse_attr.rs | 17 ++++++++-- .../proc-macro/src/parse_struct.rs | 4 +++ node/overseer/src/lib.rs | 7 ++-- 17 files changed, 141 insertions(+), 74 deletions(-) diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index f8e96102d5ff..77bf768fda32 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -94,7 +94,7 @@ impl RollingSessionWindow { // This returns the entire ancestry up to the last finalized block's height or the last item we // have in the DB. This may be somewhat expensive when first recovering from major sync. async fn determine_new_blocks( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, db: &impl DBReader, head: Hash, header: &Header, @@ -196,7 +196,7 @@ async fn determine_new_blocks( struct SessionsUnavailable; async fn load_all_sessions( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, block_hash: Hash, start: SessionIndex, end_inclusive: SessionIndex, @@ -238,7 +238,7 @@ async fn load_all_sessions( // // some backwards drift in session index is acceptable. async fn cache_session_info_for_head( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, session_window: &mut RollingSessionWindow, block_hash: Hash, block_header: &Header, @@ -357,7 +357,7 @@ struct ImportedBlockInfoEnv<'a> { // Computes information about the imported block. Returns `None` if the info couldn't be extracted - // failure to communicate with overseer, async fn imported_block_info( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, env: ImportedBlockInfoEnv<'_>, block_hash: Hash, block_header: &Header, @@ -558,7 +558,7 @@ pub struct BlockImportedCandidates { /// /// It is the responsibility of the caller to schedule wakeups for each block. pub(crate) async fn handle_new_head( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, db_writer: &dyn KeyValueDB, db_config: DatabaseConfig, diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index da03766b1ff1..8aede5450e0c 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -679,7 +679,7 @@ async fn run( // returns `true` if any of the actions was a `Conclude` command. async fn handle_actions( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, wakeups: &mut Wakeups, db: &dyn KeyValueDB, @@ -878,7 +878,7 @@ fn distribution_messages_for_activation<'a>( // Handle an incoming signal from the overseer. Returns true if execution should conclude. async fn handle_from_overseer( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, metrics: &Metrics, db_writer: &dyn KeyValueDB, @@ -992,7 +992,7 @@ async fn handle_from_overseer( } async fn handle_background_request( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &State, metrics: &Metrics, request: BackgroundRequest, @@ -1022,7 +1022,7 @@ async fn handle_background_request( } async fn handle_approved_ancestor( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, db: &impl DBReader, target: Hash, lower_bound: BlockNumber, @@ -1868,7 +1868,7 @@ fn process_wakeup( // spawned. When the background work is no longer needed, the `AbortHandle` should be dropped // to cancel the background work and any requests it has spawned. async fn launch_approval( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, mut background_tx: mpsc::Sender, session_index: SessionIndex, candidate: &CandidateReceipt, @@ -2039,7 +2039,7 @@ async fn launch_approval( // Issue and import a local approval vote. Should only be invoked after approval checks // have been done. fn issue_approval( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &State, metrics: &Metrics, request: ApprovalVoteRequest, diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index 8ed7b8873043..5c6932414362 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -576,7 +576,7 @@ where } async fn process_block_activated( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, subsystem: &mut AvailabilityStoreSubsystem, activated: Hash, ) -> Result<(), Error> { @@ -769,7 +769,7 @@ macro_rules! peek_num { } async fn process_block_finalized( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, subsystem: &AvailabilityStoreSubsystem, finalized_hash: Hash, finalized_number: BlockNumber, diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index 782fcb19ed06..b1554d1f2df5 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -30,9 +30,11 @@ #![deny(unused_crate_dependencies, unused_results)] #![warn(missing_docs)] +use polkadot_overseer::gen::{ + FromOverseer, SpawnedSubsystem, Subsystem, SubsystemResult, SubsystemError, SubsystemContext, +} use polkadot_subsystem::{ - FromOverseer, OverseerSignal, - SpawnedSubsystem, Subsystem, SubsystemResult, SubsystemError, SubsystemContext, + OverseerSignal, messages::ChainApiMessage, }; use polkadot_node_subsystem_util::{ diff --git a/node/core/runtime-api/Cargo.toml b/node/core/runtime-api/Cargo.toml index 320c6240eb19..4b59d12364cc 100644 --- a/node/core/runtime-api/Cargo.toml +++ b/node/core/runtime-api/Cargo.toml @@ -17,6 +17,7 @@ sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = polkadot-primitives = { path = "../../../primitives" } polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } +polkadot-overseer = { path = "../../overseer" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } [dev-dependencies] diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index a2cd09a8f380..2c267f54cf98 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -22,14 +22,13 @@ #![deny(unused_crate_dependencies)] #![warn(missing_docs)] -use polkadot_subsystem::{ - Subsystem, SpawnedSubsystem, SubsystemResult, SubsystemContext, - FromOverseer, OverseerSignal, - messages::{ - RuntimeApiMessage, RuntimeApiRequest as Request, - }, - errors::RuntimeApiError, +use polkadot_overseer::gen::{ + Subsystem, SpawnedSubsystem, SubsystemResult, SubsystemError as OverseerError, SubsystemContext, + FromOverseer, }; +use polkadot_subsystem::{OverseerSignal, errors::RuntimeApiError, messages::{ + RuntimeApiMessage, RuntimeApiRequest as Request, + }}; use polkadot_node_subsystem_util::metrics::{self, prometheus}; use polkadot_primitives::v1::{Block, BlockId, Hash, ParachainHost}; @@ -82,10 +81,10 @@ impl RuntimeApiSubsystem { } } -impl Subsystem for RuntimeApiSubsystem where +impl Subsystem for RuntimeApiSubsystem where Client: ProvideRuntimeApi + Send + 'static + Sync, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Context: SubsystemContext + Context: SubsystemContext { fn start(self, ctx: Context) -> SpawnedSubsystem { SpawnedSubsystem { @@ -263,12 +262,13 @@ impl RuntimeApiSubsystem where } #[tracing::instrument(skip(ctx, subsystem), fields(subsystem = LOG_TARGET))] -async fn run( - mut ctx: impl SubsystemContext, +async fn run( + mut ctx: Context, mut subsystem: RuntimeApiSubsystem, ) -> SubsystemResult<()> where Client: ProvideRuntimeApi + Send + Sync + 'static, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, + Context: SubsystemContext { loop { select! { diff --git a/node/network/collator-protocol/Cargo.toml b/node/network/collator-protocol/Cargo.toml index 6d21989ec5f1..41592aea9ce5 100644 --- a/node/network/collator-protocol/Cargo.toml +++ b/node/network/collator-protocol/Cargo.toml @@ -19,6 +19,7 @@ polkadot-primitives = { path = "../../../primitives" } polkadot-node-network-protocol = { path = "../../network/protocol" } polkadot-node-primitives = { path = "../../primitives" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } +polkadot-overseer = { path = "../../overseer" } polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } [dev-dependencies] diff --git a/node/network/collator-protocol/src/collator_side.rs b/node/network/collator-protocol/src/collator_side.rs index 266054444571..4e060e8cb69c 100644 --- a/node/network/collator-protocol/src/collator_side.rs +++ b/node/network/collator-protocol/src/collator_side.rs @@ -21,9 +21,17 @@ use sp_core::Pair; use polkadot_primitives::v1::{AuthorityDiscoveryId, CandidateHash, CandidateReceipt, CollatorPair, CoreIndex, CoreState, GroupIndex, Hash, Id as ParaId}; use polkadot_subsystem::{ - FromOverseer, OverseerSignal, PerLeafSpan, SubsystemContext, jaeger, + PerLeafSpan, jaeger, messages::{ - AllMessages, CollatorProtocolMessage, NetworkBridgeEvent, NetworkBridgeMessage, + CollatorProtocolMessage, NetworkBridgeEvent, NetworkBridgeMessage, + }, +}; +use polkadot_overseer::{ + AllMessages, + OverseerSignal, + gen::{ + Subsystem, SpawnedSubsystem, SubsystemResult, SubsystemError as OverseerError, SubsystemContext, + FromOverseer, }, }; use polkadot_node_network_protocol::{ @@ -260,7 +268,7 @@ impl State { /// elsewhere in the node. #[tracing::instrument(level = "trace", skip(ctx, runtime, state, pov), fields(subsystem = LOG_TARGET))] async fn distribute_collation( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, runtime: &mut RuntimeInfo, state: &mut State, id: ParaId, @@ -359,7 +367,7 @@ async fn distribute_collation( /// and the total number of cores. #[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn determine_core( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, para_id: ParaId, relay_parent: Hash, ) -> Result> { @@ -389,7 +397,7 @@ struct GroupValidators { /// Returns [`ValidatorId`]'s of current and next group as determined based on the `relay_parent`. #[tracing::instrument(level = "trace", skip(ctx, runtime), fields(subsystem = LOG_TARGET))] async fn determine_our_validators( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, runtime: &mut RuntimeInfo, core_index: CoreIndex, cores: usize, @@ -469,7 +477,7 @@ async fn connect_to_validators( /// set as validator for our para at the given `relay_parent`. #[tracing::instrument(level = "trace", skip(ctx, state), fields(subsystem = LOG_TARGET))] async fn advertise_collation( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, relay_parent: Hash, peer: PeerId, @@ -671,7 +679,7 @@ async fn send_collation( /// A networking messages switch. #[tracing::instrument(level = "trace", skip(ctx, runtime, state), fields(subsystem = LOG_TARGET))] async fn handle_incoming_peer_message( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, runtime: &mut RuntimeInfo, state: &mut State, origin: PeerId, @@ -869,12 +877,15 @@ async fn handle_our_view_change( /// The collator protocol collator side main loop. #[tracing::instrument(skip(ctx, collator_pair, metrics), fields(subsystem = LOG_TARGET))] -pub(crate) async fn run( - mut ctx: impl SubsystemContext, +pub(crate) async fn run( + mut ctx: Context, local_peer_id: PeerId, collator_pair: CollatorPair, metrics: Metrics, -) -> Result<()> { +) -> Result<()> +where + Context: SubsystemContext, +{ use FromOverseer::*; use OverseerSignal::*; @@ -882,7 +893,7 @@ pub(crate) async fn run( let mut runtime = RuntimeInfo::new(None); loop { - let msg = ctx.recv().fuse().await.map_err(Fatal::SubsystemReceive)?; + let msg = ctx.recv().fuse().await.map_err(Fatal::Overseer)?; match msg { Communication { msg } => { log_error( diff --git a/node/network/collator-protocol/src/error.rs b/node/network/collator-protocol/src/error.rs index 37f8df0731b2..36716ee0b628 100644 --- a/node/network/collator-protocol/src/error.rs +++ b/node/network/collator-protocol/src/error.rs @@ -18,7 +18,8 @@ //! Error handling related code and Error/Result definitions. use polkadot_node_primitives::UncheckedSignedFullStatement; -use polkadot_subsystem::SubsystemError; +use polkadot_subsystem::errors::SubsystemError; +use polkadot_overseer::SubsystemError as OverseerError; use thiserror::Error; use polkadot_node_subsystem_util::{Fault, runtime, unwrap_non_fatal}; @@ -61,6 +62,10 @@ pub enum Fatal { #[error("Receiving message from overseer failed")] SubsystemReceive(#[source] SubsystemError), + /// Receiving subsystem message from overseer failed. + #[error("Receiving message from overseer failed")] + Overseer(#[source] OverseerError), + /// Errors coming from runtime::Runtime. #[error("Error while accessing runtime information")] Runtime(#[from] #[source] runtime::Fatal), diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 24ae2407d12c..26f9c03cb273 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -29,8 +29,22 @@ use sp_keystore::SyncCryptoStorePtr; use polkadot_node_network_protocol::{PeerId, UnifiedReputationChange as Rep}; use polkadot_primitives::v1::CollatorPair; use polkadot_subsystem::{ - messages::{AllMessages, CollatorProtocolMessage, NetworkBridgeMessage}, - SpawnedSubsystem, Subsystem, SubsystemContext, SubsystemError, + messages::{CollatorProtocolMessage, NetworkBridgeMessage}, +}; + +use polkadot_overseer::{ + AllMessages, + OverseerSignal, + gen::{ + Subsystem, SpawnedSubsystem, SubsystemResult, SubsystemError as OverseerError, SubsystemContext, + FromOverseer, + }, +}; +use polkadot_subsystem::{ + errors::{RuntimeApiError, SubsystemError}, + messages::{ + RuntimeApiMessage, RuntimeApiRequest as Request, + }, }; mod error; @@ -93,7 +107,7 @@ impl CollatorProtocolSubsystem { #[tracing::instrument(skip(self, ctx), fields(subsystem = LOG_TARGET))] async fn run(self, ctx: Context) -> Result<()> where - Context: SubsystemContext, + Context: SubsystemContext, { match self.protocol_side { ProtocolSide::Validator { keystore, eviction_policy, metrics } => validator_side::run( @@ -112,9 +126,9 @@ impl CollatorProtocolSubsystem { } } -impl Subsystem for CollatorProtocolSubsystem +impl Subsystem for CollatorProtocolSubsystem where - Context: SubsystemContext + Sync + Send, + Context: SubsystemContext + Sync + Send, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self diff --git a/node/network/collator-protocol/src/validator_side.rs b/node/network/collator-protocol/src/validator_side.rs index e0a378a20e13..4ab9b06cbbc3 100644 --- a/node/network/collator-protocol/src/validator_side.rs +++ b/node/network/collator-protocol/src/validator_side.rs @@ -37,14 +37,26 @@ use polkadot_node_network_protocol::{ }; use polkadot_node_primitives::{SignedFullStatement, Statement, PoV}; use polkadot_node_subsystem_util::metrics::{self, prometheus}; +use polkadot_node_subsystem_util::SubsystemSender; use polkadot_primitives::v1::{CandidateReceipt, CollatorId, Hash, Id as ParaId}; +use polkadot_overseer::{ + AllMessages, + gen::{ + Subsystem, SpawnedSubsystem, SubsystemResult, SubsystemError as OverseerError, SubsystemContext, + FromOverseer, + }, +}; use polkadot_subsystem::{ + errors::SubsystemError, + OverseerSignal, errors::RuntimeApiError, messages::{ + RuntimeApiMessage, RuntimeApiRequest as Request, + }, jaeger, messages::{ - AllMessages, CandidateSelectionMessage, CollatorProtocolMessage, IfDisconnected, + CandidateSelectionMessage, CollatorProtocolMessage, IfDisconnected, NetworkBridgeEvent, NetworkBridgeMessage, }, - FromOverseer, OverseerSignal, PerLeafSpan, SubsystemContext, SubsystemSender, + PerLeafSpan, }; use crate::error::Fatal; @@ -312,7 +324,7 @@ struct ActiveParas { impl ActiveParas { async fn assign_incoming( &mut self, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, keystore: &SyncCryptoStorePtr, new_relay_parents: impl IntoIterator, ) { @@ -477,7 +489,7 @@ fn collator_peer_id( ) } -async fn disconnect_peer(ctx: &mut impl SubsystemContext, peer_id: PeerId) { +async fn disconnect_peer(ctx: &mut impl SubsystemContext, peer_id: PeerId) { ctx.send_message( NetworkBridgeMessage::DisconnectPeer(peer_id, PeerSet::Collation).into() ).await @@ -813,7 +825,7 @@ async fn remove_relay_parent( /// Our view has changed. #[tracing::instrument(level = "trace", skip(ctx, state, keystore), fields(subsystem = LOG_TARGET))] async fn handle_our_view_change( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, keystore: &SyncCryptoStorePtr, view: OurView, @@ -983,7 +995,7 @@ pub(crate) async fn run( eviction_policy: crate::CollatorEvictionPolicy, metrics: Metrics, ) -> Result<()> - where Context: SubsystemContext + where Context: SubsystemContext { use FromOverseer::*; use OverseerSignal::*; @@ -1060,7 +1072,7 @@ pub(crate) async fn run( // earliest possible point. This does not yet clean up any metadata, as that will be done upon // receipt of the `PeerDisconnected` event. async fn disconnect_inactive_peers( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, eviction_policy: &crate::CollatorEvictionPolicy, peers: &HashMap, ) { diff --git a/node/network/gossip-support/src/lib.rs b/node/network/gossip-support/src/lib.rs index 213f30e6d0cd..a52e24c24eff 100644 --- a/node/network/gossip-support/src/lib.rs +++ b/node/network/gossip-support/src/lib.rs @@ -25,10 +25,12 @@ use std::time::{Duration, Instant}; use futures::{channel::oneshot, FutureExt as _}; use polkadot_node_subsystem::{ messages::{ - AllMessages, GossipSupportMessage, NetworkBridgeMessage, + GossipSupportMessage, NetworkBridgeMessage, }, - ActiveLeavesUpdate, FromOverseer, OverseerSignal, - Subsystem, SpawnedSubsystem, SubsystemContext, + ActiveLeavesUpdate, OverseerSignal, +}; +use polkadot_overseer::gen::{ + FromOverseer, AllMessages, Subsystem, SpawnedSubsystem, SubsystemContext, }; use polkadot_node_subsystem_util as util; use polkadot_primitives::v1::{ @@ -113,7 +115,7 @@ impl GossipSupport { } async fn determine_relevant_authorities( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, relay_parent: Hash, ) -> Result, util::Error> { let authorities = util::request_authorities(relay_parent, ctx.sender()).await.await??; @@ -142,7 +144,7 @@ async fn ensure_i_am_an_authority( /// A helper function for making a `ConnectToValidators` request. pub async fn connect_to_authorities( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, validator_ids: Vec, peer_set: PeerSet, ) -> oneshot::Receiver { @@ -163,7 +165,7 @@ impl State { /// and issue a connection request. async fn handle_active_leaves( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, keystore: &SyncCryptoStorePtr, leaves: impl Iterator, ) -> Result<(), util::Error> { diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index 6c8315d775cc..8570d0a5d4df 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -848,7 +848,7 @@ fn check_statement_signature( async fn circulate_statement_and_dependents( peers: &mut HashMap, active_heads: &mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, relay_parent: Hash, statement: SignedFullStatement, priority_peers: Vec, @@ -948,7 +948,7 @@ fn is_statement_large(statement: &SignedFullStatement) -> bool { #[tracing::instrument(level = "trace", skip(peers, ctx), fields(subsystem = LOG_TARGET))] async fn circulate_statement<'a>( peers: &mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, relay_parent: Hash, stored: StoredStatement<'a>, mut priority_peers: Vec, @@ -1026,7 +1026,7 @@ async fn circulate_statement<'a>( async fn send_statements_about( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, relay_parent: Hash, candidate_hash: CandidateHash, active_head: &ActiveHeadData, @@ -1064,7 +1064,7 @@ async fn send_statements_about( async fn send_statements( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, relay_parent: Hash, active_head: &ActiveHeadData, metrics: &Metrics, @@ -1096,7 +1096,7 @@ async fn send_statements( } async fn report_peer( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, peer: PeerId, rep: Rep, ) { @@ -1116,7 +1116,7 @@ async fn retrieve_statement_from_message<'a>( peer: PeerId, message: protocol_v1::StatementDistributionMessage, active_head: &'a mut ActiveHeadData, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, req_sender: &mpsc::Sender, metrics: &Metrics, ) -> Option { @@ -1218,7 +1218,7 @@ async fn launch_request( meta: StatementMetadata, peer: PeerId, req_sender: mpsc::Sender, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, metrics: &Metrics, ) -> Option { @@ -1256,7 +1256,7 @@ async fn handle_incoming_message_and_circulate<'a>( peer: PeerId, peers: &mut HashMap, active_heads: &'a mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, @@ -1304,7 +1304,7 @@ async fn handle_incoming_message<'a>( peer: PeerId, peer_data: &mut PeerData, active_heads: &'a mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, @@ -1454,7 +1454,7 @@ async fn handle_incoming_message<'a>( async fn update_peer_view_and_send_unlocked( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, active_heads: &HashMap, new_view: View, metrics: &Metrics, @@ -1489,7 +1489,7 @@ async fn handle_network_update( peers: &mut HashMap, authorities: &mut HashMap, active_heads: &mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, req_sender: &mpsc::Sender, update: NetworkBridgeEvent, metrics: &Metrics, @@ -1670,7 +1670,7 @@ impl StatementDistribution { async fn handle_requester_message( &self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, peers: &mut HashMap, active_heads: &mut HashMap, req_sender: &mpsc::Sender, @@ -1790,7 +1790,7 @@ impl StatementDistribution { async fn handle_subsystem_message( &self, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, runtime: &mut RuntimeInfo, peers: &mut HashMap, authorities: &mut HashMap, diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 67c8ceaeea8c..d95393e52ae6 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -60,6 +60,7 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< extern_signal_ty: args.extern_signal_ty, extern_error_ty: args.extern_error_ty, extern_network_ty: args.extern_network_ty, + outgoing_ty: args.outgoing_ty, }; let mut additive = impl_overseer_struct(&info)?; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs index 6ddb75dbe26c..7c2ba863e1e0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs @@ -17,6 +17,7 @@ enum AttrItem { ExternNetworkType(Path), ExternOverseerSignalType(Path), ExternErrorType(Path), + OutgoingType(Path), MessageWrapperName(Ident), SignalChannelCapacity(LitInt), MessageChannelCapacity(LitInt), @@ -29,6 +30,7 @@ impl Spanned for AttrItem { AttrItem::ExternNetworkType(x) => x.span(), AttrItem::ExternOverseerSignalType(x) => x.span(), AttrItem::ExternErrorType(x) => x.span(), + AttrItem::OutgoingType(x) => x.span(), AttrItem::MessageWrapperName(x) => x.span(), AttrItem::SignalChannelCapacity(x) => x.span(), AttrItem::MessageChannelCapacity(x) => x.span(), @@ -37,9 +39,10 @@ impl Spanned for AttrItem { } const TAG_EXT_EVENT_TY: &str = "event"; -const TAG_EXT_NETWORK_TY: &str = "network"; const TAG_EXT_SIGNAL_TY: &str = "signal"; const TAG_EXT_ERROR_TY: &str = "error"; +const TAG_EXT_NETWORK_TY: &str = "network"; +const TAG_OUTGOING_TY: &str = "out"; const TAG_GEN_TY: &str = "gen"; const TAG_SIGNAL_CAPACITY: &str = "signal_capacity"; const TAG_MESSAGE_CAPACITY: &str = "message_capacity"; @@ -48,9 +51,10 @@ impl AttrItem { fn key(&self) -> &'static str { match self { AttrItem::ExternEventType(_) => TAG_EXT_EVENT_TY, - AttrItem::ExternNetworkType(_) => TAG_EXT_NETWORK_TY, AttrItem::ExternOverseerSignalType(_) => TAG_EXT_SIGNAL_TY, AttrItem::ExternErrorType(_) => TAG_EXT_ERROR_TY, + AttrItem::ExternNetworkType(_) => TAG_EXT_NETWORK_TY, + AttrItem::OutgoingType(_) => TAG_OUTGOING_TY, AttrItem::MessageWrapperName(_) => TAG_GEN_TY, AttrItem::SignalChannelCapacity(_) => TAG_SIGNAL_CAPACITY, AttrItem::MessageChannelCapacity(_) => TAG_MESSAGE_CAPACITY, @@ -69,6 +73,9 @@ impl Parse for AttrItem { } else if key == TAG_EXT_NETWORK_TY { let path = input.parse::()?; AttrItem::ExternNetworkType(path) + } else if key == TAG_OUTGOING_TY { + let path = input.parse::()?; + AttrItem::OutgoingType(path) } else if key == TAG_EXT_EVENT_TY { let path = input.parse::()?; AttrItem::ExternEventType(path) @@ -101,6 +108,7 @@ pub(crate) struct AttrArgs { /// but is not part of the band of subsystems, it's a mere proxy /// to another entity that consumes/produces messages. pub(crate) extern_network_ty: Option, + pub(crate) outgoing_ty: Option, pub(crate) signal_channel_capacity: usize, pub(crate) message_channel_capacity: usize, } @@ -163,6 +171,10 @@ impl Parse for AttrArgs { .remove(TAG_EXT_NETWORK_TY) .map(|x| if let AttrItem::ExternNetworkType(x) = x { x.clone() } else { unreachable!() }); + let outgoing_ty = unique + .remove(TAG_OUTGOING_TY) + .map(|x| if let AttrItem::OutgoingType(x) = x { x.clone() } else { unreachable!() }); + let message_wrapper = unique .remove(TAG_GEN_TY) .map(|x| if let AttrItem::MessageWrapperName(x) = x { x.clone() } else { unreachable!() }) @@ -182,6 +194,7 @@ impl Parse for AttrArgs { extern_signal_ty, extern_error_ty, extern_network_ty, + outgoing_ty, message_wrapper, }) } diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 81a26b84bb09..ab7763a15d82 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -122,6 +122,10 @@ pub(crate) struct OverseerInfo { /// Incoming event type from an external entity, commonly from the network. pub(crate) extern_network_ty: Option, + /// Type of messages that are sent to an external subsystem. + /// Merely here to be included during generation of `message_wrapper` type. + pub(crate) outgoing_ty: Option, + /// Incoming event type from the outer world, commonly from the network. pub(crate) extern_error_ty: Path, } diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index d99dfb6a32bb..fe86a41f278b 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -72,7 +72,6 @@ use futures::{ future::BoxFuture, Future, FutureExt, StreamExt, }; -use futures_timer::Delay; use lru::LruCache; use polkadot_primitives::v1::{Block, BlockId,BlockNumber, Hash, ParachainHost}; @@ -95,7 +94,8 @@ use polkadot_node_subsystem::messages::{ pub use polkadot_node_subsystem::{ OverseerSignal, errors::{SubsystemResult, SubsystemError,}, - ActiveLeavesUpdate, ActivatedLeaf, jaeger, + ActiveLeavesUpdate, ActivatedLeaf, LeafStatus, + jaeger, }; /// TODO legacy, to be deleted, left for easier integration @@ -123,6 +123,7 @@ use polkadot_overseer_gen::{ ToOverseer, MapSubsystem, }; +pub use polkadot_overseer_gen as gen; // Target for logs. // const LOG_TARGET: &'static str = "parachain::overseer"; @@ -373,7 +374,7 @@ pub struct Overseer { pub supports_parachains: SupportsParachains, /// An LRU cache for keeping track of relay-chain heads that have already been seen. - known_leaves: LruCache, + pub known_leaves: LruCache, /// Various Prometheus metrics. pub metrics: Metrics, From 78dcd08c46f651b0e6fe41ed57df14a31f046899 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 9 Jun 2021 14:29:12 +0200 Subject: [PATCH 067/161] chain-api --- Cargo.lock | 14 +++-- node/collation-generation/src/lib.rs | 2 +- node/core/backing/src/lib.rs | 30 +++++----- node/core/bitfield-signing/src/lib.rs | 6 +- node/core/candidate-selection/src/lib.rs | 14 ++--- node/core/chain-api/Cargo.toml | 1 + node/core/chain-api/src/lib.rs | 15 ++--- node/core/provisioner/src/lib.rs | 10 ++-- node/network/availability-recovery/src/lib.rs | 6 +- node/network/bridge/Cargo.toml | 1 + node/network/bridge/src/lib.rs | 59 +++++++++++-------- node/network/bridge/src/multiplexer.rs | 9 +-- node/network/bridge/src/network.rs | 2 +- .../proc-macro/src/impl_message_wrapper.rs | 48 +++++++++++++-- .../overseer-gen/proc-macro/src/impl_misc.rs | 42 +++++++++++-- .../overseer-gen/proc-macro/src/parse_attr.rs | 2 +- node/overseer/overseer-gen/src/lib.rs | 37 ++++++++---- node/overseer/src/lib.rs | 13 +++- node/overseer/src/subsystems.rs | 1 - node/subsystem-test-helpers/src/lib.rs | 2 +- node/subsystem-util/src/lib.rs | 2 +- node/subsystem-util/src/runtime/mod.rs | 14 ++--- 22 files changed, 222 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2f607307806..efa091c7f705 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "Inflector" version = "0.11.4" @@ -489,7 +491,7 @@ checksum = "6736e2428df2ca2848d846c43e88745121a6654696e349ce0054a420815a7409" [[package]] name = "beefy-gadget" version = "0.1.0" -source = "git+https://github.com/paritytech/grandpa-bridge-gadget#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" +source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" dependencies = [ "beefy-primitives", "futures 0.3.14", @@ -517,7 +519,7 @@ dependencies = [ [[package]] name = "beefy-gadget-rpc" version = "0.1.0" -source = "git+https://github.com/paritytech/grandpa-bridge-gadget#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" +source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" dependencies = [ "beefy-gadget", "beefy-primitives", @@ -538,7 +540,7 @@ dependencies = [ [[package]] name = "beefy-primitives" version = "0.1.0" -source = "git+https://github.com/paritytech/grandpa-bridge-gadget#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" +source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" dependencies = [ "parity-scale-codec", "sp-api", @@ -4651,7 +4653,7 @@ dependencies = [ [[package]] name = "pallet-beefy" version = "0.1.0" -source = "git+https://github.com/paritytech/grandpa-bridge-gadget#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" +source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" dependencies = [ "beefy-primitives", "frame-support", @@ -5858,6 +5860,7 @@ dependencies = [ "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", + "polkadot-overseer", "polkadot-primitives", "sp-core", "sp-keyring", @@ -5924,6 +5927,7 @@ dependencies = [ "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", + "polkadot-overseer", "polkadot-primitives", "sc-authority-discovery", "sc-network", @@ -6102,6 +6106,7 @@ dependencies = [ "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", + "polkadot-overseer", "polkadot-primitives", "sp-blockchain", "sp-core", @@ -6184,6 +6189,7 @@ dependencies = [ "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", + "polkadot-overseer", "polkadot-primitives", "sp-api", "sp-authority-discovery", diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index e8f5f0581cf4..65b131960184 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -80,7 +80,7 @@ impl CollationGenerationSubsystem { #[tracing::instrument(skip(self, ctx), fields(subsystem = LOG_TARGET))] async fn run(mut self, mut ctx: Context) where - Context: SubsystemContext, + Context: SubsystemContext, { // when we activate new leaves, we spawn a bunch of sub-tasks, each of which is // expected to generate precisely one message. We don't want to block the main loop diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index 61e89f96d5dc..f2913b157a80 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -293,7 +293,7 @@ fn table_attested_to_backed( } async fn store_available_data( - sender: &mut JobSender>, + sender: &mut JobSender>, id: Option, n_validators: u32, candidate_hash: CandidateHash, @@ -369,7 +369,7 @@ async fn make_pov_available( } async fn request_pov( - sender: &mut JobSender>, + sender: &mut JobSender>, relay_parent: Hash, from_validator: ValidatorIndex, candidate_hash: CandidateHash, @@ -390,7 +390,7 @@ async fn request_pov( } async fn request_candidate_validation( - sender: &mut JobSender>, + sender: &mut JobSender>, candidate: CandidateDescriptor, pov: Arc, ) -> Result { @@ -428,7 +428,7 @@ struct BackgroundValidationParams { async fn validate_and_make_available( params: BackgroundValidationParams< - impl SubsystemSender , + impl SubsystemSender, impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Sync, > ) -> Result<(), Error> { @@ -541,7 +541,7 @@ impl CandidateBackingJob { /// Run asynchronously. async fn run_loop( mut self, - mut sender: JobSender>, + mut sender: JobSender>, mut rx_to: mpsc::Receiver, span: PerLeafSpan, ) -> Result<(), Error> { @@ -574,7 +574,7 @@ impl CandidateBackingJob { async fn handle_validated_candidate_command( &mut self, root_span: &jaeger::Span, - sender: &mut JobSender>, + sender: &mut JobSender>, command: ValidatedCandidateCommand, ) -> Result<(), Error> { let candidate_hash = command.candidate_hash(); @@ -650,9 +650,9 @@ impl CandidateBackingJob { #[tracing::instrument(level = "trace", skip(self, sender, params), fields(subsystem = LOG_TARGET))] async fn background_validate_and_make_available( &mut self, - sender: &mut JobSender>, + sender: &mut JobSender>, params: BackgroundValidationParams< - impl SubsystemSender , + impl SubsystemSender, impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Send + 'static + Sync >, ) -> Result<(), Error> { @@ -676,7 +676,7 @@ impl CandidateBackingJob { &mut self, parent_span: &jaeger::Span, root_span: &jaeger::Span, - sender: &mut JobSender>, + sender: &mut JobSender>, candidate: &CandidateReceipt, pov: Arc, ) -> Result<(), Error> { @@ -727,7 +727,7 @@ impl CandidateBackingJob { async fn sign_import_and_distribute_statement( &mut self, - sender: &mut JobSender>, + sender: &mut JobSender>, statement: Statement, root_span: &jaeger::Span, ) -> Result, Error> { @@ -744,7 +744,7 @@ impl CandidateBackingJob { /// Check if there have happened any new misbehaviors and issue necessary messages. #[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))] - async fn issue_new_misbehaviors(&mut self, sender: &mut JobSender>) { + async fn issue_new_misbehaviors(&mut self, sender: &mut JobSender>) { // collect the misbehaviors to avoid double mutable self borrow issues let misbehaviors: Vec<_> = self.table.drain_misbehaviors().collect(); for (validator_id, report) in misbehaviors { @@ -761,7 +761,7 @@ impl CandidateBackingJob { #[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))] async fn import_statement( &mut self, - sender: &mut JobSender>, + sender: &mut JobSender>, statement: &SignedFullStatement, root_span: &jaeger::Span, ) -> Result, Error> { @@ -832,7 +832,7 @@ impl CandidateBackingJob { async fn process_msg( &mut self, root_span: &jaeger::Span, - sender: &mut JobSender>, + sender: &mut JobSender>, msg: CandidateBackingMessage, ) -> Result<(), Error> { match msg { @@ -898,7 +898,7 @@ impl CandidateBackingJob { #[tracing::instrument(level = "trace", skip(self, sender, attesting, span), fields(subsystem = LOG_TARGET))] async fn kick_off_validation_work( &mut self, - sender: &mut JobSender>, + sender: &mut JobSender>, attesting: AttestingData, span: Option, ) -> Result<(), Error> { @@ -955,7 +955,7 @@ impl CandidateBackingJob { async fn maybe_validate_and_import( &mut self, root_span: &jaeger::Span, - sender: &mut JobSender>, + sender: &mut JobSender>, statement: SignedFullStatement, ) -> Result<(), Error> { if let Some(summary) = self.import_statement(sender, &statement, root_span).await? { diff --git a/node/core/bitfield-signing/src/lib.rs b/node/core/bitfield-signing/src/lib.rs index 3ddc23c1db49..adbb42fb1a5e 100644 --- a/node/core/bitfield-signing/src/lib.rs +++ b/node/core/bitfield-signing/src/lib.rs @@ -74,7 +74,7 @@ pub enum Error { async fn get_core_availability( core: &CoreState, validator_idx: ValidatorIndex, - sender: &Mutex<&mut impl SubsystemSender >, + sender: &Mutex<&mut impl SubsystemSender>, span: &jaeger::Span, ) -> Result { if let &CoreState::Occupied(ref core) = core { @@ -112,7 +112,7 @@ async fn get_core_availability( /// delegates to the v1 runtime API async fn get_availability_cores( relay_parent: Hash, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, ) -> Result, Error> { let (tx, rx) = oneshot::channel(); sender @@ -137,7 +137,7 @@ async fn construct_availability_bitfield( relay_parent: Hash, span: &jaeger::Span, validator_idx: ValidatorIndex, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, ) -> Result { // get the set of availability cores from the runtime let availability_cores = { diff --git a/node/core/candidate-selection/src/lib.rs b/node/core/candidate-selection/src/lib.rs index 6101b6dcefdd..fb39779be132 100644 --- a/node/core/candidate-selection/src/lib.rs +++ b/node/core/candidate-selection/src/lib.rs @@ -224,7 +224,7 @@ impl CandidateSelectionJob { async fn run_loop( &mut self, span: &jaeger::Span, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, ) -> Result<(), Error> { let span = span.child("run-loop") .with_stage(jaeger::Stage::CandidateSelection); @@ -266,7 +266,7 @@ impl CandidateSelectionJob { #[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))] async fn handle_collation( &mut self, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, relay_parent: Hash, para_id: ParaId, collator_id: CollatorId, @@ -317,7 +317,7 @@ impl CandidateSelectionJob { #[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))] async fn handle_invalid( &mut self, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, candidate_receipt: CandidateReceipt, ) { let _timer = self.metrics.time_handle_invalid(); @@ -344,7 +344,7 @@ impl CandidateSelectionJob { async fn handle_seconded( &mut self, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, relay_parent: Hash, statement: SignedFullStatement, ) { @@ -386,7 +386,7 @@ async fn get_collation( relay_parent: Hash, para_id: ParaId, collator_id: CollatorId, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, ) -> Result<(CandidateReceipt, PoV), Error> { let (tx, rx) = oneshot::channel(); sender @@ -405,7 +405,7 @@ async fn second_candidate( relay_parent: Hash, candidate_receipt: CandidateReceipt, pov: PoV, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, metrics: &Metrics, ) { sender @@ -421,7 +421,7 @@ async fn second_candidate( async fn forward_invalidity_note( received_from: &CollatorId, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, ) { sender .send_message(CollatorProtocolMessage::ReportCollator(received_from.clone()).into()) diff --git a/node/core/chain-api/Cargo.toml b/node/core/chain-api/Cargo.toml index 0ea61dd59108..c146eab4222b 100644 --- a/node/core/chain-api/Cargo.toml +++ b/node/core/chain-api/Cargo.toml @@ -9,6 +9,7 @@ futures = "0.3.12" tracing = "0.1.26" sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" } polkadot-primitives = { path = "../../../primitives" } +polkadot-overseer = { path = "../../overseer" } polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index b1554d1f2df5..ae67222e5a1e 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -31,10 +31,11 @@ #![warn(missing_docs)] use polkadot_overseer::gen::{ - FromOverseer, SpawnedSubsystem, Subsystem, SubsystemResult, SubsystemError, SubsystemContext, -} + FromOverseer, SpawnedSubsystem, Subsystem, SubsystemContext, +}; use polkadot_subsystem::{ - OverseerSignal, + errors::{SubsystemResult, SubsystemError}, + OverseerSignal, messages::ChainApiMessage, }; use polkadot_node_subsystem_util::{ @@ -64,11 +65,11 @@ impl ChainApiSubsystem { } } -impl Subsystem for ChainApiSubsystem where +impl Subsystem for ChainApiSubsystem where Client: HeaderBackend + 'static, - Context: SubsystemContext + Context: SubsystemContext, { - fn start(self, ctx: Context) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run(ctx, self) .map_err(|e| SubsystemError::with_origin("chain-api", e)) .boxed(); @@ -81,7 +82,7 @@ impl Subsystem for ChainApiSubsystem where #[tracing::instrument(skip(ctx, subsystem), fields(subsystem = LOG_TARGET))] async fn run( - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, subsystem: ChainApiSubsystem, ) -> SubsystemResult<()> where diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index 8eeb54982e21..36ef5ffdd7c8 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -181,7 +181,7 @@ impl ProvisioningJob { async fn run_loop( mut self, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, span: PerLeafSpan, ) -> Result<(), Error> { use ProvisionerMessage::{ @@ -223,7 +223,7 @@ impl ProvisioningJob { async fn send_inherent_data( &mut self, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, return_senders: Vec>, ) { if let Err(err) = send_inherent_data( @@ -283,7 +283,7 @@ async fn send_inherent_data( bitfields: &[SignedAvailabilityBitfield], candidates: &[CandidateReceipt], return_senders: Vec>, - from_job: &mut impl SubsystemSender , + from_job: &mut impl SubsystemSender, ) -> Result<(), Error> { let availability_cores = request_availability_cores(relay_parent, from_job) .await @@ -359,7 +359,7 @@ async fn select_candidates( bitfields: &[SignedAvailabilityBitfield], candidates: &[CandidateReceipt], relay_parent: Hash, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, ) -> Result, Error> { let block_number = get_block_number_under_construction(relay_parent, sender).await?; @@ -478,7 +478,7 @@ async fn select_candidates( #[tracing::instrument(level = "trace", skip(sender), fields(subsystem = LOG_TARGET))] async fn get_block_number_under_construction( relay_parent: Hash, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, ) -> Result { let (tx, rx) = oneshot::channel(); sender diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index d6abdb4b6cf6..0f83e319d917 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -132,7 +132,7 @@ impl RequestFromBackersPhase { async fn run( &mut self, params: &InteractionParams, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, ) -> Result { tracing::trace!( target: LOG_TARGET, @@ -217,7 +217,7 @@ impl RequestChunksPhase { async fn launch_parallel_requests( &mut self, params: &InteractionParams, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, ) { let max_requests = std::cmp::min(N_PARALLEL, params.threshold); while self.requesting_chunks.len() < max_requests { @@ -329,7 +329,7 @@ impl RequestChunksPhase { async fn run( &mut self, params: &InteractionParams, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, ) -> Result { // First query the store for any chunks we've got. { diff --git a/node/network/bridge/Cargo.toml b/node/network/bridge/Cargo.toml index 7737a2c2f115..eb69ff34fe9d 100644 --- a/node/network/bridge/Cargo.toml +++ b/node/network/bridge/Cargo.toml @@ -14,6 +14,7 @@ sc-authority-discovery = { git = "https://github.com/paritytech/substrate", bran sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-consensus = { git = "https://github.com/paritytech/substrate", branch = "master" } polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } +polkadot-overseer = { path = "../../overseer" } polkadot-node-network-protocol = { path = "../protocol" } polkadot-node-subsystem-util = { path = "../../subsystem-util"} strum = "0.20.0" diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index a869c8a5718d..c74cf4c807d2 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -24,17 +24,24 @@ use parity_scale_codec::{Encode, Decode}; use parking_lot::Mutex; use futures::prelude::*; use futures::stream::BoxStream; +use polkadot_overseer::Overseer; use sc_network::Event as NetworkEvent; use sp_consensus::SyncOracle; +use polkadot_overseer::{AllMessages, OverseerSignal}; +use polkadot_overseer::gen::{ + FromOverseer, + SpawnedSubsystem, + Subsystem, SubsystemContext, SubsystemError as OverseerError, + SubsystemResult as OverseerResult, SubsystemSender, +}; use polkadot_subsystem::{ - ActivatedLeaf, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, - Subsystem, SubsystemContext, SubsystemError, SubsystemResult, SubsystemSender, + errors::{SubsystemError, SubsystemResult}, + ActivatedLeaf, ActiveLeavesUpdate, messages::StatementDistributionMessage }; use polkadot_subsystem::messages::{ - NetworkBridgeMessage, AllMessages, - CollatorProtocolMessage, NetworkBridgeEvent, + NetworkBridgeMessage, CollatorProtocolMessage, NetworkBridgeEvent, }; use polkadot_primitives::v1::{Hash, BlockNumber}; use polkadot_node_network_protocol::{ @@ -291,13 +298,13 @@ impl NetworkBridge { } } -impl Subsystem for NetworkBridge +impl Subsystem for NetworkBridge where Net: Network + Sync, AD: validator_discovery::AuthorityDiscovery, - Context: SubsystemContext, + Context: SubsystemContext, { - fn start(mut self, ctx: Context) -> SpawnedSubsystem { + fn start(mut self, ctx: Context) -> SpawnedSubsystem { // The stream of networking events has to be created at initialization, otherwise the // networking might open connections before the stream of events has been grabbed. let network_stream = self.network_service.event_stream(); @@ -324,7 +331,7 @@ struct PeerData { #[derive(Debug)] enum UnexpectedAbort { /// Received error from overseer: - SubsystemError(polkadot_subsystem::SubsystemError), + SubsystemError(SubsystemError), /// The stream of incoming events concluded. EventStreamConcluded, /// The stream of incoming requests concluded. @@ -337,6 +344,12 @@ impl From for UnexpectedAbort { } } +impl From for UnexpectedAbort { + fn from(e: OverseerError) -> Self { + UnexpectedAbort::SubsystemError(SubsystemError::from(e)) + } +} + #[derive(Default, Clone)] struct Shared(Arc>); @@ -361,7 +374,7 @@ async fn handle_subsystem_messages( metrics: Metrics, ) -> Result<(), UnexpectedAbort> where - Context: SubsystemContext, + Context: SubsystemContext, N: Network, AD: validator_discovery::AuthorityDiscovery, { @@ -563,7 +576,7 @@ where } async fn handle_network_messages( - mut sender: impl SubsystemSender , + mut sender: impl SubsystemSender, mut network_service: impl Network, mut network_stream: BoxStream<'static, NetworkEvent>, mut authority_discovery_service: AD, @@ -830,7 +843,7 @@ async fn handle_network_messages( #[tracing::instrument(skip(bridge, ctx, network_stream), fields(subsystem = LOG_TARGET))] async fn run_network( bridge: NetworkBridge, - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, network_stream: BoxStream<'static, NetworkEvent>, ) -> SubsystemResult<()> where @@ -851,7 +864,7 @@ where .get_statement_fetching() .expect("Gets initialized, must be `Some` on startup. qed."); - let (remote, network_event_handler) = handle_network_messages( + let (remote, network_event_handler) = handle_network_messages::<>( ctx.sender().clone(), network_service.clone(), network_stream, @@ -863,9 +876,9 @@ where ctx.spawn("network-bridge-network-worker", Box::pin(remote)).await?; - ctx.send_message(AllMessages::StatementDistribution( - StatementDistributionMessage::StatementFetchingReceiver(statement_receiver) - )).await; + ctx.send_message( + AllMessages::from(StatementDistributionMessage::StatementFetchingReceiver(statement_receiver)) + ).await; let subsystem_event_handler = handle_subsystem_messages( ctx, @@ -927,7 +940,7 @@ fn construct_view(live_heads: impl DoubleEndedIterator, finalized_n #[tracing::instrument(level = "trace", skip(net, ctx, shared, metrics), fields(subsystem = LOG_TARGET))] async fn update_our_view( net: &mut impl Network, - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, live_heads: &[ActivatedLeaf], shared: &Shared, finalized_number: BlockNumber, @@ -1079,21 +1092,21 @@ async fn send_collation_message( async fn dispatch_validation_event_to_all( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) { dispatch_validation_events_to_all(std::iter::once(event), ctx).await } async fn dispatch_collation_event_to_all( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) { dispatch_collation_events_to_all(std::iter::once(event), ctx).await } fn dispatch_validation_event_to_all_unbounded( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) { for msg in AllMessages::dispatch_iter(event) { ctx.send_unbounded_message(msg); @@ -1102,7 +1115,7 @@ fn dispatch_validation_event_to_all_unbounded( fn dispatch_collation_event_to_all_unbounded( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) { if let Some(msg) = event.focus().ok().map(CollatorProtocolMessage::NetworkBridgeUpdateV1) { ctx.send_unbounded_message(msg.into()); @@ -1112,7 +1125,7 @@ fn dispatch_collation_event_to_all_unbounded( #[tracing::instrument(level = "trace", skip(events, ctx), fields(subsystem = LOG_TARGET))] async fn dispatch_validation_events_to_all( events: I, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) where I: IntoIterator>, @@ -1124,14 +1137,14 @@ async fn dispatch_validation_events_to_all( #[tracing::instrument(level = "trace", skip(events, ctx), fields(subsystem = LOG_TARGET))] async fn dispatch_collation_events_to_all( events: I, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) where I: IntoIterator>, I::IntoIter: Send, { let messages_for = |event: NetworkBridgeEvent| { - event.focus().ok().map(|m| AllMessages::CollatorProtocol( + event.focus().ok().map(|m| AllMessages::CollatorProtocolMessage( CollatorProtocolMessage::NetworkBridgeUpdateV1(m) )) }; diff --git a/node/network/bridge/src/multiplexer.rs b/node/network/bridge/src/multiplexer.rs index e7d958dd8676..0437ebeee786 100644 --- a/node/network/bridge/src/multiplexer.rs +++ b/node/network/bridge/src/multiplexer.rs @@ -29,6 +29,7 @@ use sc_network::PeerId; use polkadot_node_network_protocol::request_response::{ request::IncomingRequest, v1, Protocol, RequestResponseConfig, }; +use polkadot_overseer::AllMessages; /// Multiplex incoming network requests. /// @@ -150,22 +151,22 @@ fn multiplex_single( }: network::IncomingRequest, ) -> Result { let r = match p { - Protocol::ChunkFetching => From::from(IncomingRequest::new( + Protocol::ChunkFetching => AllMessages::from(IncomingRequest::new( peer, decode_with_peer::(peer, payload)?, pending_response, )), - Protocol::CollationFetching => From::from(IncomingRequest::new( + Protocol::CollationFetching => AllMessages::from(IncomingRequest::new( peer, decode_with_peer::(peer, payload)?, pending_response, )), - Protocol::PoVFetching => From::from(IncomingRequest::new( + Protocol::PoVFetching => AllMessages::from(IncomingRequest::new( peer, decode_with_peer::(peer, payload)?, pending_response, )), - Protocol::AvailableDataFetching => From::from(IncomingRequest::new( + Protocol::AvailableDataFetching => AllMessages::from(IncomingRequest::new( peer, decode_with_peer::(peer, payload)?, pending_response, diff --git a/node/network/bridge/src/network.rs b/node/network/bridge/src/network.rs index e219f932e8f4..acdf7a539c2b 100644 --- a/node/network/bridge/src/network.rs +++ b/node/network/bridge/src/network.rs @@ -36,7 +36,7 @@ use polkadot_node_network_protocol::{ PeerId, UnifiedReputationChange as Rep, }; use polkadot_primitives::v1::{Block, Hash}; -use polkadot_subsystem::{SubsystemError, SubsystemResult}; +use polkadot_subsystem::errors::{SubsystemError, SubsystemResult}; use crate::validator_discovery::AuthorityDiscovery; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 8f5e7ee55247..72085e4f6a0e 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -1,31 +1,69 @@ use quote::quote; use syn::Result; +use syn::Path; +use syn::spanned::Spanned; +use proc_macro2::Ident; use super::*; /// Generates the wrapper type enum. pub(crate) fn impl_message_wrapper_enum(info: &OverseerInfo) -> Result { let consumes = info.consumes(); + let consumes_variant = info.consumes() + .into_iter() + .try_fold(Vec::new(), |mut acc: Vec, path: Path| { + let ident = path.get_ident().ok_or_else(||{ + syn::Error::new(path.span(), "Missing identifier to use as enum variant.") + })?; + acc.push(ident.clone()); + Ok::<_, syn::Error>(acc) + })?; + + let outgoing = &info.outgoing_ty; let message_wrapper = &info.message_wrapper; + + let (outgoing_from_impl, outgoing_decl) = if let Some(outgoing) = outgoing { + let outgoing_variant = outgoing + .get_ident() + .ok_or_else(||{ + syn::Error::new(outgoing.span(), "Missing identifier to use as enum variant for outgoing.") + })?; + (quote! { + impl ::std::convert::From< #outgoing > for #message_wrapper { + fn from(message: #outgoing) -> Self { + #message_wrapper :: #outgoing_variant ( message ) + } + } + }, + quote! { + #outgoing_variant ( #outgoing ) , + }) + } else { + (TokenStream::new(), TokenStream::new()) + }; + let ts = quote! { /// Generated message type wrapper #[allow(missing_docs)] #[derive(Debug)] pub enum #message_wrapper { #( - #consumes ( #consumes ), + #consumes_variant ( #consumes ), )* + #outgoing_decl } #( - impl ::std::convert::From< #consumes > for #message_wrapper { - fn from(message: #consumes) -> Self { - #message_wrapper :: #consumes ( message ) + impl ::std::convert::From< #consumes > for #message_wrapper { + fn from(message: #consumes) -> Self { + #message_wrapper :: #consumes_variant ( message ) + } } - } )* + + #outgoing_from_impl }; Ok(ts) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index 4967fd31a01a..b6f86ef0b7ba 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -11,6 +11,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result let subsystem_ctx_name = Ident::new(&(overseer_name.to_string() + "SubsystemContext"), overseer_name.span()); let consumes = &info.consumes(); let signal = &info.extern_signal_ty; + let wrapper_message = &info.message_wrapper; let ts = quote! { /// Connector to send messages towards all subsystems, @@ -23,24 +24,52 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result signals_received: SignalsReceived, } + /// impl for wrapping message type... + #[::polkadot_overseer_gen::async_trait] + impl SubsystemSender< #wrapper_message > for #subsystem_sender_name { + async fn send_message(&mut self, msg: #wrapper_message) { + self.channels.send_and_log_error(self.signals_received.load(), msg).await; + } + + async fn send_messages(&mut self, msgs: T) + where + T: IntoIterator + Send, + T::IntoIter: Send, + { + // This can definitely be optimized if necessary. + for msg in msgs { + self.send_message(msg).await; + } + } + + fn send_unbounded_message(&mut self, msg: #wrapper_message) { + self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg); + } + } + + // ... but also implement for all individual messages to avoid + // the necessity for manual wrapping, and do the conversion + // based on the generated `From::from` impl for the individual variants. #( #[::polkadot_overseer_gen::async_trait] impl SubsystemSender< #consumes > for #subsystem_sender_name { async fn send_message(&mut self, msg: #consumes) { - self.channels.send_and_log_error(self.signals_received.load(), msg.into()).await; + self.channels.send_and_log_error(self.signals_received.load(), #wrapper_message ::from ( msg )).await; } async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send + where + T: IntoIterator + Send, + T::IntoIter: Send, { // This can definitely be optimized if necessary. for msg in msgs { - self.send_message(msg).await; + self.send_message( #wrapper_message ::from ( msg )).await; } } fn send_unbounded_message(&mut self, msg: #consumes) { - self.channels.send_unbounded_and_log_error(self.signals_received.load(), msg.into()); + self.channels.send_unbounded_and_log_error(self.signals_received.load(), #wrapper_message ::from ( msg )); } } )* @@ -91,11 +120,14 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result #[::polkadot_overseer_gen::async_trait] impl SubsystemContext for #subsystem_ctx_name where - #subsystem_sender_name: polkadot_overseer_gen::SubsystemSender, + #subsystem_sender_name: ::polkadot_overseer_gen::SubsystemSender< #wrapper_message >, + #subsystem_sender_name: ::polkadot_overseer_gen::SubsystemSender< M >, + AllMessages: From, { type Message = M; type Signal = #signal; type Sender = #subsystem_sender_name; + type AllMessages = #wrapper_message; async fn try_recv(&mut self) -> Result>, ()> { match ::polkadot_overseer_gen::poll!(self.recv()) { diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs index 7c2ba863e1e0..dd0231549138 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs @@ -42,7 +42,7 @@ const TAG_EXT_EVENT_TY: &str = "event"; const TAG_EXT_SIGNAL_TY: &str = "signal"; const TAG_EXT_ERROR_TY: &str = "error"; const TAG_EXT_NETWORK_TY: &str = "network"; -const TAG_OUTGOING_TY: &str = "out"; +const TAG_OUTGOING_TY: &str = "outgoing"; const TAG_GEN_TY: &str = "gen"; const TAG_SIGNAL_CAPACITY: &str = "signal_capacity"; const TAG_MESSAGE_CAPACITY: &str = "message_capacity"; diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 97efc040bab7..182c37d1b7ee 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -359,12 +359,17 @@ pub enum FromOverseer { #[async_trait::async_trait] pub trait SubsystemContext: Send + 'static { /// The message type of this context. Subsystems launched with this context will expect - /// to receive messages of this type. + /// to receive messages of this type. Commonly uses the wrapping enum commonly called + /// `AllMessages`. type Message: std::fmt::Debug + Send + 'static; /// And the same for signals. type Signal: std::fmt::Debug + Send + 'static; + /// The overarching all messages enum. + /// In some cases can be identical to `Self::Message`. + type AllMessages: From + Send + 'static; /// The sender type as provided by `sender()` and underlying. - type Sender: SubsystemSender + std::fmt::Debug + Send; + type Sender: SubsystemSender + SubsystemSender + std::fmt::Debug + Send; + /// Try to asynchronously receive a message. /// @@ -386,15 +391,23 @@ pub trait SubsystemContext: Send + 'static { ) -> SubsystemResult<()>; /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: Self::Message) { - self.sender().send_message(msg).await + async fn send_message(&mut self, msg: X) + where + Self::AllMessages: From, + X: Send, + { + self.sender().send_message(::from(msg)).await } /// Send multiple direct messages to other `Subsystem`s, routed based on message type. - async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send + async fn send_messages(&mut self, msgs: T) + where + T: IntoIterator + Send, + T::IntoIter: Send, + Self::AllMessages: From, + X: Send, { - self.sender().send_messages(msgs).await + self.sender().send_messages(msgs.into_iter().map(|x| ::from(x))).await } /// Obtain the sender. @@ -419,22 +432,22 @@ where } -/// TODO FIXME +/// Sender end of a channel to interface with a subsystem. #[async_trait::async_trait] -pub trait SubsystemSender: Send + Clone + 'static { +pub trait SubsystemSender: Send + Clone + 'static { /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: M); + async fn send_message(&mut self, msg: Message); /// Send multiple direct messages to other `Subsystem`s, routed based on message type. async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send; + where T: IntoIterator + Send, T::IntoIter: Send; /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message /// type. /// /// This function should be used only when there is some other bounding factor on the messages /// sent with it. Otherwise, it risks a memory leak. - fn send_unbounded_message(&mut self, msg: M); + fn send_unbounded_message(&mut self, msg: Message); } /// A future that wraps another future with a `Delay` allowing for time-limited futures. diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index fe86a41f278b..e401dbf59649 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -125,8 +125,17 @@ use polkadot_overseer_gen::{ }; pub use polkadot_overseer_gen as gen; -// Target for logs. -// const LOG_TARGET: &'static str = "parachain::overseer"; +/// Boiler plate reduction, `Signal` and `AllMessages` are fixed for one overseer. +/// So the only variation for external entities is `M`. +pub trait SubsystemCtx : SubsystemContext { +} + +impl SubsystemCtx for T +where + T: SubsystemContext +{ +} + /// Whether a header supports parachain consensus or not. pub trait HeadSupportsParachains { diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index 0c927775d0b8..764c57552a37 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -17,7 +17,6 @@ pub struct DummySubsystem; impl Subsystem for DummySubsystem where Context: SubsystemContext, - ::Message: std::fmt::Debug + Send + 'static, { fn start(self, mut ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { diff --git a/node/subsystem-test-helpers/src/lib.rs b/node/subsystem-test-helpers/src/lib.rs index e12c91a828e6..67964b27b6ba 100644 --- a/node/subsystem-test-helpers/src/lib.rs +++ b/node/subsystem-test-helpers/src/lib.rs @@ -171,7 +171,7 @@ pub fn sender_receiver() -> (TestSubsystemSender, mpsc::UnboundedReceiver for TestSubsystemSender { +impl SubsystemSender for TestSubsystemSender { async fn send_message(&mut self, msg: AllMessages) { self.tx .send(msg) diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 666404ec8451..797804bf357a 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -285,7 +285,7 @@ impl Validator { // However, each of them returns a oneshot::Receiver, and those are resolved concurrently. let (validators, session_index) = futures::try_join!( request_validators(parent, sender).await, - request_session_index_for_child(parent, sender).await, + request_session_index_for_child::(parent, sender).await, )?; let signing_context = SigningContext { diff --git a/node/subsystem-util/src/runtime/mod.rs b/node/subsystem-util/src/runtime/mod.rs index 5daeef91e844..c2fb645d3ba2 100644 --- a/node/subsystem-util/src/runtime/mod.rs +++ b/node/subsystem-util/src/runtime/mod.rs @@ -102,7 +102,7 @@ impl RuntimeInfo { Some(index) => Ok(*index), None => { let index = - recv_runtime(request_session_index_for_child(parent, ctx.sender()).await) + recv_runtime(request_session_index_for_child::(parent, ctx.sender()).await) .await?; self.session_index_cache.put(parent, index); Ok(index) @@ -120,7 +120,7 @@ impl RuntimeInfo { M: From, Context: SubsystemContext, { - let session_index = self.get_session_index(ctx, parent).await?; + let session_index = self.get_session_index::<_,M>(ctx, parent).await?; self.get_session_info_by_index(ctx, parent, session_index).await } @@ -141,7 +141,7 @@ impl RuntimeInfo { { if !self.session_info_cache.contains(&session_index) { let session_info = - recv_runtime(request_session_info(parent, session_index, ctx.sender()).await) + recv_runtime(request_session_info::(parent, session_index, ctx.sender()).await) .await? .ok_or(NonFatal::NoSuchSession(session_index))?; let validator_info = self.get_validator_info(&session_info).await?; @@ -172,8 +172,8 @@ impl RuntimeInfo { Payload: EncodeAs + Clone, RealPayload: Encode + Clone, { - let session_index = self.get_session_index(ctx, parent).await?; - let info = self.get_session_info_by_index(ctx, parent, session_index).await?; + let session_index = self.get_session_index::<_,M>(ctx, parent).await?; + let info = self.get_session_info_by_index::<_,M>(ctx, parent, session_index).await?; Ok(check_signature(session_index, &info.session_info, parent, signed)) } @@ -255,7 +255,7 @@ pub async fn get_availability_cores(ctx: &mut Context, relay_parent: M: From, Context: SubsystemContext, { - recv_runtime(request_availability_cores(relay_parent, ctx.sender()).await).await + recv_runtime(request_availability_cores::(relay_parent, ctx.sender()).await).await } /// Variant of `request_availability_cores` that only returns occupied ones. @@ -291,6 +291,6 @@ where { // We drop `groups` here as we don't need them, because of `RuntimeInfo`. Ideally we would not // fetch them in the first place. - let (_, info) = recv_runtime(request_validator_groups(relay_parent, ctx.sender()).await).await?; + let (_, info) = recv_runtime(request_validator_groups::(relay_parent, ctx.sender()).await).await?; Ok(info) } From cab7530afc8a95e66ed0d5844598b44c92504f62 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 24 Jun 2021 17:03:08 +0200 Subject: [PATCH 068/161] split node-subsystem --- node/overseer/Cargo.toml | 2 +- node/overseer/overseer-gen/examples/dummy.rs | 4 +- .../proc-macro/src/impl_builder.rs | 16 +- .../overseer-gen/proc-macro/src/impl_misc.rs | 14 +- .../proc-macro/src/impl_overseer.rs | 17 +- node/overseer/overseer-gen/src/lib.rs | 20 +-- node/overseer/src/lib.rs | 9 +- node/overseer/src/subsystems.rs | 4 +- node/subsystem-types/Cargo.toml | 37 +++++ .../src/errors.rs | 2 +- node/subsystem-types/src/lib.rs | 152 ++++++++++++++++++ .../src/messages.rs | 0 .../src/messages/network_bridge_event.rs | 0 node/subsystem-util/Cargo.toml | 2 +- node/subsystem-util/src/lib.rs | 6 +- node/subsystem/Cargo.toml | 33 +--- node/subsystem/src/lib.rs | 150 ++++------------- 17 files changed, 274 insertions(+), 194 deletions(-) create mode 100644 node/subsystem-types/Cargo.toml rename node/{subsystem => subsystem-types}/src/errors.rs (98%) create mode 100644 node/subsystem-types/src/lib.rs rename node/{subsystem => subsystem-types}/src/messages.rs (100%) rename node/{subsystem => subsystem-types}/src/messages/network_bridge_event.rs (100%) diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml index 26d72397d5a8..694a44f23938 100644 --- a/node/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -12,10 +12,10 @@ futures = "0.3.12" futures-timer = "3.0.2" polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-node-primitives = { package = "polkadot-node-primitives", path = "../primitives" } +polkadot-node-subsystem = { package = "polkadot-node-subsystem-types", path = "../subsystem-types" } polkadot-node-subsystem-util = { path = "../subsystem-util" } polkadot-primitives = { path = "../../primitives" } polkadot-overseer-gen = { package = "polkadot-overseer-gen", path = "./overseer-gen" } -polkadot-node-subsystem = { package = "polkadot-node-subsystem", path = "../subsystem" } tracing = "0.1.26" lru = "0.6" diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index ea5f5ed6ef2d..21d35d2fb135 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -53,8 +53,8 @@ impl std::fmt::Display for Yikes { impl std::error::Error for Yikes {} -impl From for Yikes { - fn from(_: polkadot_overseer_gen::SubsystemError) -> Yikes { +impl From for Yikes { + fn from(_: polkadot_overseer_gen::OverseerError) -> Yikes { Yikes } } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index fe2d11145f7b..fb91ca5c858a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -26,6 +26,8 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Result }; - let error_ty = &info.extern_error_ty; let consumes = &info.consumes(); let subsyste_ctx_name = Ident::new(&(overseer_name.to_string() + "SubsystemContext"), overseer_name.span()); @@ -99,7 +100,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Self { // explicitly assure the required traits are implemented fn trait_from_must_be_implemented() - where E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::SubsystemError> + where E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::OverseerError> {} trait_from_must_be_implemented::< #error_ty >(); @@ -140,7 +141,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result SubsystemResult<(#overseer_name #generics, #handler)> + pub fn build(mut self) -> ::std::result::Result<(#overseer_name #generics, #handler), #error_ty> { let (events_tx, events_rx) = ::polkadot_overseer_gen::metered::channel::< #event @@ -180,7 +181,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result> + BoxFuture<'static, ::std::result::Result<(), #error_ty > > >::new(); #( @@ -245,6 +246,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Result { let signal = &info.extern_signal_ty; + let error_ty = &info.extern_error_ty; let ts = quote! { @@ -283,14 +285,14 @@ pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result>>, - ) -> SubsystemResult> + futures: &mut ::polkadot_overseer_gen::FuturesUnordered >>, + ) -> ::std::result::Result, #error_ty> where S: ::polkadot_overseer_gen::SpawnNamed, M: std::fmt::Debug + Send + 'static, TK: TaskKind, Ctx: ::polkadot_overseer_gen::SubsystemContext, - E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::SubsystemError>, + E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::OverseerError>, SubSys: ::polkadot_overseer_gen::Subsystem, { let ::polkadot_overseer_gen::SpawnedSubsystem { future, name } = s.start(ctx); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index b6f86ef0b7ba..7945ee916d52 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -129,14 +129,14 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result type Sender = #subsystem_sender_name; type AllMessages = #wrapper_message; - async fn try_recv(&mut self) -> Result>, ()> { + async fn try_recv(&mut self) -> ::std::result::Result>, ()> { match ::polkadot_overseer_gen::poll!(self.recv()) { ::polkadot_overseer_gen::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), ::polkadot_overseer_gen::Poll::Pending => Ok(None), } } - async fn recv(&mut self) -> ::polkadot_overseer_gen::SubsystemResult> { + async fn recv(&mut self) -> ::polkadot_overseer_gen::OverseerResult> { loop { // If we have a message pending an overseer signal, we only poll for signals // in the meantime. @@ -148,7 +148,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result // wait for next signal. let signal = self.signals.next().await - .ok_or(::polkadot_overseer_gen::SubsystemError::Context( + .ok_or(::polkadot_overseer_gen::OverseerError::Context( "Signal channel is terminated and empty." .to_owned() ))?; @@ -167,7 +167,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result let from_overseer = ::polkadot_overseer_gen::futures::select_biased! { signal = await_signal => { let signal = signal - .ok_or(::polkadot_overseer_gen::SubsystemError::Context( + .ok_or(::polkadot_overseer_gen::OverseerError::Context( "Signal channel is terminated and empty." .to_owned() ))?; @@ -176,7 +176,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result } msg = await_message => { let packet = msg - .ok_or(::polkadot_overseer_gen::SubsystemError::Context( + .ok_or(::polkadot_overseer_gen::OverseerError::Context( "Message channel is terminated and empty." .to_owned() ))?; @@ -206,7 +206,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn(ctx, name, fut)")] async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) - -> ::polkadot_overseer_gen::SubsystemResult<()> + -> ::polkadot_overseer_gen::OverseerResult<()> { self.to_overseer.send(::polkadot_overseer_gen::ToOverseer::SpawnJob { name, @@ -216,7 +216,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn_blocking(ctx, name, fut)")] async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) - -> ::polkadot_overseer_gen::SubsystemResult<()> + -> ::polkadot_overseer_gen::OverseerResult<()> { self.to_overseer.send(::polkadot_overseer_gen::ToOverseer::SpawnBlockingJob { name, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 2b3fd295475f..904200c4ac3b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -84,9 +84,9 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result ::polkadot_overseer_gen::SubsystemResult<()> { + pub async fn wait_terminate(&mut self, signal: #signal_ty, timeout: ::std::time::Duration) -> ::std::result::Result<(), #error_ty > { #( - let _ = self. #subsystem_name .send_signal(signal.clone()).await; + ::std::mem::drop(self. #subsystem_name .send_signal(signal.clone()).await); )* let _ = signal; @@ -110,7 +110,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result ::polkadot_overseer_gen::SubsystemResult<()> { + pub async fn broadcast_signal(&mut self, signal: #signal_ty) -> ::std::result::Result<(), #error_ty > { #( let _ = self. #subsystem_name .send_signal(signal.clone()).await; )* @@ -120,7 +120,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result ::polkadot_overseer_gen::SubsystemResult<()> { + pub async fn route_message(&mut self, message: #message_wrapper) -> ::std::result::Result<(), #error_ty > { match message { #( #message_wrapper :: #consumes ( inner ) => self. #subsystem_name .send_message( inner ).await?, @@ -157,6 +157,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result Result { let signal = &info.extern_signal_ty; + let error_ty = &info.extern_error_ty; let ts = quote::quote! { @@ -178,7 +179,7 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result ::polkadot_overseer_gen::SubsystemResult<()> { + pub async fn send_message(&mut self, message: M) -> ::std::result::Result<(), #error_ty > { const MESSAGE_TIMEOUT: Duration = Duration::from_secs(10); if let Some(ref mut instance) = self.instance { @@ -188,7 +189,7 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { - Err(::polkadot_overseer_gen::SubsystemError::SubsystemStalled(instance.name)) + Err(::polkadot_overseer_gen::OverseerError::SubsystemStalled(instance.name)) } Some(res) => res.map_err(Into::into), } @@ -200,13 +201,13 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result ::polkadot_overseer_gen::SubsystemResult<()> { + pub async fn send_signal(&mut self, signal: #signal) -> ::std::result::Result<(), #error_ty > { const SIGNAL_TIMEOUT: ::std::time::Duration = ::std::time::Duration::from_secs(10); if let Some(ref mut instance) = self.instance { match instance.tx_signal.send(signal).timeout(SIGNAL_TIMEOUT).await { None => { - Err(::polkadot_overseer_gen::SubsystemError::SubsystemStalled(instance.name)) + Err(::polkadot_overseer_gen::OverseerError::SubsystemStalled(instance.name)) } Some(res) => { let res = res.map_err(Into::into); diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 182c37d1b7ee..00f04057ae2e 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -129,7 +129,7 @@ pub enum ToOverseer { use std::fmt; -impl fmt::Debug for ToOverseer { +impl fmt::Debug for ToOverseer { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::SpawnJob{ name, .. } => writeln!(f, "SpawnJob{{ {}, ..}}", name), @@ -224,13 +224,13 @@ pub trait AnnotateErrorOrigin: 'static + Send + Sync + std::error::Error { /// An asynchronous subsystem task.. /// /// In essence it's just a newtype wrapping a `BoxFuture`. -pub struct SpawnedSubsystem +pub struct SpawnedSubsystem where E: std::error::Error + Send + Sync + 'static - + From, + + From, { /// Name of the subsystem being spawned. pub name: &'static str, @@ -248,7 +248,7 @@ pub struct SpawnedSubsystem // FIXME XXX make generic over the source error of FromOrigin #[derive(thiserror::Error, Debug)] #[allow(missing_docs)] -pub enum SubsystemError { +pub enum OverseerError { #[error(transparent)] NotifyCancellation(#[from] oneshot::Canceled), @@ -277,8 +277,8 @@ pub enum SubsystemError { }, } -/// Alias for a result with error type `SubsystemError`. -pub type SubsystemResult = std::result::Result; +/// Alias for a result with error type `OverseerError`. +pub type OverseerResult = std::result::Result; /// Collection of meters related to a subsystem. #[derive(Clone)] @@ -378,17 +378,17 @@ pub trait SubsystemContext: Send + 'static { async fn try_recv(&mut self) -> Result>, ()>; /// Receive a message. - async fn recv(&mut self) -> SubsystemResult>; + async fn recv(&mut self) -> OverseerResult>; /// Spawn a child task on the executor. - async fn spawn(&mut self, name: &'static str, s: ::std::pin::Pin + Send>>) -> SubsystemResult<()>; + async fn spawn(&mut self, name: &'static str, s: ::std::pin::Pin + Send>>) -> OverseerResult<()>; /// Spawn a blocking child task on the executor's dedicated thread pool. async fn spawn_blocking( &mut self, name: &'static str, s: ::std::pin::Pin + Send>>, - ) -> SubsystemResult<()>; + ) -> OverseerResult<()>; /// Send a direct message to some other `Subsystem`, routed based on message type. async fn send_message(&mut self, msg: X) @@ -425,7 +425,7 @@ pub trait SubsystemContext: Send + 'static { pub trait Subsystem where Ctx: SubsystemContext, - E: std::error::Error + Send + Sync + 'static + From, + E: std::error::Error + Send + Sync + 'static + From, { /// Start this `Subsystem` and return `SpawnedSubsystem`. fn start(self, ctx: Ctx) -> SpawnedSubsystem < E >; diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index e401dbf59649..0bddaa288de5 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -586,9 +586,14 @@ where Ok((overseer, Handler(handler))) } - // Stop the overseer. + /// Stop the overseer. async fn stop(mut self) { - let _ = self.wait_terminate(OverseerSignal::Conclude, ::std::time::Duration::from_secs(1_u64)).await; + ::std::mem::drop( + self.wait_terminate( + OverseerSignal::Conclude, + ::std::time::Duration::from_secs(1_u64) + ).await + ); } /// Run the `Overseer`. diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index 764c57552a37..8bb375298f87 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -2,7 +2,7 @@ use ::polkadot_overseer_gen::{ MapSubsystem, SubsystemContext, Subsystem, - SubsystemError, + OverseerError, SpawnedSubsystem, FromOverseer, }; @@ -14,7 +14,7 @@ use crate::OverseerSignal; #[derive(Clone, Copy, Debug)] pub struct DummySubsystem; -impl Subsystem for DummySubsystem +impl Subsystem for DummySubsystem where Context: SubsystemContext, { diff --git a/node/subsystem-types/Cargo.toml b/node/subsystem-types/Cargo.toml new file mode 100644 index 000000000000..339a42f38a3a --- /dev/null +++ b/node/subsystem-types/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "polkadot-node-subsystem-types" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" +description = "Subsystem traits and message definitions" + +[dependencies] +async-std = "1.8.0" +async-trait = "0.1.42" +derive_more = "0.99.11" +futures = "0.3.12" +futures-timer = "3.0.2" +mick-jaeger = "0.1.2" +lazy_static = "1.4" +tracing = "0.1.26" +parity-scale-codec = { version = "2.0.0", default-features = false, features = ["derive"] } +parking_lot = "0.11.1" +pin-project = "1.0.4" +polkadot-node-primitives = { path = "../primitives" } +polkadot-node-network-protocol = { path = "../network/protocol" } +polkadot-primitives = { path = "../../primitives" } +polkadot-statement-table = { path = "../../statement-table" } +polkadot-node-jaeger = { path = "../jaeger" } +polkadot-overseer-gen = { path = "../overseer/overseer-gen" } +sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } +smallvec = "1.6.1" +sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } +substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" } +thiserror = "1.0.23" +log = "0.4.13" + +[dev-dependencies] +assert_matches = "1.4.0" +async-trait = "0.1.42" +futures = { version = "0.3.12", features = ["thread-pool"] } +polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } diff --git a/node/subsystem/src/errors.rs b/node/subsystem-types/src/errors.rs similarity index 98% rename from node/subsystem/src/errors.rs rename to node/subsystem-types/src/errors.rs index 976a4e0ed60a..5097435f2766 100644 --- a/node/subsystem/src/errors.rs +++ b/node/subsystem-types/src/errors.rs @@ -117,7 +117,7 @@ pub enum SubsystemError { /// Generated by the `#[overseer(..)]` proc-macro #[error(transparent)] - Generated(#[from] polkadot_overseer_gen::SubsystemError), + Generated(#[from] polkadot_overseer_gen::OverseerError), /// Per origin (or subsystem) annotations to wrap an error. #[error("Error originated in {origin}")] diff --git a/node/subsystem-types/src/lib.rs b/node/subsystem-types/src/lib.rs new file mode 100644 index 000000000000..982dacc8c2dd --- /dev/null +++ b/node/subsystem-types/src/lib.rs @@ -0,0 +1,152 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Subsystem trait definitions and message types. +//! +//! Node-side logic for Polkadot is mostly comprised of Subsystems, which are discrete components +//! that communicate via message-passing. They are coordinated by an overseer, provided by a +//! separate crate. + +#![warn(missing_docs)] + +use std::{sync::Arc, fmt}; + +use polkadot_primitives::v1::{Hash, BlockNumber}; +use smallvec::SmallVec; + +pub mod errors; +pub mod messages; + +pub use polkadot_node_jaeger as jaeger; +pub use jaeger::*; + +/// How many slots are stack-reserved for active leaves updates +/// +/// If there are fewer than this number of slots, then we've wasted some stack space. +/// If there are greater than this number of slots, then we fall back to a heap vector. +const ACTIVE_LEAVES_SMALLVEC_CAPACITY: usize = 8; + + +/// The status of an activated leaf. +#[derive(Debug, Clone)] +pub enum LeafStatus { + /// A leaf is fresh when it's the first time the leaf has been encountered. + /// Most leaves should be fresh. + Fresh, + /// A leaf is stale when it's encountered for a subsequent time. This will happen + /// when the chain is reverted or the fork-choice rule abandons some chain. + Stale, +} + +impl LeafStatus { + /// Returns a bool indicating fresh status. + pub fn is_fresh(&self) -> bool { + match *self { + LeafStatus::Fresh => true, + LeafStatus::Stale => false, + } + } + + /// Returns a bool indicating stale status. + pub fn is_stale(&self) -> bool { + match *self { + LeafStatus::Fresh => false, + LeafStatus::Stale => true, + } + } +} + +/// Activated leaf. +#[derive(Debug, Clone)] +pub struct ActivatedLeaf { + /// The block hash. + pub hash: Hash, + /// The block number. + pub number: BlockNumber, + /// The status of the leaf. + pub status: LeafStatus, + /// An associated [`jaeger::Span`]. + /// + /// NOTE: Each span should only be kept active as long as the leaf is considered active and should be dropped + /// when the leaf is deactivated. + pub span: Arc, +} + +/// Changes in the set of active leaves: the parachain heads which we care to work on. +/// +/// Note that the activated and deactivated fields indicate deltas, not complete sets. +#[derive(Clone, Default)] +pub struct ActiveLeavesUpdate { + /// New relay chain blocks of interest. + pub activated: SmallVec<[ActivatedLeaf; ACTIVE_LEAVES_SMALLVEC_CAPACITY]>, + /// Relay chain block hashes no longer of interest. + pub deactivated: SmallVec<[Hash; ACTIVE_LEAVES_SMALLVEC_CAPACITY]>, +} + +impl ActiveLeavesUpdate { + /// Create a ActiveLeavesUpdate with a single activated hash + pub fn start_work(activated: ActivatedLeaf) -> Self { + Self { activated: [activated][..].into(), ..Default::default() } + } + + /// Create a ActiveLeavesUpdate with a single deactivated hash + pub fn stop_work(hash: Hash) -> Self { + Self { deactivated: [hash][..].into(), ..Default::default() } + } + + /// Is this update empty and doesn't contain any information? + pub fn is_empty(&self) -> bool { + self.activated.is_empty() && self.deactivated.is_empty() + } +} + +impl PartialEq for ActiveLeavesUpdate { + /// Equality for `ActiveLeavesUpdate` doesnt imply bitwise equality. + /// + /// Instead, it means equality when `activated` and `deactivated` are considered as sets. + fn eq(&self, other: &Self) -> bool { + self.activated.len() == other.activated.len() && self.deactivated.len() == other.deactivated.len() + && self.activated.iter().all(|a| other.activated.iter().any(|o| a.hash == o.hash)) + && self.deactivated.iter().all(|a| other.deactivated.contains(a)) + } +} + +impl fmt::Debug for ActiveLeavesUpdate { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + struct Activated<'a>(&'a [ActivatedLeaf]); + impl fmt::Debug for Activated<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.0.iter().map(|e| e.hash)).finish() + } + } + + f.debug_struct("ActiveLeavesUpdate") + .field("activated", &Activated(&self.activated)) + .field("deactivated", &self.deactivated) + .finish() + } +} + +/// Signals sent by an overseer to a subsystem. +#[derive(PartialEq, Clone, Debug)] +pub enum OverseerSignal { + /// Subsystems should adjust their jobs to start and stop work on appropriate block hashes. + ActiveLeaves(ActiveLeavesUpdate), + /// `Subsystem` is informed of a finalized block by its block hash and number. + BlockFinalized(Hash, BlockNumber), + /// Conclude the work of the `Overseer` and all `Subsystem`s. + Conclude, +} diff --git a/node/subsystem/src/messages.rs b/node/subsystem-types/src/messages.rs similarity index 100% rename from node/subsystem/src/messages.rs rename to node/subsystem-types/src/messages.rs diff --git a/node/subsystem/src/messages/network_bridge_event.rs b/node/subsystem-types/src/messages/network_bridge_event.rs similarity index 100% rename from node/subsystem/src/messages/network_bridge_event.rs rename to node/subsystem-types/src/messages/network_bridge_event.rs diff --git a/node/subsystem-util/Cargo.toml b/node/subsystem-util/Cargo.toml index aa73fe3543ba..da6f51b972e4 100644 --- a/node/subsystem-util/Cargo.toml +++ b/node/subsystem-util/Cargo.toml @@ -19,7 +19,7 @@ tracing = "0.1.26" lru = "0.6.5" polkadot-node-primitives = { path = "../primitives" } -polkadot-node-subsystem = { path = "../subsystem" } +polkadot-node-subsystem = { package = "polkadot-node-subsystem-types", path = "../subsystem-types" } polkadot-node-jaeger = { path = "../jaeger" } polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-primitives = { path = "../../primitives" } diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 797804bf357a..c988d2e3c2ab 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -35,7 +35,7 @@ pub use polkadot_overseer_gen::{ SpawnedSubsystem, Subsystem, SubsystemContext, - SubsystemError, + OverseerError, SubsystemSender, TimeoutExt, Timeout, @@ -99,7 +99,7 @@ pub enum Error { Mpsc(#[from] mpsc::SendError), /// A subsystem error #[error(transparent)] - Subsystem(#[from] SubsystemError), + Subsystem(#[from] OverseerError), /// An error in the Runtime API. #[error(transparent)] RuntimeApi(#[from] RuntimeApiError), @@ -783,7 +783,7 @@ impl JobSubsystem { } } -impl Subsystem for JobSubsystem +impl Subsystem for JobSubsystem where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext, diff --git a/node/subsystem/Cargo.toml b/node/subsystem/Cargo.toml index ccad1bf65224..61daa8a74d40 100644 --- a/node/subsystem/Cargo.toml +++ b/node/subsystem/Cargo.toml @@ -3,35 +3,10 @@ name = "polkadot-node-subsystem" version = "0.1.0" authors = ["Parity Technologies "] edition = "2018" -description = "Subsystem traits and message definitions" +description = "Subsystem traits and message definitions and the generated overseer" [dependencies] -async-std = "1.8.0" -async-trait = "0.1.42" -derive_more = "0.99.11" -futures = "0.3.12" -futures-timer = "3.0.2" -mick-jaeger = "0.1.2" -lazy_static = "1.4" -tracing = "0.1.26" -parity-scale-codec = { version = "2.0.0", default-features = false, features = ["derive"] } -parking_lot = "0.11.1" -pin-project = "1.0.4" -polkadot-node-primitives = { path = "../primitives" } -polkadot-node-network-protocol = { path = "../network/protocol" } -polkadot-primitives = { path = "../../primitives" } -polkadot-statement-table = { path = "../../statement-table" } -polkadot-node-jaeger = { path = "../jaeger" } +polkadot-overseer = { path = "../overseer" } polkadot-overseer-gen = { path = "../overseer/overseer-gen" } -sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } -smallvec = "1.6.1" -sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } -substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" } -thiserror = "1.0.23" -log = "0.4.13" - -[dev-dependencies] -assert_matches = "1.4.0" -async-trait = "0.1.42" -futures = { version = "0.3.12", features = ["thread-pool"] } -polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } +polkadot-node-subsystem-types = { path = "../subsystem-types" } +polkadot-node-jaeger = { path = "../jaeger" } diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 982dacc8c2dd..8d22545643b0 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -14,139 +14,47 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -//! Subsystem trait definitions and message types. +//! Subsystem accumulation. //! -//! Node-side logic for Polkadot is mostly comprised of Subsystems, which are discrete components -//! that communicate via message-passing. They are coordinated by an overseer, provided by a -//! separate crate. +//! Node-side types and generated overseer. #![warn(missing_docs)] -use std::{sync::Arc, fmt}; - -use polkadot_primitives::v1::{Hash, BlockNumber}; -use smallvec::SmallVec; - -pub mod errors; -pub mod messages; +use polkadot_node_subsystem_types::messages::AvailabilityStoreMessage; +pub use polkadot_node_subsystem_types::{errors, messages}; pub use polkadot_node_jaeger as jaeger; pub use jaeger::*; -/// How many slots are stack-reserved for active leaves updates -/// -/// If there are fewer than this number of slots, then we've wasted some stack space. -/// If there are greater than this number of slots, then we fall back to a heap vector. -const ACTIVE_LEAVES_SMALLVEC_CAPACITY: usize = 8; - - -/// The status of an activated leaf. -#[derive(Debug, Clone)] -pub enum LeafStatus { - /// A leaf is fresh when it's the first time the leaf has been encountered. - /// Most leaves should be fresh. - Fresh, - /// A leaf is stale when it's encountered for a subsequent time. This will happen - /// when the chain is reverted or the fork-choice rule abandons some chain. - Stale, -} - -impl LeafStatus { - /// Returns a bool indicating fresh status. - pub fn is_fresh(&self) -> bool { - match *self { - LeafStatus::Fresh => true, - LeafStatus::Stale => false, - } - } - - /// Returns a bool indicating stale status. - pub fn is_stale(&self) -> bool { - match *self { - LeafStatus::Fresh => false, - LeafStatus::Stale => true, - } - } -} - -/// Activated leaf. -#[derive(Debug, Clone)] -pub struct ActivatedLeaf { - /// The block hash. - pub hash: Hash, - /// The block number. - pub number: BlockNumber, - /// The status of the leaf. - pub status: LeafStatus, - /// An associated [`jaeger::Span`]. - /// - /// NOTE: Each span should only be kept active as long as the leaf is considered active and should be dropped - /// when the leaf is deactivated. - pub span: Arc, -} - -/// Changes in the set of active leaves: the parachain heads which we care to work on. -/// -/// Note that the activated and deactivated fields indicate deltas, not complete sets. -#[derive(Clone, Default)] -pub struct ActiveLeavesUpdate { - /// New relay chain blocks of interest. - pub activated: SmallVec<[ActivatedLeaf; ACTIVE_LEAVES_SMALLVEC_CAPACITY]>, - /// Relay chain block hashes no longer of interest. - pub deactivated: SmallVec<[Hash; ACTIVE_LEAVES_SMALLVEC_CAPACITY]>, -} -impl ActiveLeavesUpdate { - /// Create a ActiveLeavesUpdate with a single activated hash - pub fn start_work(activated: ActivatedLeaf) -> Self { - Self { activated: [activated][..].into(), ..Default::default() } - } +pub use polkadot_node_subsystem_types::{ + LeafStatus, + ActivatedLeaf, + ActiveLeavesUpdate, + OverseerSignal, +}; - /// Create a ActiveLeavesUpdate with a single deactivated hash - pub fn stop_work(hash: Hash) -> Self { - Self { deactivated: [hash][..].into(), ..Default::default() } - } +pub use polkadot_overseer::{ + AllMessages, + Overseer, + gen::{ + OverseerError, + OverseerResult, + SubsystemContext, + SubsystemMeters, + SubsystemMeterReadouts, + TimeoutExt, + }, +}; - /// Is this update empty and doesn't contain any information? - pub fn is_empty(&self) -> bool { - self.activated.is_empty() && self.deactivated.is_empty() - } -} +// Simplify usage without having to do large scale modifications of all +// subsystems at once. +pub type SubssystemSender = polkadot_overseer::gen::SubssystemSender; -impl PartialEq for ActiveLeavesUpdate { - /// Equality for `ActiveLeavesUpdate` doesnt imply bitwise equality. - /// - /// Instead, it means equality when `activated` and `deactivated` are considered as sets. - fn eq(&self, other: &Self) -> bool { - self.activated.len() == other.activated.len() && self.deactivated.len() == other.deactivated.len() - && self.activated.iter().all(|a| other.activated.iter().any(|o| a.hash == o.hash)) - && self.deactivated.iter().all(|a| other.deactivated.contains(a)) - } -} +pub type FromOverseer = polkadot_overseer::gen::FromOverseer; -impl fmt::Debug for ActiveLeavesUpdate { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - struct Activated<'a>(&'a [ActivatedLeaf]); - impl fmt::Debug for Activated<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_list().entries(self.0.iter().map(|e| e.hash)).finish() - } - } +pub type Subsystem = polkadot_overseer::gen::Subsystem; - f.debug_struct("ActiveLeavesUpdate") - .field("activated", &Activated(&self.activated)) - .field("deactivated", &self.deactivated) - .finish() - } -} +pub type SubsystemInstance = polkadot_overseer::gen::SubsystemInstance; -/// Signals sent by an overseer to a subsystem. -#[derive(PartialEq, Clone, Debug)] -pub enum OverseerSignal { - /// Subsystems should adjust their jobs to start and stop work on appropriate block hashes. - ActiveLeaves(ActiveLeavesUpdate), - /// `Subsystem` is informed of a finalized block by its block hash and number. - BlockFinalized(Hash, BlockNumber), - /// Conclude the work of the `Overseer` and all `Subsystem`s. - Conclude, -} +pub type SubsystemInstance = polkadot_overseer::gen::SubsystemInstance; From c17a5909a95c2962d6dbfd34b73620ee7b07652c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 25 Jun 2021 17:42:39 +0200 Subject: [PATCH 069/161] error handling in proc-macro --- Cargo.lock | 47 ++++++++++++------- Cargo.toml | 1 + node/core/runtime-api/src/lib.rs | 4 +- node/network/bridge/src/lib.rs | 4 +- .../collator-protocol/src/collator_side.rs | 2 +- node/network/collator-protocol/src/lib.rs | 2 +- .../collator-protocol/src/validator_side.rs | 2 +- .../proc-macro/src/impl_builder.rs | 4 +- .../overseer-gen/proc-macro/src/impl_misc.rs | 8 ++-- .../proc-macro/src/impl_replace.rs | 4 +- node/overseer/overseer-gen/src/lib.rs | 13 +++-- node/overseer/src/lib.rs | 1 + node/subsystem-util/src/lib.rs | 16 +++++-- 13 files changed, 70 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index efa091c7f705..b52ec24129cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6254,52 +6254,62 @@ dependencies = [ name = "polkadot-node-subsystem" version = "0.1.0" dependencies = [ - "assert_matches", - "async-std", + "polkadot-node-jaeger", + "polkadot-node-subsystem-types", + "polkadot-overseer", + "polkadot-overseer-gen", +] + +[[package]] +name = "polkadot-node-subsystem-test-helpers" +version = "0.1.0" +dependencies = [ "async-trait", - "derive_more", "futures 0.3.14", "futures-timer 3.0.2", - "lazy_static", - "log", - "mick-jaeger", "parity-scale-codec", "parking_lot 0.11.1", "pin-project 1.0.4", - "polkadot-node-jaeger", - "polkadot-node-network-protocol", "polkadot-node-primitives", - "polkadot-node-subsystem-test-helpers", - "polkadot-overseer-gen", + "polkadot-node-subsystem", + "polkadot-node-subsystem-util", + "polkadot-overseer", "polkadot-primitives", "polkadot-statement-table", "sc-network", "smallvec 1.6.1", "sp-core", - "substrate-prometheus-endpoint", - "thiserror", "tracing", ] [[package]] -name = "polkadot-node-subsystem-test-helpers" +name = "polkadot-node-subsystem-types" version = "0.1.0" dependencies = [ + "assert_matches", + "async-std", "async-trait", + "derive_more", "futures 0.3.14", "futures-timer 3.0.2", + "lazy_static", + "log", + "mick-jaeger", "parity-scale-codec", "parking_lot 0.11.1", "pin-project 1.0.4", + "polkadot-node-jaeger", + "polkadot-node-network-protocol", "polkadot-node-primitives", - "polkadot-node-subsystem", - "polkadot-node-subsystem-util", - "polkadot-overseer", + "polkadot-node-subsystem-test-helpers", + "polkadot-overseer-gen", "polkadot-primitives", "polkadot-statement-table", "sc-network", "smallvec 1.6.1", "sp-core", + "substrate-prometheus-endpoint", + "thiserror", "tracing", ] @@ -6321,8 +6331,8 @@ dependencies = [ "polkadot-node-jaeger", "polkadot-node-network-protocol", "polkadot-node-primitives", - "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", + "polkadot-node-subsystem-types", "polkadot-overseer-gen", "polkadot-primitives", "rand 0.8.3", @@ -6349,13 +6359,14 @@ dependencies = [ "lru", "polkadot-node-network-protocol", "polkadot-node-primitives", - "polkadot-node-subsystem", + "polkadot-node-subsystem-types", "polkadot-node-subsystem-util", "polkadot-overseer-gen", "polkadot-primitives", "sc-client-api", "sp-api", "sp-core", + "thiserror", "tracing", ] diff --git a/Cargo.toml b/Cargo.toml index d2f12e9ee28a..dcfd37230601 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,6 +76,7 @@ members = [ "node/primitives", "node/service", "node/subsystem", + "node/subsystem-types", "node/subsystem-test-helpers", "node/subsystem-util", "node/jaeger", diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index 2c267f54cf98..2e293ce4def1 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -23,7 +23,7 @@ #![warn(missing_docs)] use polkadot_overseer::gen::{ - Subsystem, SpawnedSubsystem, SubsystemResult, SubsystemError as OverseerError, SubsystemContext, + Subsystem, SpawnedSubsystem, OverseerResult, OverseerError, SubsystemContext, FromOverseer, }; use polkadot_subsystem::{OverseerSignal, errors::RuntimeApiError, messages::{ @@ -265,7 +265,7 @@ impl RuntimeApiSubsystem where async fn run( mut ctx: Context, mut subsystem: RuntimeApiSubsystem, -) -> SubsystemResult<()> where +) -> OverseerResult<()> where Client: ProvideRuntimeApi + Send + Sync + 'static, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, Context: SubsystemContext diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index c74cf4c807d2..0700dfe45d3a 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -32,8 +32,8 @@ use polkadot_overseer::{AllMessages, OverseerSignal}; use polkadot_overseer::gen::{ FromOverseer, SpawnedSubsystem, - Subsystem, SubsystemContext, SubsystemError as OverseerError, - SubsystemResult as OverseerResult, SubsystemSender, + Subsystem, SubsystemContext, OverseerError, + OverseerResult as OverseerResult, SubsystemSender, }; use polkadot_subsystem::{ errors::{SubsystemError, SubsystemResult}, diff --git a/node/network/collator-protocol/src/collator_side.rs b/node/network/collator-protocol/src/collator_side.rs index 4e060e8cb69c..a866e1717b51 100644 --- a/node/network/collator-protocol/src/collator_side.rs +++ b/node/network/collator-protocol/src/collator_side.rs @@ -30,7 +30,7 @@ use polkadot_overseer::{ AllMessages, OverseerSignal, gen::{ - Subsystem, SpawnedSubsystem, SubsystemResult, SubsystemError as OverseerError, SubsystemContext, + Subsystem, SpawnedSubsystem, OverseerResult, OverseerError, SubsystemContext, FromOverseer, }, }; diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 26f9c03cb273..7db75f637fce 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -36,7 +36,7 @@ use polkadot_overseer::{ AllMessages, OverseerSignal, gen::{ - Subsystem, SpawnedSubsystem, SubsystemResult, SubsystemError as OverseerError, SubsystemContext, + Subsystem, SpawnedSubsystem, OverseerResult, OverseerError, SubsystemContext, FromOverseer, }, }; diff --git a/node/network/collator-protocol/src/validator_side.rs b/node/network/collator-protocol/src/validator_side.rs index 4ab9b06cbbc3..34e7fe6abba4 100644 --- a/node/network/collator-protocol/src/validator_side.rs +++ b/node/network/collator-protocol/src/validator_side.rs @@ -42,7 +42,7 @@ use polkadot_primitives::v1::{CandidateReceipt, CollatorId, Hash, Id as ParaId}; use polkadot_overseer::{ AllMessages, gen::{ - Subsystem, SpawnedSubsystem, SubsystemResult, SubsystemError as OverseerError, SubsystemContext, + Subsystem, SpawnedSubsystem, OverseerResult, OverseerError, SubsystemContext, FromOverseer, }, }; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index fb91ca5c858a..8bd103b48fcb 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -286,7 +286,7 @@ pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result >>, - ) -> ::std::result::Result, #error_ty> + ) -> ::std::result::Result, #error_ty > where S: ::polkadot_overseer_gen::SpawnNamed, M: std::fmt::Debug + Send + 'static, @@ -295,7 +295,7 @@ pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result, SubSys: ::polkadot_overseer_gen::Subsystem, { - let ::polkadot_overseer_gen::SpawnedSubsystem { future, name } = s.start(ctx); + let ::polkadot_overseer_gen::SpawnedSubsystem:: { future, name } = s.start(ctx); let (tx, rx) = ::polkadot_overseer_gen::oneshot::channel(); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index 7945ee916d52..c63298e3ffeb 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -12,6 +12,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result let consumes = &info.consumes(); let signal = &info.extern_signal_ty; let wrapper_message = &info.message_wrapper; + let error_ty = &info.extern_error_ty; let ts = quote! { /// Connector to send messages towards all subsystems, @@ -128,6 +129,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result type Signal = #signal; type Sender = #subsystem_sender_name; type AllMessages = #wrapper_message; + type Error = #error_ty; async fn try_recv(&mut self) -> ::std::result::Result>, ()> { match ::polkadot_overseer_gen::poll!(self.recv()) { @@ -136,7 +138,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result } } - async fn recv(&mut self) -> ::polkadot_overseer_gen::OverseerResult> { + async fn recv(&mut self) -> ::std::result::Result, #error_ty> { loop { // If we have a message pending an overseer signal, we only poll for signals // in the meantime. @@ -206,7 +208,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn(ctx, name, fut)")] async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) - -> ::polkadot_overseer_gen::OverseerResult<()> + -> ::std::result::Result<(), #error_ty> { self.to_overseer.send(::polkadot_overseer_gen::ToOverseer::SpawnJob { name, @@ -216,7 +218,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn_blocking(ctx, name, fut)")] async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) - -> ::polkadot_overseer_gen::OverseerResult<()> + -> ::std::result::Result<(), #error_ty> { self.to_overseer.send(::polkadot_overseer_gen::ToOverseer::SpawnBlockingJob { name, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs index f3298b7dc1c1..72f99d09062b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs @@ -12,6 +12,8 @@ pub(crate) fn impl_replacable_subsystem( let overseer_name = &info.overseer_name; + let error_ty = &info.error_ty; + let field_ty = &info.builder_generic_types(); let baggage_generic_ty = &info.baggage_generic_types(); @@ -23,7 +25,7 @@ pub(crate) fn impl_replacable_subsystem( let where_clause = quote! { where - Ctx: SubsystemContext, + Ctx: SubsystemContext, S: ::polkadot_overseer_gen::SpawnNamed, #( #field_ty : Subsystem, )* }; diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 00f04057ae2e..661e2c1dbfa1 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -369,7 +369,8 @@ pub trait SubsystemContext: Send + 'static { type AllMessages: From + Send + 'static; /// The sender type as provided by `sender()` and underlying. type Sender: SubsystemSender + SubsystemSender + std::fmt::Debug + Send; - + /// The error type. + type Error: ::std::error::Error + ::std::convert::From< OverseerError > + Sync + Send + 'static; /// Try to asynchronously receive a message. /// @@ -378,17 +379,21 @@ pub trait SubsystemContext: Send + 'static { async fn try_recv(&mut self) -> Result>, ()>; /// Receive a message. - async fn recv(&mut self) -> OverseerResult>; + async fn recv(&mut self) -> Result, Self::Error>; /// Spawn a child task on the executor. - async fn spawn(&mut self, name: &'static str, s: ::std::pin::Pin + Send>>) -> OverseerResult<()>; + async fn spawn( + &mut self, + name: &'static str, + s: ::std::pin::Pin + Send>> + ) -> Result<(), Self::Error>; /// Spawn a blocking child task on the executor's dedicated thread pool. async fn spawn_blocking( &mut self, name: &'static str, s: ::std::pin::Pin + Send>>, - ) -> OverseerResult<()>; + ) -> Result<(), Self::Error>; /// Send a direct message to some other `Subsystem`, routed based on message type. async fn send_message(&mut self, msg: X) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 0bddaa288de5..c7263b2ab0e9 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -74,6 +74,7 @@ use futures::{ }; use lru::LruCache; +use polkadot_node_subsystem_util::OverseerError; use polkadot_primitives::v1::{Block, BlockId,BlockNumber, Hash, ParachainHost}; use client::{BlockImportNotification, BlockchainEvents, FinalityNotification}; use sp_api::{ApiExt, ProvideRuntimeApi}; diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index c988d2e3c2ab..f0b1bb52249f 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -28,6 +28,10 @@ use polkadot_node_subsystem::{ errors::RuntimeApiError, messages::{RuntimeApiMessage, RuntimeApiRequest, RuntimeApiSender, BoundToRelayParent}, ActiveLeavesUpdate, OverseerSignal, + errors::{ + SubsystemError, + SubsystemResult, + }, }; pub use polkadot_overseer_gen::{ @@ -99,7 +103,7 @@ pub enum Error { Mpsc(#[from] mpsc::SendError), /// A subsystem error #[error(transparent)] - Subsystem(#[from] OverseerError), + Subsystem(#[from] SubsystemError), /// An error in the Runtime API. #[error(transparent)] RuntimeApi(#[from] RuntimeApiError), @@ -117,6 +121,12 @@ pub enum Error { AlreadyForwarding, } +impl From for Error { + fn from(e: OverseerError) -> Self { + Self::from(SubsystemError::from(e)) + } +} + /// A type alias for Runtime API receivers. pub type RuntimeApiReceiver = oneshot::Receiver>; @@ -783,7 +793,7 @@ impl JobSubsystem { } } -impl Subsystem for JobSubsystem +impl Subsystem for JobSubsystem where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext, @@ -792,7 +802,7 @@ where Job::ToJob: Sync + From, Job::Metrics: Sync, { - fn start(self, ctx: Context) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { self.run(ctx).await; Ok(()) From e6834108366346db9cf970f10c6aa6e3b90f1cc0 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 25 Jun 2021 18:08:12 +0200 Subject: [PATCH 070/161] more OverseerError -> SubsystemError --- node/core/runtime-api/src/lib.rs | 16 ++++++++-------- node/network/bridge/src/lib.rs | 2 +- .../collator-protocol/src/collator_side.rs | 2 +- node/network/collator-protocol/src/error.rs | 2 +- node/network/collator-protocol/src/lib.rs | 6 +++--- .../collator-protocol/src/validator_side.rs | 2 +- .../overseer-gen/proc-macro/src/impl_overseer.rs | 8 ++++++-- node/overseer/overseer-gen/src/lib.rs | 2 +- node/overseer/src/subsystems.rs | 7 ++++--- node/subsystem/src/lib.rs | 10 ++++------ 10 files changed, 30 insertions(+), 27 deletions(-) diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index 2e293ce4def1..daae21fb73c4 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -22,10 +22,10 @@ #![deny(unused_crate_dependencies)] #![warn(missing_docs)] -use polkadot_overseer::gen::{ - Subsystem, SpawnedSubsystem, OverseerResult, OverseerError, SubsystemContext, +use polkadot_overseer::{SubsystemError, SubsystemResult, gen::{ + Subsystem, SpawnedSubsystem, OverseerError, SubsystemContext, FromOverseer, -}; +}}; use polkadot_subsystem::{OverseerSignal, errors::RuntimeApiError, messages::{ RuntimeApiMessage, RuntimeApiRequest as Request, }}; @@ -81,12 +81,12 @@ impl RuntimeApiSubsystem { } } -impl Subsystem for RuntimeApiSubsystem where +impl Subsystem for RuntimeApiSubsystem where Client: ProvideRuntimeApi + Send + 'static + Sync, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Context: SubsystemContext + Context: SubsystemContext, { - fn start(self, ctx: Context) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { SpawnedSubsystem { future: run(ctx, self).boxed(), name: "runtime-api-subsystem", @@ -265,10 +265,10 @@ impl RuntimeApiSubsystem where async fn run( mut ctx: Context, mut subsystem: RuntimeApiSubsystem, -) -> OverseerResult<()> where +) -> SubsystemResult<()> where Client: ProvideRuntimeApi + Send + Sync + 'static, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Context: SubsystemContext + Context: SubsystemContext { loop { select! { diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 0700dfe45d3a..82b0ff5ed08a 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -33,7 +33,7 @@ use polkadot_overseer::gen::{ FromOverseer, SpawnedSubsystem, Subsystem, SubsystemContext, OverseerError, - OverseerResult as OverseerResult, SubsystemSender, + SubsystemSender, }; use polkadot_subsystem::{ errors::{SubsystemError, SubsystemResult}, diff --git a/node/network/collator-protocol/src/collator_side.rs b/node/network/collator-protocol/src/collator_side.rs index a866e1717b51..34d1393ea8da 100644 --- a/node/network/collator-protocol/src/collator_side.rs +++ b/node/network/collator-protocol/src/collator_side.rs @@ -30,7 +30,7 @@ use polkadot_overseer::{ AllMessages, OverseerSignal, gen::{ - Subsystem, SpawnedSubsystem, OverseerResult, OverseerError, SubsystemContext, + Subsystem, SpawnedSubsystem, OverseerError, SubsystemContext, FromOverseer, }, }; diff --git a/node/network/collator-protocol/src/error.rs b/node/network/collator-protocol/src/error.rs index 36716ee0b628..e2e636aed9e4 100644 --- a/node/network/collator-protocol/src/error.rs +++ b/node/network/collator-protocol/src/error.rs @@ -19,7 +19,7 @@ use polkadot_node_primitives::UncheckedSignedFullStatement; use polkadot_subsystem::errors::SubsystemError; -use polkadot_overseer::SubsystemError as OverseerError; +use polkadot_overseer::OverseerError; use thiserror::Error; use polkadot_node_subsystem_util::{Fault, runtime, unwrap_non_fatal}; diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 7db75f637fce..3e4d1c4f3aab 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -36,7 +36,7 @@ use polkadot_overseer::{ AllMessages, OverseerSignal, gen::{ - Subsystem, SpawnedSubsystem, OverseerResult, OverseerError, SubsystemContext, + Subsystem, SpawnedSubsystem, OverseerError, SubsystemContext, FromOverseer, }, }; @@ -126,9 +126,9 @@ impl CollatorProtocolSubsystem { } } -impl Subsystem for CollatorProtocolSubsystem +impl Subsystem for CollatorProtocolSubsystem where - Context: SubsystemContext + Sync + Send, + Context: SubsystemContext + Sync + Send, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self diff --git a/node/network/collator-protocol/src/validator_side.rs b/node/network/collator-protocol/src/validator_side.rs index 34e7fe6abba4..124c189299fd 100644 --- a/node/network/collator-protocol/src/validator_side.rs +++ b/node/network/collator-protocol/src/validator_side.rs @@ -42,7 +42,7 @@ use polkadot_primitives::v1::{CandidateReceipt, CollatorId, Hash, Id as ParaId}; use polkadot_overseer::{ AllMessages, gen::{ - Subsystem, SpawnedSubsystem, OverseerResult, OverseerError, SubsystemContext, + Subsystem, SpawnedSubsystem, OverseerError, SubsystemContext, FromOverseer, }, }; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 904200c4ac3b..ec0e56bd84fe 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -189,7 +189,9 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { - Err(::polkadot_overseer_gen::OverseerError::SubsystemStalled(instance.name)) + Err(#error_ty :: from( + ::polkadot_overseer_gen::OverseerError::SubsystemStalled(instance.name) + )) } Some(res) => res.map_err(Into::into), } @@ -207,7 +209,9 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { - Err(::polkadot_overseer_gen::OverseerError::SubsystemStalled(instance.name)) + Err(#error_ty :: from( + ::polkadot_overseer_gen::OverseerError::SubsystemStalled(instance.name) + )) } Some(res) => { let res = res.map_err(Into::into); diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 661e2c1dbfa1..fcf39e7840a4 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -224,7 +224,7 @@ pub trait AnnotateErrorOrigin: 'static + Send + Sync + std::error::Error { /// An asynchronous subsystem task.. /// /// In essence it's just a newtype wrapping a `BoxFuture`. -pub struct SpawnedSubsystem +pub struct SpawnedSubsystem where E: std::error::Error + Send diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index 8bb375298f87..bced4a4837b0 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -1,4 +1,5 @@ +use polkadot_node_subsystem::errors::SubsystemError; use ::polkadot_overseer_gen::{ MapSubsystem, SubsystemContext, Subsystem, @@ -14,11 +15,11 @@ use crate::OverseerSignal; #[derive(Clone, Copy, Debug)] pub struct DummySubsystem; -impl Subsystem for DummySubsystem +impl Subsystem for DummySubsystem where - Context: SubsystemContext, + Context: SubsystemContext, { - fn start(self, mut ctx: Context) -> SpawnedSubsystem { + fn start(self, mut ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { loop { match ctx.recv().await { diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 8d22545643b0..aef2061ce055 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -37,9 +37,9 @@ pub use polkadot_node_subsystem_types::{ pub use polkadot_overseer::{ AllMessages, Overseer, + SubsystemError, gen::{ OverseerError, - OverseerResult, SubsystemContext, SubsystemMeters, SubsystemMeterReadouts, @@ -49,12 +49,10 @@ pub use polkadot_overseer::{ // Simplify usage without having to do large scale modifications of all // subsystems at once. -pub type SubssystemSender = polkadot_overseer::gen::SubssystemSender; +pub type SubsystemSender = polkadot_overseer::gen::SubsystemSender; -pub type FromOverseer = polkadot_overseer::gen::FromOverseer; +pub type FromOverseer = polkadot_overseer::gen::FromOverseer; -pub type Subsystem = polkadot_overseer::gen::Subsystem; - -pub type SubsystemInstance = polkadot_overseer::gen::SubsystemInstance; +pub type Subsystem = polkadot_overseer::gen::Subsystem; pub type SubsystemInstance = polkadot_overseer::gen::SubsystemInstance; From 6858084c2fcd27163141cc2aa582fe10a8015ef9 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 25 Jun 2021 18:11:49 +0200 Subject: [PATCH 071/161] locky --- Cargo.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index b52ec24129cd..6b9e0019acd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6366,7 +6366,6 @@ dependencies = [ "sc-client-api", "sp-api", "sp-core", - "thiserror", "tracing", ] From 5ae073a1b762056d1a46a15e11d5b40179bc1b8d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 28 Jun 2021 15:25:13 +0200 Subject: [PATCH 072/161] all? --- Cargo.lock | 339 +-- node/core/approval-voting/Cargo.toml | 5 +- .../approval-voting/src/approval_checking.rs | 439 +++- node/core/approval-voting/src/import.rs | 1124 +--------- node/core/approval-voting/src/lib.rs | 743 ++++--- .../approval-voting/src/persisted_entries.rs | 4 +- node/core/approval-voting/src/tests.rs | 29 +- node/core/av-store/Cargo.toml | 4 +- node/core/av-store/src/lib.rs | 151 +- node/core/av-store/src/tests.rs | 172 +- node/core/backing/Cargo.toml | 4 +- node/core/backing/src/lib.rs | 1612 +------------- node/core/backing/src/tests.rs | 1567 ++++++++++++++ node/core/bitfield-signing/Cargo.toml | 2 +- node/core/bitfield-signing/src/lib.rs | 10 +- node/core/candidate-validation/Cargo.toml | 4 +- node/core/candidate-validation/src/lib.rs | 643 +----- node/core/candidate-validation/src/tests.rs | 633 ++++++ node/core/chain-api/Cargo.toml | 9 +- node/core/chain-api/src/lib.rs | 121 +- node/core/chain-selection/Cargo.toml | 23 + node/core/chain-selection/src/backend.rs | 235 ++ node/core/chain-selection/src/lib.rs | 574 +++++ node/core/chain-selection/src/tests.rs | 1909 +++++++++++++++++ node/core/chain-selection/src/tree.rs | 584 +++++ node/core/dispute-coordinator/Cargo.toml | 29 + node/core/dispute-coordinator/src/db/mod.rs | 19 + node/core/dispute-coordinator/src/db/v1.rs | 585 +++++ node/core/dispute-coordinator/src/lib.rs | 647 ++++++ node/core/dispute-coordinator/src/tests.rs | 706 ++++++ node/core/dispute-participation/Cargo.toml | 20 + node/core/dispute-participation/src/lib.rs | 372 ++++ node/core/dispute-participation/src/tests.rs | 425 ++++ node/core/parachains-inherent/Cargo.toml | 2 +- node/core/provisioner/Cargo.toml | 2 +- node/core/provisioner/src/lib.rs | 23 +- node/core/pvf/Cargo.toml | 4 +- node/core/pvf/src/artifacts.rs | 11 +- node/core/pvf/src/executor_intf.rs | 6 +- node/core/pvf/src/pvf.rs | 4 +- node/core/runtime-api/Cargo.toml | 5 +- node/core/runtime-api/src/cache.rs | 35 +- node/core/runtime-api/src/lib.rs | 860 +------- node/core/runtime-api/src/tests.rs | 790 +++++++ node/primitives/Cargo.toml | 3 +- node/primitives/src/approval.rs | 4 - node/primitives/src/lib.rs | 144 +- primitives/Cargo.toml | 2 +- primitives/src/v0.rs | 8 + primitives/src/v1/mod.rs | 172 +- 50 files changed, 11039 insertions(+), 4779 deletions(-) create mode 100644 node/core/backing/src/tests.rs create mode 100644 node/core/candidate-validation/src/tests.rs create mode 100644 node/core/chain-selection/Cargo.toml create mode 100644 node/core/chain-selection/src/backend.rs create mode 100644 node/core/chain-selection/src/lib.rs create mode 100644 node/core/chain-selection/src/tests.rs create mode 100644 node/core/chain-selection/src/tree.rs create mode 100644 node/core/dispute-coordinator/Cargo.toml create mode 100644 node/core/dispute-coordinator/src/db/mod.rs create mode 100644 node/core/dispute-coordinator/src/db/v1.rs create mode 100644 node/core/dispute-coordinator/src/lib.rs create mode 100644 node/core/dispute-coordinator/src/tests.rs create mode 100644 node/core/dispute-participation/Cargo.toml create mode 100644 node/core/dispute-participation/src/lib.rs create mode 100644 node/core/dispute-participation/src/tests.rs create mode 100644 node/core/runtime-api/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 6b9e0019acd1..a2dcbc04fe1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -494,7 +494,7 @@ version = "0.1.0" source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" dependencies = [ "beefy-primitives", - "futures 0.3.14", + "futures 0.3.15", "hex", "log", "parity-scale-codec", @@ -523,7 +523,7 @@ source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master# dependencies = [ "beefy-gadget", "beefy-primitives", - "futures 0.3.14", + "futures 0.3.15", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -554,8 +554,8 @@ dependencies = [ name = "behavior-tests" version = "0.1.0" dependencies = [ - "env_logger 0.8.2", - "futures 0.3.14", + "env_logger 0.8.4", + "futures 0.3.15", "log", "polkadot-service", "sc-chain-spec", @@ -1171,6 +1171,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "core-foundation" version = "0.7.0" @@ -1486,10 +1492,11 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.11" +version = "0.99.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" +checksum = "5cc7b9cef1e351660e5443924e4f43ab25fbbed3e9a5f052df3677deb4d6b320" dependencies = [ + "convert_case", "proc-macro2", "quote", "syn", @@ -1711,9 +1718,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" +checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" dependencies = [ "atty", "humantime 2.0.1", @@ -1797,7 +1804,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e43f2f1833d64e33f15592464d6fdd70f349dda7b1a53088eb83cd94014008c5" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", ] [[package]] @@ -1895,7 +1902,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6447e2f8178843749e8c8003206def83ec124a7859475395777a28b5338647c" dependencies = [ "either", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "log", "num-traits", @@ -2242,9 +2249,9 @@ checksum = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" [[package]] name = "futures" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253" +checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" dependencies = [ "futures-channel", "futures-core", @@ -2257,9 +2264,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" +checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" dependencies = [ "futures-core", "futures-sink", @@ -2267,9 +2274,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" +checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" [[package]] name = "futures-cpupool" @@ -2288,7 +2295,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdcef58a173af8148b182684c9f2d5250875adbcaff7b5794073894f9d8634a9" dependencies = [ "futures 0.1.29", - "futures 0.3.14", + "futures 0.3.15", "lazy_static", "log", "parking_lot 0.9.0", @@ -2299,9 +2306,9 @@ dependencies = [ [[package]] name = "futures-executor" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d" +checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79" dependencies = [ "futures-core", "futures-task", @@ -2311,9 +2318,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" +checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" [[package]] name = "futures-lite" @@ -2332,10 +2339,11 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" +checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" dependencies = [ + "autocfg", "proc-macro-hack", "proc-macro2", "quote", @@ -2355,15 +2363,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" +checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" [[package]] name = "futures-task" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" +checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" [[package]] name = "futures-timer" @@ -2383,10 +2391,11 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" +checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" dependencies = [ + "autocfg", "futures 0.1.29", "futures-channel", "futures-core", @@ -2802,7 +2811,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project 1.0.4", + "pin-project 1.0.7", "socket2 0.3.17", "tokio 0.2.21", "tower-service", @@ -2878,7 +2887,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6d52908d4ea4ab2bc22474ba149bf1011c8e2c3ebc1ff593ae28ac44f494b6" dependencies = [ "async-io", - "futures 0.3.14", + "futures 0.3.15", "futures-lite", "if-addrs", "ipnet", @@ -2971,7 +2980,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64fa110ec7b8f493f416eed552740d10e7030ad5f63b2308f82c9608ec2df275" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "futures-timer 2.0.2", ] @@ -3232,10 +3241,10 @@ dependencies = [ "async-tls", "async-trait", "fnv", - "futures 0.3.14", + "futures 0.3.15", "jsonrpsee-types", "log", - "pin-project 1.0.4", + "pin-project 1.0.7", "serde", "serde_json", "soketto", @@ -3404,7 +3413,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb1e98ba343d0b35f9009a8844cd2b87fa3192f7e79033ac05b00aeae0f3b0b5" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "js-sys", "kvdb", "kvdb-memorydb", @@ -3474,7 +3483,7 @@ checksum = "08053fbef67cd777049ef7a95ebaca2ece370b4ed7712c3fa404d69a88cb741b" dependencies = [ "atomic", "bytes 1.0.1", - "futures 0.3.14", + "futures 0.3.15", "lazy_static", "libp2p-core", "libp2p-deflate", @@ -3500,7 +3509,7 @@ dependencies = [ "libp2p-yamux", "parity-multiaddr", "parking_lot 0.11.1", - "pin-project 1.0.4", + "pin-project 1.0.7", "smallvec 1.6.1", "wasm-timer", ] @@ -3516,7 +3525,7 @@ dependencies = [ "ed25519-dalek", "either", "fnv", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "lazy_static", "libsecp256k1", @@ -3525,7 +3534,7 @@ dependencies = [ "multistream-select", "parity-multiaddr", "parking_lot 0.11.1", - "pin-project 1.0.4", + "pin-project 1.0.7", "prost", "prost-build", "rand 0.7.3", @@ -3546,7 +3555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2181a641cd15f9b6ba71b1335800f309012a0a97a29ffaabbbf40e9d3d58f08" dependencies = [ "flate2", - "futures 0.3.14", + "futures 0.3.15", "libp2p-core", ] @@ -3557,7 +3566,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62e63dab8b5ff35e0c101a3e51e843ba782c07bbb1682f5fd827622e0d02b98b" dependencies = [ "async-std-resolver", - "futures 0.3.14", + "futures 0.3.15", "libp2p-core", "log", "smallvec 1.6.1", @@ -3572,7 +3581,7 @@ checksum = "48a9b570f6766301d9c4aa00fce3554cad1598e2f466debbc4dde909028417cf" dependencies = [ "cuckoofilter", "fnv", - "futures 0.3.14", + "futures 0.3.15", "libp2p-core", "libp2p-swarm", "log", @@ -3593,7 +3602,7 @@ dependencies = [ "byteorder", "bytes 1.0.1", "fnv", - "futures 0.3.14", + "futures 0.3.15", "hex_fmt", "libp2p-core", "libp2p-swarm", @@ -3614,7 +3623,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f668f00efd9883e8b7bcc582eaf0164615792608f886f6577da18bcbeea0a46" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "libp2p-core", "libp2p-swarm", "log", @@ -3635,7 +3644,7 @@ dependencies = [ "bytes 1.0.1", "either", "fnv", - "futures 0.3.14", + "futures 0.3.15", "libp2p-core", "libp2p-swarm", "log", @@ -3659,7 +3668,7 @@ dependencies = [ "async-io", "data-encoding", "dns-parser", - "futures 0.3.14", + "futures 0.3.15", "if-watch", "lazy_static", "libp2p-core", @@ -3679,7 +3688,7 @@ checksum = "85e9b544335d1ed30af71daa96edbefadef6f19c7a55f078b9fc92c87163105d" dependencies = [ "asynchronous-codec 0.6.0", "bytes 1.0.1", - "futures 0.3.14", + "futures 0.3.15", "libp2p-core", "log", "nohash-hasher", @@ -3697,7 +3706,7 @@ checksum = "36db0f0db3b0433f5b9463f1c0cd9eadc0a3734a9170439ce501ff99733a88bd" dependencies = [ "bytes 1.0.1", "curve25519-dalek 3.0.0", - "futures 0.3.14", + "futures 0.3.15", "lazy_static", "libp2p-core", "log", @@ -3717,7 +3726,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4bfaffac63bf3c7ec11ed9d8879d455966ddea7e78ee14737f0b6dce0d1cd1" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "libp2p-core", "libp2p-swarm", "log", @@ -3734,7 +3743,7 @@ checksum = "0c8c37b4d2a075b4be8442760a5f8c037180f0c8dd5b5734b9978ab868b3aa11" dependencies = [ "asynchronous-codec 0.6.0", "bytes 1.0.1", - "futures 0.3.14", + "futures 0.3.15", "libp2p-core", "log", "prost", @@ -3749,9 +3758,9 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce3374f3b28162db9d3442c9347c4f14cb01e8290052615c7d341d40eae0599" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "log", - "pin-project 1.0.4", + "pin-project 1.0.7", "rand 0.7.3", "salsa20", "sha3", @@ -3765,12 +3774,12 @@ checksum = "0b8786aca3f18671d8776289706a5521f6c9124a820f69e358de214b9939440d" dependencies = [ "asynchronous-codec 0.6.0", "bytes 1.0.1", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "libp2p-core", "libp2p-swarm", "log", - "pin-project 1.0.4", + "pin-project 1.0.7", "prost", "prost-build", "rand 0.7.3", @@ -3788,7 +3797,7 @@ checksum = "1cdbe172f08e6d0f95fa8634e273d4c4268c4063de2e33e7435194b0130c62e3" dependencies = [ "async-trait", "bytes 1.0.1", - "futures 0.3.14", + "futures 0.3.15", "libp2p-core", "libp2p-swarm", "log", @@ -3807,7 +3816,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e04d8e1eef675029ec728ba14e8d0da7975d84b6679b699b4ae91a1de9c3a92" dependencies = [ "either", - "futures 0.3.14", + "futures 0.3.15", "libp2p-core", "log", "rand 0.7.3", @@ -3833,7 +3842,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b1a27d21c477951799e99d5c105d78868258502ce092988040a808d5a19bbd9" dependencies = [ "async-io", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "if-watch", "ipnet", @@ -3850,7 +3859,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffd6564bb3b7ff203661ccbb69003c2b551e34cef974f2d6c6a28306a12170b5" dependencies = [ "async-std", - "futures 0.3.14", + "futures 0.3.15", "libp2p-core", "log", ] @@ -3861,7 +3870,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cef45d61e43c313531b5e903e4e8415212ff6338e0c54c47da5b9b412b5760de" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "js-sys", "libp2p-core", "parity-send-wrapper", @@ -3876,7 +3885,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cace60995ef6f637e4752cccbb2590f6bc358e8741a0d066307636c69a4b3a74" dependencies = [ "either", - "futures 0.3.14", + "futures 0.3.15", "futures-rustls", "libp2p-core", "log", @@ -3893,7 +3902,7 @@ version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f35da42cfc6d5cb0dcf3ad6881bc68d146cdf38f98655e09e33fbba4d13eabc4" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "libp2p-core", "parking_lot 0.11.1", "thiserror", @@ -4174,7 +4183,7 @@ version = "0.1.0" dependencies = [ "assert_matches", "derive_more", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", ] @@ -4184,7 +4193,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c023c3f16109e7f33aa451f773fd61070e265b4977d0b6e344a51049296dd7df" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "rand 0.7.3", "thrift", ] @@ -4355,9 +4364,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10ddc0eb0117736f19d556355464fc87efc8ad98b29e3fd84f02531eb6e90840" dependencies = [ "bytes 1.0.1", - "futures 0.3.14", + "futures 0.3.15", "log", - "pin-project 1.0.4", + "pin-project 1.0.7", "smallvec 1.6.1", "unsigned-varint 0.6.0", ] @@ -5649,11 +5658,11 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.4" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2" +checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" dependencies = [ - "pin-project-internal 1.0.4", + "pin-project-internal 1.0.7", ] [[package]] @@ -5669,9 +5678,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.4" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa25a6393f22ce819b0f50e0be89287292fda8d425be38ee0ca14c4931d9e71" +checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" dependencies = [ "proc-macro2", "quote", @@ -5714,7 +5723,7 @@ version = "0.9.4" dependencies = [ "assert_cmd", "color-eyre", - "futures 0.3.14", + "futures 0.3.15", "nix", "parity-util-mem", "polkadot-cli", @@ -5728,8 +5737,8 @@ name = "polkadot-approval-distribution" version = "0.1.0" dependencies = [ "assert_matches", - "env_logger 0.8.2", - "futures 0.3.14", + "env_logger 0.8.4", + "futures 0.3.15", "log", "polkadot-node-network-protocol", "polkadot-node-primitives", @@ -5749,8 +5758,8 @@ version = "0.1.0" dependencies = [ "assert_matches", "bitvec", - "env_logger 0.8.2", - "futures 0.3.14", + "env_logger 0.8.4", + "futures 0.3.15", "log", "maplit", "polkadot-node-network-protocol", @@ -5769,7 +5778,7 @@ name = "polkadot-availability-distribution" version = "0.1.0" dependencies = [ "assert_matches", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "lru", "maplit", @@ -5800,8 +5809,8 @@ name = "polkadot-availability-recovery" version = "0.1.0" dependencies = [ "assert_matches", - "env_logger 0.8.2", - "futures 0.3.14", + "env_logger 0.8.4", + "futures 0.3.15", "futures-timer 3.0.2", "log", "lru", @@ -5828,7 +5837,7 @@ name = "polkadot-cli" version = "0.9.4" dependencies = [ "frame-benchmarking-cli", - "futures 0.3.14", + "futures 0.3.15", "log", "polkadot-node-core-pvf", "polkadot-service", @@ -5851,8 +5860,8 @@ version = "0.1.0" dependencies = [ "always-assert", "assert_matches", - "env_logger 0.8.2", - "futures 0.3.14", + "env_logger 0.8.4", + "futures 0.3.15", "futures-timer 3.0.2", "log", "polkadot-node-network-protocol", @@ -5899,7 +5908,7 @@ name = "polkadot-gossip-support" version = "0.1.0" dependencies = [ "assert_matches", - "futures 0.3.14", + "futures 0.3.15", "polkadot-node-network-protocol", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", @@ -5919,7 +5928,7 @@ version = "0.1.0" dependencies = [ "assert_matches", "async-trait", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "parity-scale-codec", "parking_lot 0.11.1", @@ -5942,7 +5951,7 @@ dependencies = [ name = "polkadot-node-collation-generation" version = "0.1.0" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "parity-scale-codec", "polkadot-erasure-coding", "polkadot-node-primitives", @@ -5963,10 +5972,11 @@ dependencies = [ "assert_matches", "bitvec", "derive_more", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "kvdb", "kvdb-memorydb", + "lru", "maplit", "merlin", "parity-scale-codec", @@ -6000,8 +6010,8 @@ version = "0.1.0" dependencies = [ "assert_matches", "bitvec", - "env_logger 0.8.2", - "futures 0.3.14", + "env_logger 0.8.4", + "futures 0.3.15", "futures-timer 3.0.2", "kvdb", "kvdb-memorydb", @@ -6027,7 +6037,7 @@ version = "0.1.0" dependencies = [ "assert_matches", "bitvec", - "futures 0.3.14", + "futures 0.3.15", "polkadot-erasure-coding", "polkadot-node-primitives", "polkadot-node-subsystem", @@ -6049,7 +6059,7 @@ dependencies = [ name = "polkadot-node-core-bitfield-signing" version = "0.1.0" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", @@ -6064,7 +6074,7 @@ dependencies = [ name = "polkadot-node-core-candidate-selection" version = "0.1.0" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", @@ -6082,7 +6092,7 @@ version = "0.1.0" dependencies = [ "assert_matches", "async-trait", - "futures 0.3.14", + "futures 0.3.15", "parity-scale-codec", "polkadot-node-core-pvf", "polkadot-node-primitives", @@ -6101,13 +6111,16 @@ dependencies = [ name = "polkadot-node-core-chain-api" version = "0.1.0" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "maplit", + "parity-scale-codec", + "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", - "polkadot-overseer", "polkadot-primitives", + "sc-client-api", + "sc-consensus-babe", "sp-blockchain", "sp-core", "tracing", @@ -6118,7 +6131,7 @@ name = "polkadot-node-core-parachains-inherent" version = "0.1.0" dependencies = [ "async-trait", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "polkadot-node-subsystem", "polkadot-overseer", @@ -6135,7 +6148,7 @@ name = "polkadot-node-core-provisioner" version = "0.1.0" dependencies = [ "bitvec", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", @@ -6155,12 +6168,12 @@ dependencies = [ "assert_matches", "async-process", "async-std", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "hex-literal", "libc", "parity-scale-codec", - "pin-project 1.0.4", + "pin-project 1.0.7", "polkadot-core-primitives", "polkadot-parachain", "rand 0.8.3", @@ -6182,14 +6195,13 @@ dependencies = [ name = "polkadot-node-core-runtime-api" version = "0.1.0" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "memory-lru", "parity-util-mem", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", - "polkadot-overseer", "polkadot-primitives", "sp-api", "sp-authority-discovery", @@ -6219,7 +6231,7 @@ dependencies = [ name = "polkadot-node-network-protocol" version = "0.1.0" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "parity-scale-codec", "polkadot-node-jaeger", "polkadot-node-primitives", @@ -6233,7 +6245,7 @@ dependencies = [ name = "polkadot-node-primitives" version = "0.1.0" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "parity-scale-codec", "polkadot-parachain", "polkadot-primitives", @@ -6244,6 +6256,7 @@ dependencies = [ "sp-consensus-babe", "sp-consensus-vrf", "sp-core", + "sp-keystore", "sp-maybe-compressed-blob", "sp-runtime", "thiserror", @@ -6265,11 +6278,11 @@ name = "polkadot-node-subsystem-test-helpers" version = "0.1.0" dependencies = [ "async-trait", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "parity-scale-codec", "parking_lot 0.11.1", - "pin-project 1.0.4", + "pin-project 1.0.7", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-util", @@ -6290,14 +6303,14 @@ dependencies = [ "async-std", "async-trait", "derive_more", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "lazy_static", "log", "mick-jaeger", "parity-scale-codec", "parking_lot 0.11.1", - "pin-project 1.0.4", + "pin-project 1.0.7", "polkadot-node-jaeger", "polkadot-node-network-protocol", "polkadot-node-primitives", @@ -6319,15 +6332,15 @@ version = "0.1.0" dependencies = [ "assert_matches", "async-trait", - "env_logger 0.8.2", - "futures 0.3.14", + "env_logger 0.8.4", + "futures 0.3.15", "futures-timer 3.0.2", "log", "lru", "metered-channel", "parity-scale-codec", "parking_lot 0.11.1", - "pin-project 1.0.4", + "pin-project 1.0.7", "polkadot-node-jaeger", "polkadot-node-network-protocol", "polkadot-node-primitives", @@ -6353,7 +6366,7 @@ dependencies = [ "assert_matches", "async-trait", "femme", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "kv-log-macro", "lru", @@ -6374,11 +6387,11 @@ name = "polkadot-overseer-gen" version = "0.1.0" dependencies = [ "async-trait", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "futures-util", "metered-channel", - "pin-project 1.0.4", + "pin-project 1.0.7", "polkadot-node-network-protocol", "polkadot-node-subsystem", "polkadot-overseer-gen-proc-macro", @@ -6417,7 +6430,7 @@ dependencies = [ [[package]] name = "polkadot-primitives" -version = "0.9.4" +version = "0.9.7" dependencies = [ "bitvec", "frame-system", @@ -6623,7 +6636,7 @@ dependencies = [ "frame-support", "frame-support-test", "frame-system", - "futures 0.3.14", + "futures 0.3.15", "hex-literal", "libsecp256k1", "log", @@ -6670,10 +6683,10 @@ version = "0.9.4" dependencies = [ "beefy-gadget", "beefy-primitives", - "env_logger 0.8.2", + "env_logger 0.8.4", "frame-benchmarking", "frame-system-rpc-runtime-api", - "futures 0.3.14", + "futures 0.3.15", "hex-literal", "kusama-runtime", "kvdb", @@ -6763,7 +6776,7 @@ version = "0.1.0" dependencies = [ "arrayvec 0.5.2", "assert_matches", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "indexmap", "parity-scale-codec", @@ -6798,7 +6811,7 @@ dependencies = [ name = "polkadot-test-client" version = "0.9.4" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "parity-scale-codec", "polkadot-node-subsystem", "polkadot-primitives", @@ -6889,7 +6902,7 @@ dependencies = [ "frame-benchmarking", "frame-system", "futures 0.1.29", - "futures 0.3.14", + "futures 0.3.15", "hex", "pallet-balances", "pallet-staking", @@ -7528,7 +7541,7 @@ name = "remote-externalities" version = "0.9.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ - "env_logger 0.8.2", + "env_logger 0.8.4", "hex", "jsonrpsee-proc-macros", "jsonrpsee-ws-client", @@ -7775,7 +7788,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4da5fcb054c46f5a5dff833b129285a93d3f0179531735e6c866e8cc307d2020" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "pin-project 0.4.23", "static_assertions", ] @@ -7821,7 +7834,7 @@ dependencies = [ "async-trait", "derive_more", "either", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "ip_network", "libp2p", @@ -7847,7 +7860,7 @@ name = "sc-basic-authorship" version = "0.9.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "log", "parity-scale-codec", @@ -7920,7 +7933,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364b dependencies = [ "chrono", "fdlimit", - "futures 0.3.14", + "futures 0.3.15", "hex", "libp2p", "log", @@ -7958,7 +7971,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364b dependencies = [ "derive_more", "fnv", - "futures 0.3.14", + "futures 0.3.15", "hash-db", "kvdb", "lazy_static", @@ -8035,7 +8048,7 @@ dependencies = [ "async-trait", "derive_more", "fork-tree", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "log", "merlin", @@ -8079,7 +8092,7 @@ version = "0.9.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ "derive_more", - "futures 0.3.14", + "futures 0.3.15", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -8118,7 +8131,7 @@ dependencies = [ "assert_matches", "async-trait", "derive_more", - "futures 0.3.14", + "futures 0.3.15", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -8151,7 +8164,7 @@ version = "0.9.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ "async-trait", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "impl-trait-for-tuples", "log", @@ -8272,13 +8285,13 @@ dependencies = [ "dyn-clone", "finality-grandpa", "fork-tree", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "linked-hash-map", "log", "parity-scale-codec", "parking_lot 0.11.1", - "pin-project 1.0.4", + "pin-project 1.0.7", "rand 0.7.3", "sc-block-builder", "sc-client-api", @@ -8310,7 +8323,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364b dependencies = [ "derive_more", "finality-grandpa", - "futures 0.3.14", + "futures 0.3.15", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -8333,7 +8346,7 @@ version = "0.9.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ "derive_more", - "futures 0.3.14", + "futures 0.3.15", "log", "num-traits", "parity-scale-codec", @@ -8354,7 +8367,7 @@ version = "0.9.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ "ansi_term 0.12.1", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "log", "parity-util-mem", @@ -8373,7 +8386,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364b dependencies = [ "async-trait", "derive_more", - "futures 0.3.14", + "futures 0.3.15", "futures-util", "hex", "merlin", @@ -8422,7 +8435,7 @@ dependencies = [ "erased-serde", "fnv", "fork-tree", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "hex", "ip_network", @@ -8434,7 +8447,7 @@ dependencies = [ "nohash-hasher", "parity-scale-codec", "parking_lot 0.11.1", - "pin-project 1.0.4", + "pin-project 1.0.7", "prost", "prost-build", "rand 0.7.3", @@ -8463,7 +8476,7 @@ name = "sc-network-gossip" version = "0.9.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "libp2p", "log", @@ -8482,7 +8495,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364b dependencies = [ "bytes 0.5.6", "fnv", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "hex", "hyper 0.13.9", @@ -8508,7 +8521,7 @@ name = "sc-peerset" version = "3.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "libp2p", "log", "serde_json", @@ -8530,7 +8543,7 @@ name = "sc-rpc" version = "3.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "hash-db", "jsonrpc-core", "jsonrpc-pubsub", @@ -8566,7 +8579,7 @@ version = "0.9.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ "derive_more", - "futures 0.3.14", + "futures 0.3.15", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -8612,7 +8625,7 @@ dependencies = [ "directories", "exit-future", "futures 0.1.29", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "hash-db", "jsonrpc-core", @@ -8622,7 +8635,7 @@ dependencies = [ "parity-scale-codec", "parity-util-mem", "parking_lot 0.11.1", - "pin-project 1.0.4", + "pin-project 1.0.7", "rand 0.7.3", "sc-block-builder", "sc-chain-spec", @@ -8709,11 +8722,11 @@ version = "3.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ "chrono", - "futures 0.3.14", + "futures 0.3.15", "libp2p", "log", "parking_lot 0.11.1", - "pin-project 1.0.4", + "pin-project 1.0.7", "rand 0.7.3", "serde", "serde_json", @@ -8777,7 +8790,7 @@ version = "3.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ "derive_more", - "futures 0.3.14", + "futures 0.3.15", "linked-hash-map", "log", "parity-util-mem", @@ -8798,7 +8811,7 @@ name = "sc-transaction-pool" version = "3.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "futures-diagnose", "intervalier", "log", @@ -9216,7 +9229,7 @@ dependencies = [ "base64 0.12.3", "bytes 0.5.6", "flate2", - "futures 0.3.14", + "futures 0.3.15", "httparse", "log", "rand 0.7.3", @@ -9332,7 +9345,7 @@ name = "sp-blockchain" version = "3.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "log", "lru", "parity-scale-codec", @@ -9360,7 +9373,7 @@ version = "0.9.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ "async-trait", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "libp2p", "log", @@ -9435,7 +9448,7 @@ dependencies = [ "byteorder", "dyn-clonable", "ed25519-dalek", - "futures 0.3.14", + "futures 0.3.15", "hash-db", "hash256-std-hasher", "hex", @@ -9536,7 +9549,7 @@ name = "sp-io" version = "3.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "hash-db", "libsecp256k1", "log", @@ -9574,7 +9587,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364b dependencies = [ "async-trait", "derive_more", - "futures 0.3.14", + "futures 0.3.15", "merlin", "parity-scale-codec", "parking_lot 0.11.1", @@ -9824,7 +9837,7 @@ version = "3.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ "derive_more", - "futures 0.3.14", + "futures 0.3.15", "log", "parity-scale-codec", "serde", @@ -9868,7 +9881,7 @@ name = "sp-utils" version = "3.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "futures-core", "futures-timer 3.0.2", "lazy_static", @@ -10065,7 +10078,7 @@ dependencies = [ "chrono", "console_error_panic_hook", "futures 0.1.29", - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "getrandom 0.2.1", "js-sys", @@ -10097,7 +10110,7 @@ version = "3.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ "frame-system-rpc-runtime-api", - "futures 0.3.14", + "futures 0.3.15", "jsonrpc-core", "jsonrpc-core-client", "jsonrpc-derive", @@ -10135,7 +10148,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364b dependencies = [ "async-trait", "futures 0.1.29", - "futures 0.3.14", + "futures 0.3.15", "hash-db", "hex", "parity-scale-codec", @@ -10162,7 +10175,7 @@ name = "substrate-test-utils" version = "3.0.0" source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "substrate-test-utils-derive", "tokio 0.2.21", ] @@ -10286,7 +10299,7 @@ dependencies = [ name = "test-parachain-adder-collator" version = "0.9.4" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "futures-timer 3.0.2", "log", "parity-scale-codec", @@ -10335,7 +10348,7 @@ source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364b dependencies = [ "env_logger 0.7.1", "frame-system", - "futures 0.3.14", + "futures 0.3.15", "jsonrpc-core", "log", "sc-basic-authorship", @@ -11345,7 +11358,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "js-sys", "parking_lot 0.11.1", "pin-utils", @@ -11898,7 +11911,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d9028f208dd5e63c614be69f115c1b53cacc1111437d4c765185856666c107" dependencies = [ - "futures 0.3.14", + "futures 0.3.15", "log", "nohash-hasher", "parking_lot 0.11.1", diff --git a/node/core/approval-voting/Cargo.toml b/node/core/approval-voting/Cargo.toml index 53b6a354ace6..104a52621e9e 100644 --- a/node/core/approval-voting/Cargo.toml +++ b/node/core/approval-voting/Cargo.toml @@ -5,15 +5,16 @@ authors = ["Parity Technologies "] edition = "2018" [dependencies] -futures = "0.3.8" +futures = "0.3.15" futures-timer = "3.0.2" parity-scale-codec = { version = "2.0.0", default-features = false, features = ["bit-vec", "derive"] } tracing = "0.1.26" bitvec = { version = "0.20.1", default-features = false, features = ["alloc"] } +lru = "0.6" merlin = "2.0" schnorrkel = "0.9.1" kvdb = "0.9.0" -derive_more = "0.99.1" +derive_more = "0.99.14" polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } diff --git a/node/core/approval-voting/src/approval_checking.rs b/node/core/approval-voting/src/approval_checking.rs index f5b5f53533f4..0843d574fc41 100644 --- a/node/core/approval-voting/src/approval_checking.rs +++ b/node/core/approval-voting/src/approval_checking.rs @@ -17,10 +17,11 @@ //! Utilities for checking whether a candidate has been approved under a given block. use polkadot_node_primitives::approval::DelayTranche; +use polkadot_primitives::v1::ValidatorIndex; use bitvec::slice::BitSlice; use bitvec::order::Lsb0 as BitOrderLsb0; -use crate::persisted_entries::{ApprovalEntry, CandidateEntry}; +use crate::persisted_entries::{TrancheEntry, ApprovalEntry, CandidateEntry}; use crate::time::Tick; /// The required tranches of assignments needed to determine whether a candidate is approved. @@ -263,6 +264,85 @@ impl State { } } +/// Constructs an infinite iterator from an array of `TrancheEntry` values. Any missing tranches +/// are filled with empty assignments, as they are needed to compute the approved tranches. +fn filled_tranche_iterator<'a>( + tranches: &'a [TrancheEntry], +) -> impl Iterator { + let mut gap_end = None; + + let approval_entries_filled = tranches + .iter() + .flat_map(move |tranche_entry| { + let tranche = tranche_entry.tranche(); + let assignments = tranche_entry.assignments(); + + // The new gap_start immediately follows the prior gap_end, if one exists. + // Otherwise, on the first pass, the new gap_start is set to the first + // tranche so that the range below will be empty. + let gap_start = gap_end.map(|end| end + 1).unwrap_or(tranche); + gap_end = Some(tranche); + + (gap_start..tranche).map(|i| (i, &[] as &[_])) + .chain(std::iter::once((tranche, assignments))) + }); + + let pre_end = tranches.first().map(|t| t.tranche()); + let post_start = tranches.last().map_or(0, |t| t.tranche() + 1); + + let pre = pre_end.into_iter() + .flat_map(|pre_end| (0..pre_end).map(|i| (i, &[] as &[_]))); + let post = (post_start..).map(|i| (i, &[] as &[_])); + + pre.chain(approval_entries_filled).chain(post) +} + +/// Computes the number of no_show validators in a set of assignments given the relevant approvals +/// and tick parameters. This method also returns the next tick at which a no_show will occur +/// amongst the set of validators that have not submitted an approval. +/// +/// If the returned `next_no_show` is not None, there are two possible cases for the value of +/// based on the earliest assignment `tick` of a non-approving, yet-to-be-no-show validator: +/// - if `tick` <= `clock_drift`: the value will always be `clock_drift` + `no_show_duration`. +/// - if `tick` > `clock_drift`: the value is equal to `tick` + `no_show_duration`. +fn count_no_shows( + assignments: &[(ValidatorIndex, Tick)], + approvals: &BitSlice, + clock_drift: Tick, + no_show_duration: Tick, + drifted_tick_now: Tick, +) -> (usize, Option) { + let mut next_no_show = None; + let no_shows = assignments.iter() + .map(|(v_index, tick)| (v_index, tick.saturating_sub(clock_drift) + no_show_duration)) + .filter(|&(v_index, no_show_at)| { + let has_approved = if let Some(approved) = approvals.get(v_index.0 as usize) { + *approved + } else { + return false; + }; + + let is_no_show = !has_approved && no_show_at <= drifted_tick_now; + + if !is_no_show && !has_approved { + // When doing the comparison above, no_show_at and drifted_tick_now are calculated + // with the clock_drift removed. The reason for adding back the clock_drift in + // computing next_no_show is so that the scheduler knows the deadline at which + // *this node* should observe whether or not the validator is a no show. Recall + // that when the when drifted_tick_now is computed during that subsequent wake up, + // the clock drift will be removed again to do the comparison above. + next_no_show = super::min_prefer_some( + next_no_show, + Some(no_show_at + clock_drift), + ); + } + + is_no_show + }).count(); + + (no_shows, next_no_show) +} + /// Determine the amount of tranches of assignments needed to determine approval of a candidate. pub fn tranches_to_approve( approval_entry: &ApprovalEntry, @@ -288,31 +368,7 @@ pub fn tranches_to_approve( // these empty tranches, so we create an iterator to fill the gaps. // // This iterator has an infinitely long amount of non-empty tranches appended to the end. - let tranches_with_gaps_filled = { - let mut gap_end = 0; - - let approval_entries_filled = approval_entry.tranches() - .iter() - .flat_map(move |tranche_entry| { - let tranche = tranche_entry.tranche(); - let assignments = tranche_entry.assignments(); - - let gap_start = gap_end + 1; - gap_end = tranche; - - (gap_start..tranche).map(|i| (i, &[] as &[_])) - .chain(std::iter::once((tranche, assignments))) - }); - - let pre_end = approval_entry.tranches().first().map(|t| t.tranche()); - let post_start = approval_entry.tranches().last().map_or(0, |t| t.tranche() + 1); - - let pre = pre_end.into_iter() - .flat_map(|pre_end| (0..pre_end).map(|i| (i, &[] as &[_]))); - let post = (post_start..).map(|i| (i, &[] as &[_])); - - pre.chain(approval_entries_filled).chain(post) - }; + let tranches_with_gaps_filled = filled_tranche_iterator(approval_entry.tranches()); tranches_with_gaps_filled .scan(Some(initial_state), |state, (tranche, assignments)| { @@ -329,33 +385,23 @@ pub fn tranches_to_approve( return None; } - let n_assignments = assignments.len(); + // Count the number of valid validator assignments. + let n_assignments = assignments.iter() + .filter(|(v_index, _)| v_index.0 < n_validators as u32) + .count(); // count no-shows. An assignment is a no-show if there is no corresponding approval vote // after a fixed duration. // // While we count the no-shows, we also determine the next possible no-show we might // see within this tranche. - let mut next_no_show = None; - let no_shows = { - let next_no_show = &mut next_no_show; - assignments.iter() - .map(|(v_index, tick)| (v_index, tick.saturating_sub(clock_drift) + no_show_duration)) - .filter(|&(v_index, no_show_at)| { - let has_approved = approvals.get(v_index.0 as usize).map(|b| *b).unwrap_or(false); - - let is_no_show = !has_approved && no_show_at <= drifted_tick_now; - - if !is_no_show && !has_approved { - *next_no_show = super::min_prefer_some( - *next_no_show, - Some(no_show_at + clock_drift), - ); - } - - is_no_show - }).count() - }; + let (no_shows, next_no_show) = count_no_shows( + assignments, + approvals, + clock_drift, + no_show_duration, + drifted_tick_now, + ); let s = s.advance(n_assignments, no_shows, next_no_show); let output = s.output(tranche, needed_approvals, n_validators, no_show_duration); @@ -384,7 +430,7 @@ pub fn tranches_to_approve( mod tests { use super::*; - use polkadot_primitives::v1::{GroupIndex, ValidatorIndex}; + use polkadot_primitives::v1::GroupIndex; use bitvec::bitvec; use bitvec::order::Lsb0 as BitOrderLsb0; @@ -938,6 +984,303 @@ mod tests { }, ); } + + #[test] + fn validator_indexes_out_of_range_are_ignored_in_assignments() { + let block_tick = 20; + let no_show_duration = 10; + let needed_approvals = 3; + + let mut candidate: CandidateEntry = approval_db::v1::CandidateEntry { + candidate: Default::default(), + session: 0, + block_assignments: Default::default(), + approvals: bitvec![BitOrderLsb0, u8; 0; 3], + }.into(); + + for i in 0..3 { + candidate.mark_approval(ValidatorIndex(i)); + } + + let approval_entry = approval_db::v1::ApprovalEntry { + tranches: vec![ + // Assignments with invalid validator indexes. + approval_db::v1::TrancheEntry { + tranche: 1, + assignments: (2..5).map(|i| (ValidatorIndex(i), 1.into())).collect(), + }, + ], + assignments: bitvec![BitOrderLsb0, u8; 1; 3], + our_assignment: None, + our_approval_sig: None, + backing_group: GroupIndex(0), + approved: false, + }.into(); + + let approvals = bitvec![BitOrderLsb0, u8; 0; 3]; + + let tranche_now = 10; + assert_eq!( + tranches_to_approve( + &approval_entry, + &approvals, + tranche_now, + block_tick, + no_show_duration, + needed_approvals, + ), + RequiredTranches::Pending { + considered: 10, + next_no_show: None, + maximum_broadcast: DelayTranche::max_value(), + clock_drift: 0, + }, + ); + } + + #[test] + fn filled_tranche_iterator_yields_sequential_tranches() { + const PREFIX: u32 = 10; + + let test_tranches = vec![ + vec![], // empty set + vec![0], // zero start + vec![0, 3], // zero start with gap + vec![2], // non-zero start + vec![2, 4], // non-zero start with gap + vec![0, 1, 2], // zero start with run and no gap + vec![2, 3, 4, 8], // non-zero start with run and gap + vec![0, 1, 2, 5, 6, 7], // zero start with runs and gap + ]; + + for test_tranche in test_tranches { + let mut approval_entry: ApprovalEntry = approval_db::v1::ApprovalEntry { + tranches: Vec::new(), + backing_group: GroupIndex(0), + our_assignment: None, + our_approval_sig: None, + assignments: bitvec![BitOrderLsb0, u8; 0; 3], + approved: false, + }.into(); + + // Populate the requested tranches. The assignemnts aren't inspected in + // this test. + for &t in &test_tranche { + approval_entry.import_assignment(t, ValidatorIndex(0), 0) + } + + let filled_tranches = filled_tranche_iterator(approval_entry.tranches()); + + // Take the first PREFIX entries and map them to their tranche. + let tranches: Vec = filled_tranches + .take(PREFIX as usize) + .map(|e| e.0) + .collect(); + + // We expect this sequence to be sequential. + let exp_tranches: Vec = (0..PREFIX).collect(); + assert_eq!(tranches, exp_tranches, "for test tranches: {:?}", test_tranche); + } + } + + #[derive(Debug)] + struct NoShowTest { + assignments: Vec<(ValidatorIndex, Tick)>, + approvals: Vec, + clock_drift: crate::time::Tick, + no_show_duration: crate::time::Tick, + drifted_tick_now: crate::time::Tick, + exp_no_shows: usize, + exp_next_no_show: Option, + } + + fn test_count_no_shows(test: NoShowTest) { + let n_validators = 4; + + let mut approvals = bitvec![BitOrderLsb0, u8; 0; n_validators]; + for &v_index in &test.approvals { + approvals.set(v_index, true); + } + + let (no_shows, next_no_show) = count_no_shows( + &test.assignments, + &approvals, + test.clock_drift, + test.no_show_duration, + test.drifted_tick_now, + ); + assert_eq!(no_shows, test.exp_no_shows, "for test: {:?}", test); + assert_eq!(next_no_show, test.exp_next_no_show, "for test {:?}", test); + } + + #[test] + fn count_no_shows_empty_assignments() { + test_count_no_shows(NoShowTest { + assignments: vec![], + approvals: vec![], + clock_drift: 0, + no_show_duration: 0, + drifted_tick_now: 0, + exp_no_shows: 0, + exp_next_no_show: None, + }) + } + + #[test] + fn count_no_shows_single_validator_is_next_no_show() { + test_count_no_shows(NoShowTest { + assignments: vec![(ValidatorIndex(1), 21)], + approvals: vec![], + clock_drift: 10, + no_show_duration: 10, + drifted_tick_now: 20, + exp_no_shows: 0, + exp_next_no_show: Some(31), + }) + } + + #[test] + fn count_no_shows_single_validator_approval_at_drifted_tick_now() { + test_count_no_shows(NoShowTest { + assignments: vec![(ValidatorIndex(1), 20)], + approvals: vec![1], + clock_drift: 10, + no_show_duration: 10, + drifted_tick_now: 20, + exp_no_shows: 0, + exp_next_no_show: None, + }) + } + + #[test] + fn count_no_shows_single_validator_approval_after_drifted_tick_now() { + test_count_no_shows(NoShowTest { + assignments: vec![(ValidatorIndex(1), 21)], + approvals: vec![1], + clock_drift: 10, + no_show_duration: 10, + drifted_tick_now: 20, + exp_no_shows: 0, + exp_next_no_show: None, + }) + } + + #[test] + fn count_no_shows_two_validators_next_no_show_ordered_first() { + test_count_no_shows(NoShowTest { + assignments: vec![(ValidatorIndex(1), 21), (ValidatorIndex(2), 22)], + approvals: vec![], + clock_drift: 10, + no_show_duration: 10, + drifted_tick_now: 20, + exp_no_shows: 0, + exp_next_no_show: Some(31), + }) + } + + #[test] + fn count_no_shows_two_validators_next_no_show_ordered_last() { + test_count_no_shows(NoShowTest { + assignments: vec![(ValidatorIndex(1), 22), (ValidatorIndex(2), 21)], + approvals: vec![], + clock_drift: 10, + no_show_duration: 10, + drifted_tick_now: 20, + exp_no_shows: 0, + exp_next_no_show: Some(31), + }) + } + + #[test] + fn count_no_shows_three_validators_one_almost_late_one_no_show_one_approving() { + test_count_no_shows(NoShowTest { + assignments: vec![(ValidatorIndex(1), 21), (ValidatorIndex(2), 20), (ValidatorIndex(3), 20)], + approvals: vec![3], + clock_drift: 10, + no_show_duration: 10, + drifted_tick_now: 20, + exp_no_shows: 1, + exp_next_no_show: Some(31), + }) + } + + #[test] + fn count_no_shows_three_no_show_validators() { + test_count_no_shows(NoShowTest { + assignments: vec![(ValidatorIndex(1), 20), (ValidatorIndex(2), 20), (ValidatorIndex(3), 20)], + approvals: vec![], + clock_drift: 10, + no_show_duration: 10, + drifted_tick_now: 20, + exp_no_shows: 3, + exp_next_no_show: None, + }) + } + + #[test] + fn count_no_shows_three_approving_validators() { + test_count_no_shows(NoShowTest { + assignments: vec![(ValidatorIndex(1), 20), (ValidatorIndex(2), 20), (ValidatorIndex(3), 20)], + approvals: vec![1, 2, 3], + clock_drift: 10, + no_show_duration: 10, + drifted_tick_now: 20, + exp_no_shows: 0, + exp_next_no_show: None, + }) + } + + #[test] + fn count_no_shows_earliest_possible_next_no_show_is_clock_drift_plus_no_show_duration() { + test_count_no_shows(NoShowTest { + assignments: vec![(ValidatorIndex(1), 0)], + approvals: vec![], + clock_drift: 10, + no_show_duration: 20, + drifted_tick_now: 0, + exp_no_shows: 0, + exp_next_no_show: Some(30), + }) + } + + #[test] + fn count_no_shows_assignment_tick_equal_to_clock_drift_yields_earliest_possible_next_no_show() { + test_count_no_shows(NoShowTest { + assignments: vec![(ValidatorIndex(1), 10)], + approvals: vec![], + clock_drift: 10, + no_show_duration: 20, + drifted_tick_now: 0, + exp_no_shows: 0, + exp_next_no_show: Some(30), + }) + } + + #[test] + fn count_no_shows_validator_index_out_of_approvals_range_is_ignored_as_no_show() { + test_count_no_shows(NoShowTest { + assignments: vec![(ValidatorIndex(1000), 20)], + approvals: vec![], + clock_drift: 10, + no_show_duration: 10, + drifted_tick_now: 20, + exp_no_shows: 0, + exp_next_no_show: None, + }) + } + + #[test] + fn count_no_shows_validator_index_out_of_approvals_range_is_ignored_as_next_no_show() { + test_count_no_shows(NoShowTest { + assignments: vec![(ValidatorIndex(1000), 21)], + approvals: vec![], + clock_drift: 10, + no_show_duration: 10, + drifted_tick_now: 20, + exp_no_shows: 0, + exp_next_no_show: None, + }) + } } #[test] diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index 77bf768fda32..bcb59f6ba7b0 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -34,8 +34,12 @@ use polkadot_node_subsystem::{ }, SubsystemContext, SubsystemError, SubsystemResult, }; +use polkadot_node_subsystem_util::determine_new_blocks; +use polkadot_node_subsystem_util::rolling_session_window::{ + RollingSessionWindow, SessionWindowUpdate, +}; use polkadot_primitives::v1::{ - Hash, SessionIndex, SessionInfo, CandidateEvent, Header, CandidateHash, + Hash, SessionIndex, CandidateEvent, Header, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, BlockNumber, ConsensusLog, }; use polkadot_node_primitives::approval::{ @@ -58,285 +62,7 @@ use crate::persisted_entries::CandidateEntry; use crate::criteria::{AssignmentCriteria, OurAssignment}; use crate::time::{slot_number_to_tick, Tick}; -use super::{APPROVAL_SESSIONS, LOG_TARGET, State, DBReader}; - -/// A rolling window of sessions. -#[derive(Default)] -pub struct RollingSessionWindow { - pub earliest_session: Option, - pub session_info: Vec, -} - -impl RollingSessionWindow { - pub fn session_info(&self, index: SessionIndex) -> Option<&SessionInfo> { - self.earliest_session.and_then(|earliest| { - if index < earliest { - None - } else { - self.session_info.get((index - earliest) as usize) - } - }) - - } - - pub fn latest_session(&self) -> Option { - self.earliest_session - .map(|earliest| earliest + (self.session_info.len() as SessionIndex).saturating_sub(1)) - } -} - -// Given a new chain-head hash, this determines the hashes of all new blocks we should track -// metadata for, given this head. The list will typically include the `head` hash provided unless -// that block is already known, in which case the list should be empty. This is guaranteed to be -// a subset of the ancestry of `head`, as well as `head`, starting from `head` and moving -// backwards. -// -// This returns the entire ancestry up to the last finalized block's height or the last item we -// have in the DB. This may be somewhat expensive when first recovering from major sync. -async fn determine_new_blocks( - ctx: &mut impl SubsystemContext, - db: &impl DBReader, - head: Hash, - header: &Header, - finalized_number: BlockNumber, -) -> SubsystemResult> { - const ANCESTRY_STEP: usize = 4; - - // Early exit if the block is in the DB or too early. - { - let already_known = db.load_block_entry(&head)? - .is_some(); - - let before_relevant = header.number <= finalized_number; - - if already_known || before_relevant { - return Ok(Vec::new()); - } - } - - let mut ancestry = vec![(head, header.clone())]; - - // Early exit if the parent hash is in the DB. - if db.load_block_entry(&header.parent_hash)? - .is_some() - { - return Ok(ancestry); - } - - 'outer: loop { - let &(ref last_hash, ref last_header) = ancestry.last() - .expect("ancestry has length 1 at initialization and is only added to; qed"); - - // If we iterated back to genesis, which can happen at the beginning of chains. - if last_header.number <= 1 { - break 'outer - } - - let (tx, rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::Ancestors { - hash: *last_hash, - k: ANCESTRY_STEP, - response_channel: tx, - }.into()).await; - - // Continue past these errors. - let batch_hashes = match rx.await { - Err(_) | Ok(Err(_)) => break 'outer, - Ok(Ok(ancestors)) => ancestors, - }; - - let batch_headers = { - let (batch_senders, batch_receivers) = (0..batch_hashes.len()) - .map(|_| oneshot::channel()) - .unzip::<_, _, Vec<_>, Vec<_>>(); - - for (hash, sender) in batch_hashes.iter().cloned().zip(batch_senders) { - ctx.send_message(ChainApiMessage::BlockHeader(hash, sender).into()).await; - } - - let mut requests = futures::stream::FuturesOrdered::new(); - batch_receivers.into_iter().map(|rx| async move { - match rx.await { - Err(_) | Ok(Err(_)) => None, - Ok(Ok(h)) => h, - } - }) - .for_each(|x| requests.push(x)); - - let batch_headers: Vec<_> = requests - .flat_map(|x: Option

| stream::iter(x)) - .collect() - .await; - - // Any failed header fetch of the batch will yield a `None` result that will - // be skipped. Any failure at this stage means we'll just ignore those blocks - // as the chain DB has failed us. - if batch_headers.len() != batch_hashes.len() { break 'outer } - batch_headers - }; - - for (hash, header) in batch_hashes.into_iter().zip(batch_headers) { - let is_known = db.load_block_entry(&hash)?.is_some(); - - let is_relevant = header.number > finalized_number; - - if is_known || !is_relevant { - break 'outer - } - - ancestry.push((hash, header)); - } - } - - Ok(ancestry) -} - -// Sessions unavailable in state to cache. -#[derive(Debug)] -struct SessionsUnavailable; - -async fn load_all_sessions( - ctx: &mut impl SubsystemContext, - block_hash: Hash, - start: SessionIndex, - end_inclusive: SessionIndex, -) -> Result, SessionsUnavailable> { - let mut v = Vec::new(); - for i in start..=end_inclusive { - let (tx, rx)= oneshot::channel(); - ctx.send_message(RuntimeApiMessage::Request( - block_hash, - RuntimeApiRequest::SessionInfo(i, tx), - ).into()).await; - - let session_info = match rx.await { - Ok(Ok(Some(s))) => s, - Ok(Ok(None)) => { - tracing::debug!( - target: LOG_TARGET, - "Session {} is missing from session-info state of block {}", - i, - block_hash, - ); - - return Err(SessionsUnavailable); - } - Ok(Err(_)) | Err(_) => return Err(SessionsUnavailable), - }; - - v.push(session_info); - } - - Ok(v) -} - -// When inspecting a new import notification, updates the session info cache to match -// the session of the imported block. -// -// this only needs to be called on heads where we are directly notified about import, as sessions do -// not change often and import notifications are expected to be typically increasing in session number. -// -// some backwards drift in session index is acceptable. -async fn cache_session_info_for_head( - ctx: &mut impl SubsystemContext, - session_window: &mut RollingSessionWindow, - block_hash: Hash, - block_header: &Header, -) -> Result<(), SessionsUnavailable> { - let session_index = { - let (s_tx, s_rx) = oneshot::channel(); - - // The genesis is guaranteed to be at the beginning of the session and its parent state - // is non-existent. Therefore if we're at the genesis, we request using its state and - // not the parent. - ctx.send_message(RuntimeApiMessage::Request( - if block_header.number == 0 { block_hash } else { block_header.parent_hash }, - RuntimeApiRequest::SessionIndexForChild(s_tx), - ).into()).await; - - match s_rx.await { - Ok(Ok(s)) => s, - Ok(Err(_)) | Err(_) => return Err(SessionsUnavailable), - } - }; - - match session_window.earliest_session { - None => { - // First block processed on start-up. - - let window_start = session_index.saturating_sub(APPROVAL_SESSIONS - 1); - - tracing::debug!( - target: LOG_TARGET, "Loading approval window from session {}..={}", - window_start, session_index, - ); - - match load_all_sessions(ctx, block_hash, window_start, session_index).await { - Err(SessionsUnavailable) => { - tracing::debug!( - target: LOG_TARGET, - "Could not load sessions {}..={} from block {:?} in session {}", - window_start, session_index, block_hash, session_index, - ); - - return Err(SessionsUnavailable); - }, - Ok(s) => { - session_window.earliest_session = Some(window_start); - session_window.session_info = s; - } - } - } - Some(old_window_start) => { - let latest = session_window.latest_session().expect("latest always exists if earliest does; qed"); - - // Either cached or ancient. - if session_index <= latest { return Ok(()) } - - let old_window_end = latest; - - let window_start = session_index.saturating_sub(APPROVAL_SESSIONS - 1); - tracing::info!( - target: LOG_TARGET, "Moving approval window from session {}..={} to {}..={}", - old_window_start, old_window_end, - window_start, session_index, - ); - - // keep some of the old window, if applicable. - let overlap_start = window_start.saturating_sub(old_window_start); - - let fresh_start = if latest < window_start { - window_start - } else { - latest + 1 - }; - - match load_all_sessions(ctx, block_hash, fresh_start, session_index).await { - Err(SessionsUnavailable) => { - tracing::warn!( - target: LOG_TARGET, - "Could not load sessions {}..={} from block {:?} in session {}", - latest + 1, session_index, block_hash, session_index, - ); - - return Err(SessionsUnavailable); - } - Ok(s) => { - let outdated = std::cmp::min(overlap_start as usize, session_window.session_info.len()); - session_window.session_info.drain(..outdated); - session_window.session_info.extend(s); - // we need to account for this case: - // window_start ................................... session_index - // old_window_start ........... latest - let new_earliest = std::cmp::max(window_start, old_window_start); - session_window.earliest_session = Some(new_earliest); - } - } - } - } - - Ok(()) -} +use super::{LOG_TARGET, State, DBReader}; struct ImportedBlockInfo { included_candidates: Vec<(CandidateHash, CandidateReceipt, CoreIndex, GroupIndex)>, @@ -357,7 +83,7 @@ struct ImportedBlockInfoEnv<'a> { // Computes information about the imported block. Returns `None` if the info couldn't be extracted - // failure to communicate with overseer, async fn imported_block_info( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, env: ImportedBlockInfoEnv<'_>, block_hash: Hash, block_header: &Header, @@ -401,7 +127,7 @@ async fn imported_block_info( Err(_) => return Ok(None), }; - if env.session_window.earliest_session.as_ref().map_or(true, |e| &session_index < e) { + if env.session_window.earliest_session().map_or(true, |e| session_index < e) { tracing::debug!(target: LOG_TARGET, "Block {} is from ancient session {}. Skipping", block_hash, session_index); @@ -558,7 +284,7 @@ pub struct BlockImportedCandidates { /// /// It is the responsibility of the caller to schedule wakeups for each block. pub(crate) async fn handle_new_head( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, db_writer: &dyn KeyValueDB, db_config: DatabaseConfig, @@ -591,28 +317,38 @@ pub(crate) async fn handle_new_head( } }; - if let Err(SessionsUnavailable) - = cache_session_info_for_head( - ctx, - &mut state.session_window, - head, - &header, - ).await - { - tracing::debug!( - target: LOG_TARGET, - "Could not cache session info when processing head {:?}", - head, - ); + match state.session_window.cache_session_info_for_head(ctx, head, &header).await { + Err(e) => { + tracing::debug!( + target: LOG_TARGET, + ?head, + ?e, + "Could not cache session info when processing head.", + ); - return Ok(Vec::new()) + return Ok(Vec::new()) + } + Ok(a @ SessionWindowUpdate::Advanced { .. }) => { + tracing::info!( + target: LOG_TARGET, + update = ?a, + "Advanced session window for approvals", + ); + } + Ok(_) => {} } // If we've just started the node and haven't yet received any finality notifications, // we don't do any look-back. Approval voting is only for nodes were already online. - let finalized_number = finalized_number.unwrap_or(header.number.saturating_sub(1)); - - let new_blocks = determine_new_blocks(ctx, &state.db, head, &header, finalized_number) + let lower_bound_number = finalized_number.unwrap_or(header.number.saturating_sub(1)); + + let new_blocks = determine_new_blocks( + ctx.sender(), + |h| state.db.load_block_entry(h).map(|e| e.is_some()), + head, + &header, + lower_bound_number, + ) .map_err(|e| SubsystemError::with_origin("approval-voting", e)) .await?; @@ -815,7 +551,8 @@ mod tests { use super::*; use polkadot_node_subsystem_test_helpers::make_subsystem_context; use polkadot_node_primitives::approval::{VRFOutput, VRFProof}; - use polkadot_primitives::v1::ValidatorIndex; + use polkadot_primitives::v1::{SessionInfo, ValidatorIndex}; + use polkadot_node_subsystem::messages::AllMessages; use sp_core::testing::TaskExecutor; use sp_runtime::{Digest, DigestItem}; use sp_consensus_babe::{ @@ -827,7 +564,7 @@ mod tests { use merlin::Transcript; use std::{pin::Pin, sync::Arc}; - use crate::{criteria, BlockEntry}; + use crate::{APPROVAL_SESSIONS, criteria, BlockEntry}; const DATA_COL: u32 = 0; const NUM_COLUMNS: u32 = 1; @@ -883,7 +620,7 @@ mod tests { fn blank_state() -> State { State { - session_window: RollingSessionWindow::default(), + session_window: RollingSessionWindow::new(APPROVAL_SESSIONS), keystore: Arc::new(LocalKeystore::in_memory()), slot_duration_millis: 6_000, db: TestDB::default(), @@ -896,94 +633,15 @@ mod tests { -> State { State { - session_window: RollingSessionWindow { - earliest_session: Some(index), - session_info: vec![info], - }, + session_window: RollingSessionWindow::with_session_info( + APPROVAL_SESSIONS, + index, + vec![info], + ), ..blank_state() } } - #[derive(Clone)] - struct TestChain { - start_number: BlockNumber, - headers: Vec
, - numbers: HashMap, - } - - impl TestChain { - fn new(start: BlockNumber, len: usize) -> Self { - assert!(len > 0, "len must be at least 1"); - - let base = Header { - digest: Default::default(), - extrinsics_root: Default::default(), - number: start, - state_root: Default::default(), - parent_hash: Default::default(), - }; - - let base_hash = base.hash(); - - let mut chain = TestChain { - start_number: start, - headers: vec![base], - numbers: vec![(base_hash, start)].into_iter().collect(), - }; - - for _ in 1..len { - chain.grow() - } - - chain - } - - fn grow(&mut self) { - let next = { - let last = self.headers.last().unwrap(); - Header { - digest: Default::default(), - extrinsics_root: Default::default(), - number: last.number + 1, - state_root: Default::default(), - parent_hash: last.hash(), - } - }; - - self.numbers.insert(next.hash(), next.number); - self.headers.push(next); - } - - fn header_by_number(&self, number: BlockNumber) -> Option<&Header> { - if number < self.start_number { - None - } else { - self.headers.get((number - self.start_number) as usize) - } - } - - fn header_by_hash(&self, hash: &Hash) -> Option<&Header> { - self.numbers.get(hash).and_then(|n| self.header_by_number(*n)) - } - - fn hash_by_number(&self, number: BlockNumber) -> Option { - self.header_by_number(number).map(|h| h.hash()) - } - - fn ancestry(&self, hash: &Hash, k: BlockNumber) -> Vec { - let n = match self.numbers.get(hash) { - None => return Vec::new(), - Some(&n) => n, - }; - - (0..k) - .map(|i| i + 1) - .filter_map(|i| self.header_by_number(n - i)) - .map(|h| h.hash()) - .collect() - } - } - struct MockAssignmentCriteria; impl AssignmentCriteria for MockAssignmentCriteria { @@ -1019,340 +677,6 @@ mod tests { (VRFOutput(o.to_output()), VRFProof(p)) } - #[test] - fn determine_new_blocks_back_to_finalized() { - let pool = TaskExecutor::new(); - let (mut ctx, mut handle) = make_subsystem_context::<(), _>(pool.clone()); - - let db = TestDB::default(); - - let chain = TestChain::new(10, 9); - - let head = chain.header_by_number(18).unwrap().clone(); - let head_hash = head.hash(); - let finalized_number = 12; - - // Finalized block should be omitted. The head provided to `determine_new_blocks` - // should be included. - let expected_ancestry = (13..=18) - .map(|n| chain.header_by_number(n).map(|h| (h.hash(), h.clone())).unwrap()) - .rev() - .collect::>(); - - let test_fut = Box::pin(async move { - let ancestry = determine_new_blocks( - &mut ctx, - &db, - head_hash, - &head, - finalized_number, - ).await.unwrap(); - - assert_eq!( - ancestry, - expected_ancestry, - ); - }); - - let aux_fut = Box::pin(async move { - assert_matches!( - handle.recv().await, - AllMessages::ChainApi(ChainApiMessage::Ancestors { - hash: h, - k, - response_channel: tx, - }) => { - assert_eq!(h, head_hash); - assert_eq!(k, 4); - let _ = tx.send(Ok(chain.ancestry(&h, k as _))); - } - ); - - for _ in 0..4 { - assert_matches!( - handle.recv().await, - AllMessages::ChainApi(ChainApiMessage::BlockHeader(h, tx)) => { - let _ = tx.send(Ok(chain.header_by_hash(&h).map(|h| h.clone()))); - } - ); - } - - assert_matches!( - handle.recv().await, - AllMessages::ChainApi(ChainApiMessage::Ancestors { - hash: h, - k, - response_channel: tx, - }) => { - assert_eq!(h, chain.hash_by_number(14).unwrap()); - assert_eq!(k, 4); - let _ = tx.send(Ok(chain.ancestry(&h, k as _))); - } - ); - - for _ in 0..4 { - assert_matches!( - handle.recv().await, - AllMessages::ChainApi(ChainApiMessage::BlockHeader(h, tx)) => { - let _ = tx.send(Ok(chain.header_by_hash(&h).map(|h| h.clone()))); - } - ); - } - - }); - - futures::executor::block_on(futures::future::join(test_fut, aux_fut)); - } - - #[test] - fn determine_new_blocks_back_to_known() { - let pool = TaskExecutor::new(); - let (mut ctx, mut handle) = make_subsystem_context::<(), _>(pool.clone()); - - let mut db = TestDB::default(); - - let chain = TestChain::new(10, 9); - - let head = chain.header_by_number(18).unwrap().clone(); - let head_hash = head.hash(); - let finalized_number = 12; - let known_number = 15; - let known_hash = chain.hash_by_number(known_number).unwrap(); - - db.block_entries.insert( - known_hash, - crate::approval_db::v1::BlockEntry { - block_hash: known_hash, - parent_hash: Default::default(), - block_number: known_number, - session: 1, - slot: Slot::from(100), - relay_vrf_story: Default::default(), - candidates: Vec::new(), - approved_bitfield: Default::default(), - children: Vec::new(), - }.into(), - ); - - // Known block should be omitted. The head provided to `determine_new_blocks` - // should be included. - let expected_ancestry = (16..=18) - .map(|n| chain.header_by_number(n).map(|h| (h.hash(), h.clone())).unwrap()) - .rev() - .collect::>(); - - let test_fut = Box::pin(async move { - let ancestry = determine_new_blocks( - &mut ctx, - &db, - head_hash, - &head, - finalized_number, - ).await.unwrap(); - - assert_eq!( - ancestry, - expected_ancestry, - ); - }); - - let aux_fut = Box::pin(async move { - assert_matches!( - handle.recv().await, - AllMessages::ChainApi(ChainApiMessage::Ancestors { - hash: h, - k, - response_channel: tx, - }) => { - assert_eq!(h, head_hash); - assert_eq!(k, 4); - let _ = tx.send(Ok(chain.ancestry(&h, k as _))); - } - ); - - for _ in 0u32..4 { - assert_matches!( - handle.recv().await, - AllMessages::ChainApi(ChainApiMessage::BlockHeader(h, tx)) => { - let _ = tx.send(Ok(chain.header_by_hash(&h).map(|h| h.clone()))); - } - ); - } - }); - - futures::executor::block_on(futures::future::join(test_fut, aux_fut)); - } - - #[test] - fn determine_new_blocks_already_known_is_empty() { - let pool = TaskExecutor::new(); - let (mut ctx, _handle) = make_subsystem_context::<(), _>(pool.clone()); - - let mut db = TestDB::default(); - - let chain = TestChain::new(10, 9); - - let head = chain.header_by_number(18).unwrap().clone(); - let head_hash = head.hash(); - let finalized_number = 0; - - db.block_entries.insert( - head_hash, - crate::approval_db::v1::BlockEntry { - block_hash: head_hash, - parent_hash: Default::default(), - block_number: 18, - session: 1, - slot: Slot::from(100), - relay_vrf_story: Default::default(), - candidates: Vec::new(), - approved_bitfield: Default::default(), - children: Vec::new(), - }.into(), - ); - - // Known block should be omitted. - let expected_ancestry = Vec::new(); - - let test_fut = Box::pin(async move { - let ancestry = determine_new_blocks( - &mut ctx, - &db, - head_hash, - &head, - finalized_number, - ).await.unwrap(); - - assert_eq!( - ancestry, - expected_ancestry, - ); - }); - - futures::executor::block_on(test_fut); - } - - #[test] - fn determine_new_blocks_parent_known_is_fast() { - let pool = TaskExecutor::new(); - let (mut ctx, _handle) = make_subsystem_context::<(), _>(pool.clone()); - - let mut db = TestDB::default(); - - let chain = TestChain::new(10, 9); - - let head = chain.header_by_number(18).unwrap().clone(); - let head_hash = head.hash(); - let finalized_number = 0; - let parent_hash = chain.hash_by_number(17).unwrap(); - - db.block_entries.insert( - parent_hash, - crate::approval_db::v1::BlockEntry { - block_hash: parent_hash, - parent_hash: Default::default(), - block_number: 18, - session: 1, - slot: Slot::from(100), - relay_vrf_story: Default::default(), - candidates: Vec::new(), - approved_bitfield: Default::default(), - children: Vec::new(), - }.into(), - ); - - // New block should be the only new one. - let expected_ancestry = vec![(head_hash, head.clone())]; - - let test_fut = Box::pin(async move { - let ancestry = determine_new_blocks( - &mut ctx, - &db, - head_hash, - &head, - finalized_number, - ).await.unwrap(); - - assert_eq!( - ancestry, - expected_ancestry, - ); - }); - - futures::executor::block_on(test_fut); - } - - #[test] - fn determine_new_block_before_finality_is_empty() { - let pool = TaskExecutor::new(); - let (mut ctx, _handle) = make_subsystem_context::<(), _>(pool.clone()); - - let chain = TestChain::new(10, 9); - - let head = chain.header_by_number(18).unwrap().clone(); - let head_hash = head.hash(); - let parent_hash = chain.hash_by_number(17).unwrap(); - let mut db = TestDB::default(); - - db.block_entries.insert( - parent_hash, - crate::approval_db::v1::BlockEntry { - block_hash: parent_hash, - parent_hash: Default::default(), - block_number: 18, - session: 1, - slot: Slot::from(100), - relay_vrf_story: Default::default(), - candidates: Vec::new(), - approved_bitfield: Default::default(), - children: Vec::new(), - }.into(), - ); - - let test_fut = Box::pin(async move { - let after_finality = determine_new_blocks( - &mut ctx, - &db, - head_hash, - &head, - 17, - ).await.unwrap(); - - let at_finality = determine_new_blocks( - &mut ctx, - &db, - head_hash, - &head, - 18, - ).await.unwrap(); - - let before_finality = determine_new_blocks( - &mut ctx, - &db, - head_hash, - &head, - 19, - ).await.unwrap(); - - assert_eq!( - after_finality, - vec![(head_hash, head.clone())], - ); - - assert_eq!( - at_finality, - Vec::new(), - ); - - assert_eq!( - before_finality, - Vec::new(), - ); - }); - - futures::executor::block_on(test_fut); - } - fn dummy_session_info(index: SessionIndex) -> SessionInfo { SessionInfo { validators: Vec::new(), @@ -1422,14 +746,11 @@ mod tests { .map(|(r, c, g)| (r.hash(), r.clone(), *c, *g)) .collect::>(); - let session_window = { - let mut window = RollingSessionWindow::default(); - - window.earliest_session = Some(session); - window.session_info.push(session_info); - - window - }; + let session_window = RollingSessionWindow::with_session_info( + APPROVAL_SESSIONS, + session, + vec![session_info], + ); let header = header.clone(); Box::pin(async move { @@ -1536,14 +857,11 @@ mod tests { .collect::>(); let test_fut = { - let session_window = { - let mut window = RollingSessionWindow::default(); - - window.earliest_session = Some(session); - window.session_info.push(session_info); - - window - }; + let session_window = RollingSessionWindow::with_session_info( + APPROVAL_SESSIONS, + session, + vec![session_info], + ); let header = header.clone(); Box::pin(async move { @@ -1644,7 +962,7 @@ mod tests { .collect::>(); let test_fut = { - let session_window = RollingSessionWindow::default(); + let session_window = RollingSessionWindow::new(APPROVAL_SESSIONS); let header = header.clone(); Box::pin(async move { @@ -1747,14 +1065,11 @@ mod tests { .map(|(r, c, g)| (r.hash(), r.clone(), *c, *g)) .collect::>(); - let session_window = { - let mut window = RollingSessionWindow::default(); - - window.earliest_session = Some(session); - window.session_info.push(session_info); - - window - }; + let session_window = RollingSessionWindow::with_session_info( + APPROVAL_SESSIONS, + session, + vec![session_info], + ); let header = header.clone(); Box::pin(async move { @@ -1850,8 +1165,7 @@ mod tests { let slot = Slot::from(10); - let chain = TestChain::new(4, 1); - let parent_hash = chain.header_by_number(4).unwrap().hash(); + let parent_hash = Hash::repeat_byte(0x01); let header = Header { digest: { @@ -2018,318 +1332,4 @@ mod tests { futures::executor::block_on(futures::future::join(test_fut, aux_fut)); } - - fn cache_session_info_test( - expected_start_session: SessionIndex, - session: SessionIndex, - mut window: RollingSessionWindow, - expect_requests_from: SessionIndex, - ) { - let header = Header { - digest: Digest::default(), - extrinsics_root: Default::default(), - number: 5, - state_root: Default::default(), - parent_hash: Default::default(), - }; - - let pool = TaskExecutor::new(); - let (mut ctx, mut handle) = make_subsystem_context::<(), _>(pool.clone()); - - let hash = header.hash(); - - let test_fut = { - let header = header.clone(); - Box::pin(async move { - cache_session_info_for_head( - &mut ctx, - &mut window, - hash, - &header, - ).await.unwrap(); - - assert_eq!(window.earliest_session, Some(expected_start_session)); - assert_eq!( - window.session_info, - (expected_start_session..=session).map(dummy_session_info).collect::>(), - ); - }) - }; - - let aux_fut = Box::pin(async move { - assert_matches!( - handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - h, - RuntimeApiRequest::SessionIndexForChild(s_tx), - )) => { - assert_eq!(h, header.parent_hash); - let _ = s_tx.send(Ok(session)); - } - ); - - for i in expect_requests_from..=session { - assert_matches!( - handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - h, - RuntimeApiRequest::SessionInfo(j, s_tx), - )) => { - assert_eq!(h, hash); - assert_eq!(i, j); - let _ = s_tx.send(Ok(Some(dummy_session_info(i)))); - } - ); - } - }); - - futures::executor::block_on(futures::future::join(test_fut, aux_fut)); - } - - #[test] - fn cache_session_info_first_early() { - cache_session_info_test( - 0, - 1, - RollingSessionWindow::default(), - 0, - ); - } - - #[test] - fn cache_session_info_does_not_underflow() { - let window = RollingSessionWindow { - earliest_session: Some(1), - session_info: vec![dummy_session_info(1)], - }; - - cache_session_info_test( - 1, - 2, - window, - 2, - ); - } - - #[test] - fn cache_session_info_first_late() { - cache_session_info_test( - (100 as SessionIndex).saturating_sub(APPROVAL_SESSIONS - 1), - 100, - RollingSessionWindow::default(), - (100 as SessionIndex).saturating_sub(APPROVAL_SESSIONS - 1), - ); - } - - #[test] - fn cache_session_info_jump() { - let window = RollingSessionWindow { - earliest_session: Some(50), - session_info: vec![dummy_session_info(50), dummy_session_info(51), dummy_session_info(52)], - }; - - cache_session_info_test( - (100 as SessionIndex).saturating_sub(APPROVAL_SESSIONS - 1), - 100, - window, - (100 as SessionIndex).saturating_sub(APPROVAL_SESSIONS - 1), - ); - } - - #[test] - fn cache_session_info_roll_full() { - let start = 99 - (APPROVAL_SESSIONS - 1); - let window = RollingSessionWindow { - earliest_session: Some(start), - session_info: (start..=99).map(dummy_session_info).collect(), - }; - - cache_session_info_test( - (100 as SessionIndex).saturating_sub(APPROVAL_SESSIONS - 1), - 100, - window, - 100, // should only make one request. - ); - } - - #[test] - fn cache_session_info_roll_many_full() { - let start = 97 - (APPROVAL_SESSIONS - 1); - let window = RollingSessionWindow { - earliest_session: Some(start), - session_info: (start..=97).map(dummy_session_info).collect(), - }; - - cache_session_info_test( - (100 as SessionIndex).saturating_sub(APPROVAL_SESSIONS - 1), - 100, - window, - 98, - ); - } - - #[test] - fn cache_session_info_roll_early() { - let start = 0; - let window = RollingSessionWindow { - earliest_session: Some(start), - session_info: (0..=1).map(dummy_session_info).collect(), - }; - - cache_session_info_test( - 0, - 2, - window, - 2, // should only make one request. - ); - } - - #[test] - fn cache_session_info_roll_many_early() { - let start = 0; - let window = RollingSessionWindow { - earliest_session: Some(start), - session_info: (0..=1).map(dummy_session_info).collect(), - }; - - cache_session_info_test( - 0, - 3, - window, - 2, - ); - } - - #[test] - fn any_session_unavailable_for_caching_means_no_change() { - let session: SessionIndex = 6; - let start_session = session.saturating_sub(APPROVAL_SESSIONS - 1); - - let header = Header { - digest: Digest::default(), - extrinsics_root: Default::default(), - number: 5, - state_root: Default::default(), - parent_hash: Default::default(), - }; - - let pool = TaskExecutor::new(); - let (mut ctx, mut handle) = make_subsystem_context::<(), _>(pool.clone()); - - let mut window = RollingSessionWindow::default(); - let hash = header.hash(); - - let test_fut = { - let header = header.clone(); - Box::pin(async move { - let res = cache_session_info_for_head( - &mut ctx, - &mut window, - hash, - &header, - ).await; - - assert_matches!(res, Err(SessionsUnavailable)); - }) - }; - - let aux_fut = Box::pin(async move { - assert_matches!( - handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - h, - RuntimeApiRequest::SessionIndexForChild(s_tx), - )) => { - assert_eq!(h, header.parent_hash); - let _ = s_tx.send(Ok(session)); - } - ); - - for i in start_session..=session { - assert_matches!( - handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - h, - RuntimeApiRequest::SessionInfo(j, s_tx), - )) => { - assert_eq!(h, hash); - assert_eq!(i, j); - - let _ = s_tx.send(Ok(if i == session { - None - } else { - Some(dummy_session_info(i)) - })); - } - ); - } - }); - - futures::executor::block_on(futures::future::join(test_fut, aux_fut)); - } - - #[test] - fn request_session_info_for_genesis() { - let session: SessionIndex = 0; - - let header = Header { - digest: Digest::default(), - extrinsics_root: Default::default(), - number: 0, - state_root: Default::default(), - parent_hash: Default::default(), - }; - - let pool = TaskExecutor::new(); - let (mut ctx, mut handle) = make_subsystem_context::<(), _>(pool.clone()); - - let mut window = RollingSessionWindow::default(); - let hash = header.hash(); - - let test_fut = { - let header = header.clone(); - Box::pin(async move { - cache_session_info_for_head( - &mut ctx, - &mut window, - hash, - &header, - ).await.unwrap(); - - assert_eq!(window.earliest_session, Some(session)); - assert_eq!( - window.session_info, - vec![dummy_session_info(session)], - ); - }) - }; - - let aux_fut = Box::pin(async move { - assert_matches!( - handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - h, - RuntimeApiRequest::SessionIndexForChild(s_tx), - )) => { - assert_eq!(h, hash); - let _ = s_tx.send(Ok(session)); - } - ); - - assert_matches!( - handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - h, - RuntimeApiRequest::SessionInfo(s, s_tx), - )) => { - assert_eq!(h, hash); - assert_eq!(s, session); - - let _ = s_tx.send(Ok(Some(dummy_session_info(s)))); - } - ); - }); - - futures::executor::block_on(futures::future::join(test_fut, aux_fut)); - } } diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 8aede5450e0c..54943c40b660 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -23,30 +23,31 @@ use polkadot_node_subsystem::{ messages::{ - AssignmentCheckResult, ApprovalCheckResult, ApprovalVotingMessage, - RuntimeApiMessage, RuntimeApiRequest, ChainApiMessage, ApprovalDistributionMessage, - ValidationFailed, CandidateValidationMessage, AvailabilityRecoveryMessage, + AssignmentCheckError, AssignmentCheckResult, ApprovalCheckError, ApprovalCheckResult, + ApprovalVotingMessage, RuntimeApiMessage, RuntimeApiRequest, ChainApiMessage, + ApprovalDistributionMessage, CandidateValidationMessage, + AvailabilityRecoveryMessage, }, errors::RecoveryError, Subsystem, SubsystemContext, SubsystemError, SubsystemResult, SpawnedSubsystem, - FromOverseer, OverseerSignal, + FromOverseer, OverseerSignal, SubsystemSender, }; use polkadot_node_subsystem_util::{ + TimeoutExt, metrics::{self, prometheus}, + rolling_session_window::RollingSessionWindow, }; use polkadot_primitives::v1::{ ValidatorIndex, Hash, SessionIndex, SessionInfo, CandidateHash, - CandidateReceipt, BlockNumber, PersistedValidationData, - ValidationCode, CandidateDescriptor, ValidatorPair, ValidatorSignature, ValidatorId, - CandidateIndex, GroupIndex, + CandidateReceipt, BlockNumber, + ValidatorPair, ValidatorSignature, ValidatorId, + CandidateIndex, GroupIndex, ApprovalVote, }; -use polkadot_node_primitives::{ValidationResult, PoV}; +use polkadot_node_primitives::ValidationResult; use polkadot_node_primitives::approval::{ - IndirectAssignmentCert, IndirectSignedApprovalVote, ApprovalVote, DelayTranche, - BlockApprovalMeta, + IndirectAssignmentCert, IndirectSignedApprovalVote, DelayTranche, BlockApprovalMeta, }; use polkadot_node_jaeger as jaeger; -use parity_scale_codec::Encode; use sc_keystore::LocalKeystore; use sp_consensus::SyncOracle; use sp_consensus_slots::Slot; @@ -55,12 +56,14 @@ use sp_application_crypto::Pair; use kvdb::KeyValueDB; use futures::prelude::*; -use futures::future::RemoteHandle; -use futures::channel::{mpsc, oneshot}; +use futures::future::{BoxFuture, RemoteHandle}; +use futures::channel::oneshot; +use futures::stream::FuturesUnordered; use std::collections::{BTreeMap, HashMap, HashSet}; use std::collections::btree_map::Entry; use std::sync::Arc; +use std::time::Duration; use approval_checking::RequiredTranches; use persisted_entries::{ApprovalEntry, CandidateEntry, BlockEntry}; @@ -80,6 +83,8 @@ use crate::approval_db::v1::Config as DatabaseConfig; mod tests; const APPROVAL_SESSIONS: SessionIndex = 6; +const APPROVAL_CHECKING_TIMEOUT: Duration = Duration::from_secs(120); +const APPROVAL_CACHE_SIZE: usize = 1024; const LOG_TARGET: &str = "parachain::approval-voting"; /// Configuration for the approval voting subsystem @@ -122,13 +127,14 @@ pub struct ApprovalVotingSubsystem { #[derive(Clone)] struct MetricsInner { imported_candidates_total: prometheus::Counter, - assignments_produced_total: prometheus::Counter, - approvals_produced_total: prometheus::Counter, + assignments_produced: prometheus::Histogram, + approvals_produced_total: prometheus::CounterVec, no_shows_total: prometheus::Counter, wakeups_triggered_total: prometheus::Counter, candidate_approval_time_ticks: prometheus::Histogram, block_approval_time_ticks: prometheus::Histogram, time_db_transaction: prometheus::Histogram, + time_recover_and_approve: prometheus::Histogram, } /// Aproval Voting metrics. @@ -142,15 +148,39 @@ impl Metrics { } } - fn on_assignment_produced(&self) { + fn on_assignment_produced(&self, tranche: DelayTranche) { if let Some(metrics) = &self.0 { - metrics.assignments_produced_total.inc(); + metrics.assignments_produced.observe(tranche as f64); + } + } + + fn on_approval_stale(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_produced_total.with_label_values(&["stale"]).inc() + } + } + + fn on_approval_invalid(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_produced_total.with_label_values(&["invalid"]).inc() + } + } + + fn on_approval_unavailable(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_produced_total.with_label_values(&["unavailable"]).inc() + } + } + + fn on_approval_error(&self) { + if let Some(metrics) = &self.0 { + metrics.approvals_produced_total.with_label_values(&["internal error"]).inc() } } fn on_approval_produced(&self) { if let Some(metrics) = &self.0 { - metrics.approvals_produced_total.inc(); + metrics.approvals_produced_total.with_label_values(&["success"]).inc() } } @@ -181,6 +211,10 @@ impl Metrics { fn time_db_transaction(&self) -> Option { self.0.as_ref().map(|metrics| metrics.time_db_transaction.start_timer()) } + + fn time_recover_and_approve(&self) -> Option { + self.0.as_ref().map(|metrics| metrics.time_recover_and_approve.start_timer()) + } } impl metrics::Metrics for Metrics { @@ -195,17 +229,22 @@ impl metrics::Metrics for Metrics { )?, registry, )?, - assignments_produced_total: prometheus::register( - prometheus::Counter::new( - "parachain_assignments_produced_total", - "Number of assignments produced by the approval voting subsystem", + assignments_produced: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "parachain_assignments_produced", + "Assignments and tranches produced by the approval voting subsystem", + ).buckets(vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 15.0, 25.0, 40.0, 70.0]), )?, registry, )?, approvals_produced_total: prometheus::register( - prometheus::Counter::new( - "parachain_approvals_produced_total", - "Number of approvals produced by the approval voting subsystem", + prometheus::CounterVec::new( + prometheus::Opts::new( + "parachain_approvals_produced_total", + "Number of approvals produced by the approval voting subsystem", + ), + &["status"] )?, registry, )?, @@ -250,6 +289,15 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + time_recover_and_approve: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "parachain_time_recover_and_approve", + "Time spent recovering and approving data in approval voting", + ) + )?, + registry, + )?, }; Ok(Metrics(Some(metrics))) @@ -298,21 +346,10 @@ impl Subsystem for ApprovalVotingSubsystem } } -enum BackgroundRequest { - ApprovalVote(ApprovalVoteRequest), - CandidateValidation( - PersistedValidationData, - ValidationCode, - CandidateDescriptor, - Arc, - oneshot::Sender>, - ), -} - +#[derive(Debug, Clone)] struct ApprovalVoteRequest { validator_index: ValidatorIndex, block_hash: Hash, - candidate_index: usize, } #[derive(Default)] @@ -496,8 +533,111 @@ struct ApprovalStatus { block_tick: Tick, } +#[derive(Copy, Clone)] +enum ApprovalOutcome { + Approved, + Failed, + TimedOut, +} + +struct ApprovalState { + validator_index: ValidatorIndex, + candidate_hash: CandidateHash, + approval_outcome: ApprovalOutcome, +} + +impl ApprovalState { + fn approved( + validator_index: ValidatorIndex, + candidate_hash: CandidateHash, + ) -> Self { + Self { + validator_index, + candidate_hash, + approval_outcome: ApprovalOutcome::Approved, + } + } + fn failed( + validator_index: ValidatorIndex, + candidate_hash: CandidateHash, + ) -> Self { + Self { + validator_index, + candidate_hash, + approval_outcome: ApprovalOutcome::Failed, + } + } +} + +struct CurrentlyCheckingSet { + candidate_hash_map: HashMap>, + currently_checking: FuturesUnordered>, +} + +impl Default for CurrentlyCheckingSet { + fn default() -> Self { + Self { + candidate_hash_map: HashMap::new(), + currently_checking: FuturesUnordered::new(), + } + } +} + +impl CurrentlyCheckingSet { + // This function will lazily launch approval voting work whenever the + // candidate is not already undergoing validation. + pub async fn insert_relay_block_hash( + &mut self, + candidate_hash: CandidateHash, + validator_index: ValidatorIndex, + relay_block: Hash, + launch_work: impl Future>>, + ) -> SubsystemResult<()> { + let val = self.candidate_hash_map + .entry(candidate_hash) + .or_insert(Default::default()); + + if let Err(k) = val.binary_search_by_key(&relay_block, |v| *v) { + let _ = val.insert(k, relay_block); + let work = launch_work.await?; + self.currently_checking.push( + Box::pin(async move { + match work.timeout(APPROVAL_CHECKING_TIMEOUT).await { + None => ApprovalState { + candidate_hash, + validator_index, + approval_outcome: ApprovalOutcome::TimedOut, + }, + Some(approval_state) => approval_state, + } + }) + ); + } + + Ok(()) + } + + pub async fn next( + &mut self, + approvals_cache: &mut lru::LruCache, + ) -> (Vec, ApprovalState) { + if !self.currently_checking.is_empty() { + if let Some(approval_state) = self.currently_checking + .next() + .await + { + let out = self.candidate_hash_map.remove(&approval_state.candidate_hash).unwrap_or_default(); + approvals_cache.put(approval_state.candidate_hash.clone(), approval_state.approval_outcome.clone()); + return (out, approval_state); + } + } + + future::pending().await + } +} + struct State { - session_window: import::RollingSessionWindow, + session_window: RollingSessionWindow, keystore: Arc, slot_duration_millis: u64, db: T, @@ -557,7 +697,7 @@ impl State { } } -#[derive(Debug)] +#[derive(Debug, Clone)] enum Action { ScheduleWakeup { block_hash: Hash, @@ -568,19 +708,20 @@ enum Action { WriteBlockEntry(BlockEntry), WriteCandidateEntry(CandidateHash, CandidateEntry), LaunchApproval { + candidate_hash: CandidateHash, indirect_cert: IndirectAssignmentCert, - relay_block_number: BlockNumber, + assignment_tranche: DelayTranche, + relay_block_hash: Hash, candidate_index: CandidateIndex, session: SessionIndex, candidate: CandidateReceipt, backing_group: GroupIndex, }, + IssueApproval(CandidateHash, ApprovalVoteRequest), BecomeActive, Conclude, } -type BackgroundTaskMap = BTreeMap>>; - async fn run( mut ctx: C, mut subsystem: ApprovalVotingSubsystem, @@ -589,9 +730,8 @@ async fn run( ) -> SubsystemResult<()> where C: SubsystemContext { - let (background_tx, background_rx) = mpsc::channel::(64); let mut state = State { - session_window: Default::default(), + session_window: RollingSessionWindow::new(APPROVAL_SESSIONS), keystore: subsystem.keystore, slot_duration_millis: subsystem.slot_duration_millis, db: ApprovalDBV1Reader::new(subsystem.db.clone(), subsystem.db_config.clone()), @@ -600,12 +740,10 @@ async fn run( }; let mut wakeups = Wakeups::default(); - - // map block numbers to background work. - let mut background_tasks = BTreeMap::new(); + let mut currently_checking_set = CurrentlyCheckingSet::default(); + let mut approvals_cache = lru::LruCache::new(APPROVAL_CACHE_SIZE); let mut last_finalized_height: Option = None; - let mut background_rx = background_rx.fuse(); let db_writer = &*subsystem.db; @@ -632,10 +770,6 @@ async fn run( &mut wakeups, ).await?; - if let Some(finalized_height) = last_finalized_height { - cleanup_background_tasks(finalized_height, &mut background_tasks); - } - if let Mode::Syncing(ref mut oracle) = subsystem.mode { if !oracle.is_major_syncing() { // note that we're active before processing other actions. @@ -645,28 +779,46 @@ async fn run( actions } - background_request = background_rx.next().fuse() => { - if let Some(req) = background_request { - handle_background_request( - &mut ctx, - &mut state, - &subsystem.metrics, - req, - ).await? - } else { - Vec::new() + approval_state = currently_checking_set.next(&mut approvals_cache).fuse() => { + let mut actions = Vec::new(); + let ( + relay_block_hashes, + ApprovalState { + validator_index, + candidate_hash, + approval_outcome, + } + ) = approval_state; + + if matches!(approval_outcome, ApprovalOutcome::Approved) { + let mut approvals: Vec = relay_block_hashes + .into_iter() + .map(|block_hash| + Action::IssueApproval( + candidate_hash, + ApprovalVoteRequest { + validator_index, + block_hash, + }, + ) + ) + .collect(); + actions.append(&mut approvals); } + + actions } }; if handle_actions( &mut ctx, + &mut state, &subsystem.metrics, &mut wakeups, + &mut currently_checking_set, + &mut approvals_cache, db_writer, subsystem.db_config, - &background_tx, - &mut background_tasks, &mut subsystem.mode, actions, ).await? { @@ -677,22 +829,42 @@ async fn run( Ok(()) } +// Handle actions is a function that accepts a set of instructions +// and subsequently updates the underlying approvals_db in accordance +// with the linear set of instructions passed in. Therefore, actions +// must be processed in series to ensure that earlier actions are not +// negated/corrupted by later actions being executed out-of-order. +// +// However, certain Actions can cause additional actions to need to be +// processed by this function. In order to preserve linearity, we would +// need to handle these newly generated actions before we finalize +// completing additional actions in the submitted sequence of actions. +// +// Since recursive async functions are not not stable yet, we are +// forced to modify the actions iterator on the fly whenever a new set +// of actions are generated by handling a single action. +// +// This particular problem statement is specified in issue 3311: +// https://github.com/paritytech/polkadot/issues/3311 +// // returns `true` if any of the actions was a `Conclude` command. async fn handle_actions( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, + state: &mut State, metrics: &Metrics, wakeups: &mut Wakeups, + currently_checking_set: &mut CurrentlyCheckingSet, + approvals_cache: &mut lru::LruCache, db: &dyn KeyValueDB, db_config: DatabaseConfig, - background_tx: &mpsc::Sender, - background_tasks: &mut BackgroundTaskMap, mode: &mut Mode, - actions: impl IntoIterator, + actions: Vec, ) -> SubsystemResult { let mut transaction = approval_db::v1::Transaction::new(db_config); let mut conclude = false; - for action in actions { + let mut actions_iter = actions.into_iter(); + while let Some(action) = actions_iter.next() { match action { Action::ScheduleWakeup { block_hash, @@ -708,9 +880,33 @@ async fn handle_actions( Action::WriteCandidateEntry(candidate_hash, candidate_entry) => { transaction.put_candidate_entry(candidate_hash, candidate_entry.into()); } + Action::IssueApproval(candidate_hash, approval_request) => { + let mut sender = ctx.sender().clone(); + // Note that the IssueApproval action will create additional + // actions that will need to all be processed before we can + // handle the next action in the set passed to the ambient + // function. + // + // In order to achieve this, we append the existing iterator + // to the end of the iterator made up of these newly generated + // actions. + // + // Note that chaining these iterators is O(n) as we must consume + // the prior iterator. + let next_actions: Vec = issue_approval( + &mut sender, + state, + metrics, + candidate_hash, + approval_request, + )?.into_iter().map(|v| v.clone()).chain(actions_iter).collect(); + actions_iter = next_actions.into_iter(); + } Action::LaunchApproval { + candidate_hash, indirect_cert, - relay_block_number, + assignment_tranche, + relay_block_hash, candidate_index, session, candidate, @@ -719,7 +915,7 @@ async fn handle_actions( // Don't launch approval work if the node is syncing. if let Mode::Syncing(_) = *mode { continue } - metrics.on_assignment_produced(); + metrics.on_assignment_produced(assignment_tranche); let block_hash = indirect_cert.block_hash; let validator_index = indirect_cert.validator; @@ -728,19 +924,42 @@ async fn handle_actions( candidate_index, ).into()); - let handle = launch_approval( - ctx, - background_tx.clone(), - session, - &candidate, - validator_index, - block_hash, - candidate_index as _, - backing_group, - ).await?; - - if let Some(handle) = handle { - background_tasks.entry(relay_block_number).or_default().push(handle); + match approvals_cache.get(&candidate_hash) { + Some(ApprovalOutcome::Approved) => { + let new_actions: Vec = std::iter::once( + Action::IssueApproval( + candidate_hash, + ApprovalVoteRequest { + validator_index, + block_hash, + } + ) + ) + .map(|v| v.clone()) + .chain(actions_iter) + .collect(); + actions_iter = new_actions.into_iter(); + }, + None => { + let ctx = &mut *ctx; + currently_checking_set.insert_relay_block_hash( + candidate_hash, + validator_index, + relay_block_hash, + async move { + launch_approval( + ctx, + metrics.clone(), + session, + candidate, + validator_index, + block_hash, + backing_group, + ).await + } + ).await?; + } + Some(_) => {}, } } Action::BecomeActive => { @@ -766,19 +985,6 @@ async fn handle_actions( Ok(conclude) } -// Clean up all background tasks which are no longer needed as they correspond to a -// finalized block. -fn cleanup_background_tasks( - current_finalized_block: BlockNumber, - tasks: &mut BackgroundTaskMap, -) { - let after = tasks.split_off(&(current_finalized_block + 1)); - *tasks = after; - - // tasks up to the finalized block are dropped, and `RemoteHandle` cancels - // the task on drop. -} - fn distribution_messages_for_activation<'a>( db: impl DBReader + 'a, ) -> SubsystemResult> { @@ -878,7 +1084,7 @@ fn distribution_messages_for_activation<'a>( // Handle an incoming signal from the overseer. Returns true if execution should conclude. async fn handle_from_overseer( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, state: &mut State, metrics: &Metrics, db_writer: &dyn KeyValueDB, @@ -991,38 +1197,8 @@ async fn handle_from_overseer( Ok(actions) } -async fn handle_background_request( - ctx: &mut impl SubsystemContext, - state: &State, - metrics: &Metrics, - request: BackgroundRequest, -) -> SubsystemResult> { - match request { - BackgroundRequest::ApprovalVote(vote_request) => { - issue_approval(ctx, state, metrics, vote_request) - } - BackgroundRequest::CandidateValidation( - validation_data, - validation_code, - descriptor, - pov, - tx, - ) => { - ctx.send_message(CandidateValidationMessage::ValidateFromExhaustive( - validation_data, - validation_code, - descriptor, - pov, - tx, - ).into()).await; - - Ok(Vec::new()) - } - } -} - async fn handle_approved_ancestor( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, db: &impl DBReader, target: Hash, lower_bound: BlockNumber, @@ -1226,15 +1402,6 @@ async fn handle_approved_ancestor( Ok(all_approved_max) } -fn approval_signing_payload( - approval_vote: ApprovalVote, - session_index: SessionIndex, -) -> Vec { - const MAGIC: [u8; 4] = *b"APPR"; - - (MAGIC, approval_vote, session_index).encode() -} - // `Option::cmp` treats `None` as less than `Some`. fn min_prefer_some( a: Option, @@ -1329,14 +1496,17 @@ fn check_and_import_assignment( let tick_now = state.clock.tick_now(); let block_entry = match state.db.load_block_entry(&assignment.block_hash)? { Some(b) => b, - None => return Ok((AssignmentCheckResult::Bad, Vec::new())), + None => return Ok((AssignmentCheckResult::Bad( + AssignmentCheckError::UnknownBlock(assignment.block_hash), + ), Vec::new())), }; let session_info = match state.session_info(block_entry.session()) { Some(s) => s, None => { - tracing::warn!(target: LOG_TARGET, "Unknown session info for {}", block_entry.session()); - return Ok((AssignmentCheckResult::Bad, Vec::new())); + return Ok((AssignmentCheckResult::Bad( + AssignmentCheckError::UnknownSessionIndex(block_entry.session()), + ), Vec::new())); } }; @@ -1344,20 +1514,17 @@ fn check_and_import_assignment( = match block_entry.candidate(candidate_index as usize) { Some((c, h)) => (*c, *h), - None => return Ok((AssignmentCheckResult::Bad, Vec::new())), // no candidate at core. + None => return Ok((AssignmentCheckResult::Bad( + AssignmentCheckError::InvalidCandidateIndex(candidate_index), + ), Vec::new())), // no candidate at core. }; let mut candidate_entry = match state.db.load_candidate_entry(&assigned_candidate_hash)? { Some(c) => c, None => { - tracing::warn!( - target: LOG_TARGET, - "Missing candidate entry {} referenced in live block {}", - assigned_candidate_hash, - assignment.block_hash, - ); - - return Ok((AssignmentCheckResult::Bad, Vec::new())); + return Ok((AssignmentCheckResult::Bad( + AssignmentCheckError::InvalidCandidate(candidate_index, assigned_candidate_hash), + ), Vec::new())); } }; @@ -1367,7 +1534,9 @@ fn check_and_import_assignment( candidate_entry.approval_entry_mut(&assignment.block_hash) { Some(a) => a, - None => return Ok((AssignmentCheckResult::Bad, Vec::new())), + None => return Ok((AssignmentCheckResult::Bad( + AssignmentCheckError::Internal(assignment.block_hash, assigned_candidate_hash), + ), Vec::new())), }; let res = state.assignment_criteria.check_assignment_cert( @@ -1380,7 +1549,9 @@ fn check_and_import_assignment( ); let tranche = match res { - Err(crate::criteria::InvalidAssignment) => return Ok((AssignmentCheckResult::Bad, Vec::new())), + Err(crate::criteria::InvalidAssignment) => return Ok((AssignmentCheckResult::Bad( + AssignmentCheckError::InvalidCert(assignment.validator), + ), Vec::new())), Ok(tranche) => { let current_tranche = state.clock.tranche_now( state.slot_duration_millis, @@ -1450,56 +1621,69 @@ fn check_and_import_approval( let block_entry = match state.db.load_block_entry(&approval.block_hash)? { Some(b) => b, - None => respond_early!(ApprovalCheckResult::Bad) + None => { + respond_early!(ApprovalCheckResult::Bad( + ApprovalCheckError::UnknownBlock(approval.block_hash), + )) + } }; let session_info = match state.session_info(block_entry.session()) { Some(s) => s, None => { - tracing::warn!(target: LOG_TARGET, "Unknown session info for {}", block_entry.session()); - respond_early!(ApprovalCheckResult::Bad) + respond_early!(ApprovalCheckResult::Bad( + ApprovalCheckError::UnknownSessionIndex(block_entry.session()), + )) } }; let approved_candidate_hash = match block_entry.candidate(approval.candidate_index as usize) { Some((_, h)) => *h, - None => respond_early!(ApprovalCheckResult::Bad) + None => respond_early!(ApprovalCheckResult::Bad( + ApprovalCheckError::InvalidCandidateIndex(approval.candidate_index), + )) }; - let approval_payload = approval_signing_payload( - ApprovalVote(approved_candidate_hash), - block_entry.session(), - ); + let approval_payload = ApprovalVote(approved_candidate_hash) + .signing_payload(block_entry.session()); let pubkey = match session_info.validators.get(approval.validator.0 as usize) { Some(k) => k, - None => respond_early!(ApprovalCheckResult::Bad) + None => respond_early!(ApprovalCheckResult::Bad( + ApprovalCheckError::InvalidValidatorIndex(approval.validator), + )) }; let approval_sig_valid = approval.signature.verify(approval_payload.as_slice(), pubkey); if !approval_sig_valid { - respond_early!(ApprovalCheckResult::Bad) + respond_early!(ApprovalCheckResult::Bad( + ApprovalCheckError::InvalidSignature(approval.validator), + )) } let candidate_entry = match state.db.load_candidate_entry(&approved_candidate_hash)? { Some(c) => c, None => { - tracing::warn!( - target: LOG_TARGET, - "Unknown candidate entry for {}", - approved_candidate_hash, - ); - - respond_early!(ApprovalCheckResult::Bad) + respond_early!(ApprovalCheckResult::Bad( + ApprovalCheckError::InvalidCandidate(approval.candidate_index, approved_candidate_hash), + )) } }; // Don't accept approvals until assignment. - if candidate_entry.approval_entry(&approval.block_hash) - .map_or(true, |e| !e.is_assigned(approval.validator)) - { - respond_early!(ApprovalCheckResult::Bad) + match candidate_entry.approval_entry(&approval.block_hash) { + None => { + respond_early!(ApprovalCheckResult::Bad( + ApprovalCheckError::Internal(approval.block_hash, approved_candidate_hash), + )) + } + Some(e) if !e.is_assigned(approval.validator) => { + respond_early!(ApprovalCheckResult::Bad( + ApprovalCheckError::NoAssignment(approval.validator), + )) + } + _ => {}, } // importing the approval can be heavy as it may trigger acceptance for a series of blocks. @@ -1807,7 +1991,7 @@ fn process_wakeup( (Vec::new(), None) }; - if let Some((cert, val_index)) = maybe_cert { + if let Some((cert, val_index, tranche)) = maybe_cert { let indirect_cert = IndirectAssignmentCert { block_hash: relay_block, validator: val_index, @@ -1828,8 +2012,10 @@ fn process_wakeup( // sanity: should always be present. actions.push(Action::LaunchApproval { + candidate_hash, indirect_cert, - relay_block_number: block_entry.block_number(), + assignment_tranche: tranche, + relay_block_hash: relay_block, candidate_index: i as _, session: block_entry.session(), candidate: candidate_entry.candidate_receipt().clone(), @@ -1868,18 +2054,39 @@ fn process_wakeup( // spawned. When the background work is no longer needed, the `AbortHandle` should be dropped // to cancel the background work and any requests it has spawned. async fn launch_approval( - ctx: &mut impl SubsystemContext, - mut background_tx: mpsc::Sender, + ctx: &mut impl SubsystemContext, + metrics: Metrics, session_index: SessionIndex, - candidate: &CandidateReceipt, + candidate: CandidateReceipt, validator_index: ValidatorIndex, block_hash: Hash, - candidate_index: usize, backing_group: GroupIndex, -) -> SubsystemResult>> { +) -> SubsystemResult> { let (a_tx, a_rx) = oneshot::channel(); let (code_tx, code_rx) = oneshot::channel(); - let (context_num_tx, context_num_rx) = oneshot::channel(); + + // The background future returned by this function may + // be dropped before completing. This guard is used to ensure that the approval + // work is correctly counted as stale even if so. + struct StaleGuard(Option); + + impl StaleGuard { + fn take(mut self) -> Metrics { + self.0.take().expect(" + consumed after take; so this cannot be called twice; \ + nothing in this function reaches into the struct to avoid this API; \ + qed + ") + } + } + + impl Drop for StaleGuard { + fn drop(&mut self) { + if let Some(metrics) = self.0.as_ref() { + metrics.on_approval_stale(); + } + } + } let candidate_hash = candidate.hash(); @@ -1890,6 +2097,7 @@ async fn launch_approval( "Recovering data.", ); + let timer = metrics.time_recover_and_approve(); ctx.send_message(AvailabilityRecoveryMessage::RecoverAvailableData( candidate.clone(), session_index, @@ -1897,70 +2105,74 @@ async fn launch_approval( a_tx, ).into()).await; - ctx.send_message( - ChainApiMessage::BlockNumber(candidate.descriptor.relay_parent, context_num_tx).into() - ).await; - - let in_context_number = match context_num_rx.await { - Ok(Ok(Some(n))) => n, - Ok(Ok(None)) | Ok(Err(_)) | Err(_) => { - tracing::warn!( - target: LOG_TARGET, - "Could not launch approval work for candidate {:?}: Number of block {} unknown", - (candidate_hash, candidate.descriptor.para_id), - candidate.descriptor.relay_parent, - ); - - return Ok(None); - } - }; - ctx.send_message( RuntimeApiMessage::Request( block_hash, - RuntimeApiRequest::HistoricalValidationCode( - candidate.descriptor.para_id, - in_context_number, + RuntimeApiRequest::ValidationCodeByHash( + candidate.descriptor.validation_code_hash, code_tx, ), ).into() ).await; let candidate = candidate.clone(); + let metrics_guard = StaleGuard(Some(metrics)); + let mut sender = ctx.sender().clone(); let background = async move { + // Force the move of the timer into the background task. + let _timer = timer; let _span = jaeger::Span::from_encodable((block_hash, candidate_hash), "launch-approval") .with_relay_parent(block_hash) .with_candidate(candidate_hash) .with_stage(jaeger::Stage::ApprovalChecking); let available_data = match a_rx.await { - Err(_) => return, + Err(_) => return ApprovalState::failed( + validator_index, + candidate_hash, + ), Ok(Ok(a)) => a, - Ok(Err(RecoveryError::Unavailable)) => { - tracing::warn!( - target: LOG_TARGET, - "Data unavailable for candidate {:?}", - (candidate_hash, candidate.descriptor.para_id), - ); - // do nothing. we'll just be a no-show and that'll cause others to rise up. - return; - } - Ok(Err(RecoveryError::Invalid)) => { - tracing::warn!( - target: LOG_TARGET, - "Data recovery invalid for candidate {:?}", - (candidate_hash, candidate.descriptor.para_id), - ); + Ok(Err(e)) => { + match &e { + &RecoveryError::Unavailable => { + tracing::warn!( + target: LOG_TARGET, + "Data unavailable for candidate {:?}", + (candidate_hash, candidate.descriptor.para_id), + ); + // do nothing. we'll just be a no-show and that'll cause others to rise up. + metrics_guard.take().on_approval_unavailable(); + } + &RecoveryError::Invalid => { + tracing::warn!( + target: LOG_TARGET, + "Data recovery invalid for candidate {:?}", + (candidate_hash, candidate.descriptor.para_id), + ); - // TODO: dispute. Either the merkle trie is bad or the erasure root is. - // https://github.com/paritytech/polkadot/issues/2176 - return; + // TODO: dispute. Either the merkle trie is bad or the erasure root is. + // https://github.com/paritytech/polkadot/issues/2176 + metrics_guard.take().on_approval_invalid(); + } + } + return ApprovalState::failed( + validator_index, + candidate_hash, + ); } }; let validation_code = match code_rx.await { - Err(_) => return, - Ok(Err(_)) => return, + Err(_) => + return ApprovalState::failed( + validator_index, + candidate_hash, + ), + Ok(Err(_)) => + return ApprovalState::failed( + validator_index, + candidate_hash, + ), Ok(Ok(Some(code))) => code, Ok(Ok(None)) => { tracing::warn!( @@ -1972,23 +2184,32 @@ async fn launch_approval( // No dispute necessary, as this indicates that the chain is not behaving // according to expectations. - return; + metrics_guard.take().on_approval_unavailable(); + return ApprovalState::failed( + validator_index, + candidate_hash, + ); } }; let (val_tx, val_rx) = oneshot::channel(); let para_id = candidate.descriptor.para_id; - let _ = background_tx.send(BackgroundRequest::CandidateValidation( + + sender.send_message(CandidateValidationMessage::ValidateFromExhaustive( available_data.validation_data, validation_code, candidate.descriptor, available_data.pov, val_tx, - )).await; + ).into()).await; match val_rx.await { - Err(_) => return, + Err(_) => + return ApprovalState::failed( + validator_index, + candidate_hash, + ), Ok(Ok(ValidationResult::Valid(_, _))) => { // Validation checked out. Issue an approval command. If the underlying service is unreachable, // then there isn't anything we can do. @@ -2000,11 +2221,11 @@ async fn launch_approval( "Candidate Valid", ); - let _ = background_tx.send(BackgroundRequest::ApprovalVote(ApprovalVoteRequest { + let _ = metrics_guard.take(); + return ApprovalState::approved( validator_index, - block_hash, - candidate_index, - })).await; + candidate_hash, + ); } Ok(Ok(ValidationResult::Invalid(reason))) => { tracing::warn!( @@ -2017,6 +2238,12 @@ async fn launch_approval( // TODO: issue dispute, but not for timeouts. // https://github.com/paritytech/polkadot/issues/2176 + metrics_guard.take().on_approval_invalid(); + + return ApprovalState::failed( + validator_index, + candidate_hash, + ); } Ok(Err(e)) => { tracing::error!( @@ -2024,31 +2251,55 @@ async fn launch_approval( err = ?e, "Failed to validate candidate due to internal error", ); - - return + metrics_guard.take().on_approval_error(); + return ApprovalState::failed( + validator_index, + candidate_hash, + ); } } }; let (background, remote_handle) = background.remote_handle(); ctx.spawn("approval-checks", Box::pin(background)) - .await - .map(move |()| Some(remote_handle)) + .map(move |()| remote_handle) } // Issue and import a local approval vote. Should only be invoked after approval checks // have been done. fn issue_approval( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemSender, state: &State, metrics: &Metrics, - request: ApprovalVoteRequest, + candidate_hash: CandidateHash, + ApprovalVoteRequest { validator_index, block_hash }: ApprovalVoteRequest, ) -> SubsystemResult> { - let ApprovalVoteRequest { validator_index, block_hash, candidate_index } = request; - let block_entry = match state.db.load_block_entry(&block_hash)? { Some(b) => b, - None => return Ok(Vec::new()), // not a cause for alarm - just lost a race with pruning, most likely. + None => { + // not a cause for alarm - just lost a race with pruning, most likely. + metrics.on_approval_stale(); + return Ok(Vec::new()) + } + }; + + let candidate_index = match block_entry + .candidates() + .iter() + .position(|e| e.1 == candidate_hash) + { + None => { + tracing::warn!( + target: LOG_TARGET, + "Candidate hash {} is not present in the block entry's candidates for relay block {}", + candidate_hash, + block_entry.parent_hash(), + ); + + metrics.on_approval_error(); + return Ok(Vec::new()); + } + Some(idx) => idx, }; let session_info = match state.session_info(block_entry.session()) { @@ -2061,11 +2312,12 @@ fn issue_approval( block_entry.session(), ); + metrics.on_approval_error(); return Ok(Vec::new()); } }; - let candidate_hash = match block_entry.candidate(candidate_index) { + let candidate_hash = match block_entry.candidate(candidate_index as usize) { Some((_, h)) => h.clone(), None => { tracing::warn!( @@ -2075,6 +2327,7 @@ fn issue_approval( block_hash, ); + metrics.on_approval_error(); return Ok(Vec::new()); } }; @@ -2089,6 +2342,7 @@ fn issue_approval( block_hash, ); + metrics.on_approval_error(); return Ok(Vec::new()); } }; @@ -2103,6 +2357,7 @@ fn issue_approval( block_entry.session(), ); + metrics.on_approval_error(); return Ok(Vec::new()); } }; @@ -2122,6 +2377,7 @@ fn issue_approval( block_entry.session(), ); + metrics.on_approval_error(); return Ok(Vec::new()); } }; @@ -2167,10 +2423,7 @@ fn sign_approval( ) -> Option { let key = keystore.key_pair::(public).ok().flatten()?; - let payload = approval_signing_payload( - ApprovalVote(candidate_hash), - session_index, - ); + let payload = ApprovalVote(candidate_hash).signing_payload(session_index); Some(key.sign(&payload[..])) } diff --git a/node/core/approval-voting/src/persisted_entries.rs b/node/core/approval-voting/src/persisted_entries.rs index 646fdb19fe93..ce0f5689c154 100644 --- a/node/core/approval-voting/src/persisted_entries.rs +++ b/node/core/approval-voting/src/persisted_entries.rs @@ -93,7 +93,7 @@ impl ApprovalEntry { // Note that our assignment is triggered. No-op if already triggered. pub fn trigger_our_assignment(&mut self, tick_now: Tick) - -> Option<(AssignmentCert, ValidatorIndex)> + -> Option<(AssignmentCert, ValidatorIndex, DelayTranche)> { let our = self.our_assignment.as_mut().and_then(|a| { if a.triggered() { return None } @@ -105,7 +105,7 @@ impl ApprovalEntry { our.map(|a| { self.import_assignment(a.tranche(), a.validator_index(), tick_now); - (a.cert().clone(), a.validator_index()) + (a.cert().clone(), a.validator_index(), a.tranche()) }) } diff --git a/node/core/approval-voting/src/tests.rs b/node/core/approval-voting/src/tests.rs index e6805479b044..5603c362fd24 100644 --- a/node/core/approval-voting/src/tests.rs +++ b/node/core/approval-voting/src/tests.rs @@ -21,6 +21,7 @@ use polkadot_node_primitives::approval::{ RELAY_VRF_MODULO_CONTEXT, DelayTranche, }; use polkadot_node_subsystem_test_helpers::make_subsystem_context; +use polkadot_node_subsystem::messages::AllMessages; use sp_core::testing::TaskExecutor; use parking_lot::Mutex; @@ -190,7 +191,7 @@ impl DBReader for TestStore { fn blank_state() -> State { State { - session_window: import::RollingSessionWindow::default(), + session_window: RollingSessionWindow::new(APPROVAL_SESSIONS), keystore: Arc::new(LocalKeystore::in_memory()), slot_duration_millis: SLOT_DURATION_MILLIS, db: TestStore::default(), @@ -203,10 +204,11 @@ fn single_session_state(index: SessionIndex, info: SessionInfo) -> State { State { - session_window: import::RollingSessionWindow { - earliest_session: Some(index), - session_info: vec![info], - }, + session_window: RollingSessionWindow::with_session_info( + APPROVAL_SESSIONS, + index, + vec![info], + ), ..blank_state() } } @@ -230,7 +232,7 @@ fn sign_approval( candidate_hash: CandidateHash, session_index: SessionIndex, ) -> ValidatorSignature { - key.sign(&super::approval_signing_payload(ApprovalVote(candidate_hash), session_index)).into() + key.sign(&ApprovalVote(candidate_hash).signing_payload(session_index)).into() } #[derive(Clone)] @@ -395,8 +397,9 @@ fn rejects_bad_assignment() { assert!(res.1.iter().any(|action| matches!(action, Action::WriteCandidateEntry(..)))); // unknown hash + let unknown_hash = Hash::repeat_byte(0x02); let assignment = IndirectAssignmentCert { - block_hash: Hash::repeat_byte(0x02), + block_hash: unknown_hash, validator: ValidatorIndex(0), cert: garbage_assignment_cert( AssignmentCertKind::RelayVRFModulo { @@ -410,7 +413,7 @@ fn rejects_bad_assignment() { assignment, candidate_index, ).unwrap(); - assert_eq!(res.0, AssignmentCheckResult::Bad); + assert_eq!(res.0, AssignmentCheckResult::Bad(AssignmentCheckError::UnknownBlock(unknown_hash))); let mut state = State { assignment_criteria: Box::new(MockAssignmentCriteria::check_only(|| { @@ -425,7 +428,7 @@ fn rejects_bad_assignment() { assignment_good, candidate_index, ).unwrap(); - assert_eq!(res.0, AssignmentCheckResult::Bad); + assert_eq!(res.0, AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCert(ValidatorIndex(0)))); } #[test] @@ -493,7 +496,7 @@ fn rejects_assignment_with_unknown_candidate() { assignment.clone(), candidate_index, ).unwrap(); - assert_eq!(res.0, AssignmentCheckResult::Bad); + assert_eq!(res.0, AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidateIndex(candidate_index))); } #[test] @@ -577,7 +580,7 @@ fn rejects_approval_before_assignment() { |r| r ).unwrap(); - assert_eq!(res, ApprovalCheckResult::Bad); + assert_eq!(res, ApprovalCheckResult::Bad(ApprovalCheckError::NoAssignment(ValidatorIndex(0)))); assert!(actions.is_empty()); } @@ -609,7 +612,7 @@ fn rejects_approval_if_no_candidate_entry() { |r| r ).unwrap(); - assert_eq!(res, ApprovalCheckResult::Bad); + assert_eq!(res, ApprovalCheckResult::Bad(ApprovalCheckError::InvalidCandidate(0, candidate_hash))); assert!(actions.is_empty()); } @@ -647,7 +650,7 @@ fn rejects_approval_if_no_block_entry() { |r| r ).unwrap(); - assert_eq!(res, ApprovalCheckResult::Bad); + assert_eq!(res, ApprovalCheckResult::Bad(ApprovalCheckError::UnknownBlock(block_hash))); assert!(actions.is_empty()); } diff --git a/node/core/av-store/Cargo.toml b/node/core/av-store/Cargo.toml index 7fb72cb98a2a..1880b5b80fff 100644 --- a/node/core/av-store/Cargo.toml +++ b/node/core/av-store/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Parity Technologies "] edition = "2018" [dependencies] -futures = "0.3.12" +futures = "0.3.15" futures-timer = "3.0.2" kvdb = "0.9.0" thiserror = "1.0.23" @@ -22,7 +22,7 @@ polkadot-node-primitives = { path = "../../primitives" } [dev-dependencies] log = "0.4.13" -env_logger = "0.8.2" +env_logger = "0.8.4" assert_matches = "1.4.0" kvdb-memorydb = "0.9.0" diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index 5c6932414362..4c646f5e2d5b 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -19,7 +19,7 @@ #![recursion_limit="256"] #![warn(missing_docs)] -use std::collections::HashMap; +use std::collections::{HashMap, HashSet, BTreeSet}; use std::io; use std::sync::Arc; use std::time::{Duration, SystemTime, SystemTimeError, UNIX_EPOCH}; @@ -31,7 +31,7 @@ use kvdb::{KeyValueDB, DBTransaction}; use polkadot_primitives::v1::{ Hash, BlockNumber, CandidateEvent, ValidatorIndex, CandidateHash, - CandidateReceipt, + CandidateReceipt, Header, }; use polkadot_node_primitives::{ ErasureChunk, AvailableData, @@ -41,7 +41,10 @@ use polkadot_subsystem::{ ActiveLeavesUpdate, errors::{ChainApiError, RuntimeApiError}, }; -use polkadot_node_subsystem_util::metrics::{self, prometheus}; +use polkadot_node_subsystem_util::{ + self as util, + metrics::{self, prometheus}, +}; use polkadot_subsystem::messages::{ AvailabilityStoreMessage, ChainApiMessage, RuntimeApiMessage, RuntimeApiRequest, }; @@ -444,6 +447,8 @@ pub struct AvailabilityStoreSubsystem { pruning_config: PruningConfig, config: Config, db: Arc, + known_blocks: KnownUnfinalizedBlocks, + finalized_number: Option, metrics: Metrics, clock: Box, } @@ -478,6 +483,41 @@ impl AvailabilityStoreSubsystem { db, metrics, clock, + known_blocks: KnownUnfinalizedBlocks::default(), + finalized_number: None, + } + } +} + +/// We keep the hashes and numbers of all unfinalized +/// processed blocks in memory. +#[derive(Default, Debug)] +struct KnownUnfinalizedBlocks { + by_hash: HashSet, + by_number: BTreeSet<(BlockNumber, Hash)>, +} + +impl KnownUnfinalizedBlocks { + /// Check whether the block has been already processed. + fn is_known(&self, hash: &Hash) -> bool { + self.by_hash.contains(hash) + } + + /// Insert a new block into the known set. + fn insert(&mut self, hash: Hash, number: BlockNumber) { + self.by_hash.insert(hash); + self.by_number.insert((number, hash)); + } + + /// Prune all finalized blocks. + fn prune_finalized(&mut self, finalized: BlockNumber) { + // split_off returns everything after the given key, including the key + let split_point = finalized.saturating_add(1); + let mut finalized = self.by_number.split_off(&(split_point, Hash::zero())); + // after split_off `finalized` actually contains unfinalized blocks, we need to swap + std::mem::swap(&mut self.by_number, &mut finalized); + for (_, block) in finalized { + self.by_hash.remove(&block); } } } @@ -498,7 +538,6 @@ where } } -#[tracing::instrument(skip(subsystem, ctx), fields(subsystem = LOG_TARGET))] async fn run(mut subsystem: AvailabilityStoreSubsystem, mut ctx: Context) where Context: SubsystemContext, @@ -524,7 +563,6 @@ where } } -#[tracing::instrument(level = "trace", skip(subsystem, ctx), fields(subsystem = LOG_TARGET))] async fn run_iteration( ctx: &mut Context, subsystem: &mut AvailabilityStoreSubsystem, @@ -549,6 +587,8 @@ where FromOverseer::Signal(OverseerSignal::BlockFinalized(hash, number)) => { let _timer = subsystem.metrics.time_process_block_finalized(); + subsystem.finalized_number = Some(number); + subsystem.known_blocks.prune_finalized(number); process_block_finalized( ctx, &subsystem, @@ -576,25 +616,17 @@ where } async fn process_block_activated( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, subsystem: &mut AvailabilityStoreSubsystem, activated: Hash, ) -> Result<(), Error> { let now = subsystem.clock.now()?; - let candidate_events = { + let block_header = { let (tx, rx) = oneshot::channel(); - ctx.send_message( - RuntimeApiMessage::Request(activated, RuntimeApiRequest::CandidateEvents(tx)).into() - ).await; - rx.await?? - }; - - let block_number = { - let (tx, rx) = oneshot::channel(); ctx.send_message( - ChainApiMessage::BlockNumber(activated, tx).into() + ChainApiMessage::BlockHeader(activated, tx).into() ).await; match rx.await?? { @@ -602,41 +634,77 @@ async fn process_block_activated( Some(n) => n, } }; + let block_number = block_header.number; - let block_header = { - let (tx, rx) = oneshot::channel(); + let new_blocks = util::determine_new_blocks( + ctx.sender(), + |hash| -> Result { + Ok(subsystem.known_blocks.is_known(hash)) + }, + activated, + &block_header, + subsystem.finalized_number.unwrap_or(block_number.saturating_sub(1)), + ).await?; + + let mut tx = DBTransaction::new(); + // determine_new_blocks is descending in block height + for (hash, header) in new_blocks.into_iter().rev() { + process_new_head( + ctx, + &subsystem.db, + &mut tx, + &subsystem.config, + &subsystem.pruning_config, + now, + hash, + header, + ).await?; + subsystem.known_blocks.insert(hash, block_number); + } + subsystem.db.write(tx)?; + Ok(()) +} + +async fn process_new_head( + ctx: &mut impl SubsystemContext, + db: &Arc, + db_transaction: &mut DBTransaction, + config: &Config, + pruning_config: &PruningConfig, + now: Duration, + hash: Hash, + header: Header, +) -> Result<(), Error> { + + let candidate_events = { + let (tx, rx) = oneshot::channel(); ctx.send_message( - ChainApiMessage::BlockHeader(activated, tx).into() + RuntimeApiMessage::Request(hash, RuntimeApiRequest::CandidateEvents(tx)).into() ).await; - match rx.await?? { - None => return Ok(()), - Some(n) => n, - } + rx.await?? }; - // We need to request the number of validators based on the parent state, as that is the number of validators - // used to create this block. + // We need to request the number of validators based on the parent state, + // as that is the number of validators used to create this block. let n_validators = { let (tx, rx) = oneshot::channel(); ctx.send_message( - RuntimeApiMessage::Request(block_header.parent_hash, RuntimeApiRequest::Validators(tx)).into() + RuntimeApiMessage::Request(header.parent_hash, RuntimeApiRequest::Validators(tx)).into() ).await; rx.await??.len() }; - let mut tx = DBTransaction::new(); - for event in candidate_events { match event { CandidateEvent::CandidateBacked(receipt, _head, _core_index, _group_index) => { note_block_backed( - &subsystem.db, - &mut tx, - &subsystem.config, - &subsystem.pruning_config, + db, + db_transaction, + config, + pruning_config, now, n_validators, receipt, @@ -644,11 +712,11 @@ async fn process_block_activated( } CandidateEvent::CandidateIncluded(receipt, _head, _core_index, _group_index) => { note_block_included( - &subsystem.db, - &mut tx, - &subsystem.config, - &subsystem.pruning_config, - (block_number, activated), + db, + db_transaction, + config, + pruning_config, + (header.number, hash), receipt, )?; } @@ -656,8 +724,6 @@ async fn process_block_activated( } } - subsystem.db.write(tx)?; - Ok(()) } @@ -734,9 +800,10 @@ fn note_block_included( State::Unfinalized(at, mut within) => { if let Err(i) = within.binary_search(&be_block) { within.insert(i, be_block); + State::Unfinalized(at, within) + } else { + return Ok(()); } - - State::Unfinalized(at, within) } State::Finalized(_at) => { // This should never happen as a candidate would have to be included after @@ -769,7 +836,7 @@ macro_rules! peek_num { } async fn process_block_finalized( - ctx: &mut impl SubsystemContext, + ctx: &mut impl SubsystemContext, subsystem: &AvailabilityStoreSubsystem, finalized_hash: Hash, finalized_number: BlockNumber, diff --git a/node/core/av-store/src/tests.rs b/node/core/av-store/src/tests.rs index 278e05cad9ac..e923545a785a 100644 --- a/node/core/av-store/src/tests.rs +++ b/node/core/av-store/src/tests.rs @@ -32,7 +32,8 @@ use polkadot_primitives::v1::{ use polkadot_node_primitives::{AvailableData, BlockData, PoV}; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_subsystem::{ - ActiveLeavesUpdate, errors::RuntimeApiError, jaeger, ActivatedLeaf, LeafStatus, + ActiveLeavesUpdate, errors::RuntimeApiError, jaeger, messages::AllMessages, ActivatedLeaf, + LeafStatus, }; use polkadot_node_subsystem_test_helpers as test_helpers; use sp_keyring::Sr25519Keyring; @@ -265,6 +266,25 @@ fn runtime_api_error_does_not_stop_the_subsystem() { }), ).await; + let header = Header { + parent_hash: Hash::zero(), + number: 1, + state_root: Hash::zero(), + extrinsics_root: Hash::zero(), + digest: Default::default(), + }; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::BlockHeader( + relay_parent, + tx, + )) => { + assert_eq!(relay_parent, new_leaf); + tx.send(Ok(Some(header))).unwrap(); + } + ); + // runtime api call fails assert_matches!( overseer_recv(&mut virtual_overseer).await, @@ -764,6 +784,134 @@ fn stored_data_kept_until_finalized() { }); } +#[test] +fn we_dont_miss_anything_if_import_notifications_are_missed() { + let store = Arc::new(kvdb_memorydb::create(columns::NUM_COLUMNS)); + let test_state = TestState::default(); + + test_harness(test_state.clone(), store.clone(), |mut virtual_overseer| async move { + overseer_signal( + &mut virtual_overseer, + OverseerSignal::BlockFinalized(Hash::zero(), 1) + ).await; + + let header = Header { + parent_hash: Hash::repeat_byte(3), + number: 4, + state_root: Hash::zero(), + extrinsics_root: Hash::zero(), + digest: Default::default(), + }; + let new_leaf = Hash::repeat_byte(4); + + overseer_signal( + &mut virtual_overseer, + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: vec![ActivatedLeaf { + hash: new_leaf, + number: 4, + status: LeafStatus::Fresh, + span: Arc::new(jaeger::Span::Disabled), + }].into(), + deactivated: vec![].into(), + }), + ).await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::BlockHeader( + relay_parent, + tx, + )) => { + assert_eq!(relay_parent, new_leaf); + tx.send(Ok(Some(header))).unwrap(); + } + ); + + let new_heads = vec![ + (Hash::repeat_byte(2), Hash::repeat_byte(1)), + (Hash::repeat_byte(3), Hash::repeat_byte(2)), + (Hash::repeat_byte(4), Hash::repeat_byte(3)), + ]; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::Ancestors { + hash, + k, + response_channel: tx, + }) => { + assert_eq!(hash, new_leaf); + assert_eq!(k, 2); + let _ = tx.send(Ok(vec![ + Hash::repeat_byte(3), + Hash::repeat_byte(2), + ])); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::BlockHeader( + relay_parent, + tx, + )) => { + assert_eq!(relay_parent, Hash::repeat_byte(3)); + tx.send(Ok(Some(Header { + parent_hash: Hash::repeat_byte(2), + number: 3, + state_root: Hash::zero(), + extrinsics_root: Hash::zero(), + digest: Default::default(), + }))).unwrap(); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::BlockHeader( + relay_parent, + tx, + )) => { + assert_eq!(relay_parent, Hash::repeat_byte(2)); + tx.send(Ok(Some(Header { + parent_hash: Hash::repeat_byte(1), + number: 2, + state_root: Hash::zero(), + extrinsics_root: Hash::zero(), + digest: Default::default(), + }))).unwrap(); + } + ); + + for (head, parent) in new_heads { + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + relay_parent, + RuntimeApiRequest::CandidateEvents(tx), + )) => { + assert_eq!(relay_parent, head); + tx.send(Ok(Vec::new())).unwrap(); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + relay_parent, + RuntimeApiRequest::Validators(tx), + )) => { + assert_eq!(relay_parent, parent); + tx.send(Ok(Vec::new())).unwrap(); + } + ); + } + + virtual_overseer + }); +} + #[test] fn forkfullness_works() { let store = Arc::new(kvdb_memorydb::create(columns::NUM_COLUMNS)); @@ -1004,37 +1152,27 @@ async fn import_leaf( assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - relay_parent, - RuntimeApiRequest::CandidateEvents(tx), - )) => { - assert_eq!(relay_parent, new_leaf); - tx.send(Ok(events)).unwrap(); - } - ); - - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::ChainApi(ChainApiMessage::BlockNumber( + AllMessages::ChainApi(ChainApiMessage::BlockHeader( relay_parent, tx, )) => { assert_eq!(relay_parent, new_leaf); - tx.send(Ok(Some(block_number))).unwrap(); + tx.send(Ok(Some(header))).unwrap(); } ); assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::ChainApi(ChainApiMessage::BlockHeader( + AllMessages::RuntimeApi(RuntimeApiMessage::Request( relay_parent, - tx, + RuntimeApiRequest::CandidateEvents(tx), )) => { assert_eq!(relay_parent, new_leaf); - tx.send(Ok(Some(header))).unwrap(); + tx.send(Ok(events)).unwrap(); } ); + assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::RuntimeApi(RuntimeApiMessage::Request( diff --git a/node/core/backing/Cargo.toml b/node/core/backing/Cargo.toml index 7ea51e305029..aaac65f4955e 100644 --- a/node/core/backing/Cargo.toml +++ b/node/core/backing/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Parity Technologies "] edition = "2018" [dependencies] -futures = "0.3.12" +futures = "0.3.15" sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } polkadot-primitives = { path = "../../../primitives" } polkadot-node-primitives = { path = "../../primitives" } @@ -23,6 +23,6 @@ sp-application-crypto = { git = "https://github.com/paritytech/substrate", branc sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } -futures = { version = "0.3.12", features = ["thread-pool"] } +futures = { version = "0.3.15", features = ["thread-pool"] } assert_matches = "1.4.0" polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index f2913b157a80..5075eac48fb5 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. +// Copyright 2020-2021 Parity Technologies (UK) Ltd. // This file is part of Polkadot. // Polkadot is free software: you can redistribute it and/or modify @@ -39,7 +39,7 @@ use polkadot_subsystem::{ jaeger, messages::{ AllMessages, AvailabilityDistributionMessage, AvailabilityStoreMessage, - CandidateBackingMessage, CandidateSelectionMessage, CandidateValidationMessage, + CandidateBackingMessage, CandidateValidationMessage, CollatorProtocolMessage, ProvisionableData, ProvisionerMessage, RuntimeApiRequest, StatementDistributionMessage, ValidationFailed } @@ -67,6 +67,9 @@ use statement_table::{ }; use thiserror::Error; +#[cfg(test)] +mod tests; + const LOG_TARGET: &str = "parachain::candidate-backing"; /// Errors that can occur in candidate backing. @@ -241,7 +244,6 @@ fn primitive_statement_to_table(s: &SignedFullStatement) -> TableSignedStatement } } -#[tracing::instrument(level = "trace", skip(attested, table_context), fields(subsystem = LOG_TARGET))] fn table_attested_to_backed( attested: TableAttestedCandidate< ParaId, @@ -293,7 +295,7 @@ fn table_attested_to_backed( } async fn store_available_data( - sender: &mut JobSender>, + sender: &mut JobSender, id: Option, n_validators: u32, candidate_hash: CandidateHash, @@ -317,9 +319,8 @@ async fn store_available_data( // // This will compute the erasure root internally and compare it to the expected erasure root. // This returns `Err()` iff there is an internal error. Otherwise, it returns either `Ok(Ok(()))` or `Ok(Err(_))`. -#[tracing::instrument(level = "trace", skip(sender, pov, span), fields(subsystem = LOG_TARGET))] async fn make_pov_available( - sender: &mut JobSender>, + sender: &mut JobSender, validator_index: Option, n_validators: usize, pov: Arc, @@ -369,7 +370,7 @@ async fn make_pov_available( } async fn request_pov( - sender: &mut JobSender>, + sender: &mut JobSender, relay_parent: Hash, from_validator: ValidatorIndex, candidate_hash: CandidateHash, @@ -390,7 +391,7 @@ async fn request_pov( } async fn request_candidate_validation( - sender: &mut JobSender>, + sender: &mut JobSender, candidate: CandidateDescriptor, pov: Arc, ) -> Result { @@ -428,7 +429,7 @@ struct BackgroundValidationParams { async fn validate_and_make_available( params: BackgroundValidationParams< - impl SubsystemSender, + impl SubsystemSender, impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Sync, > ) -> Result<(), Error> { @@ -541,7 +542,7 @@ impl CandidateBackingJob { /// Run asynchronously. async fn run_loop( mut self, - mut sender: JobSender>, + mut sender: JobSender, mut rx_to: mpsc::Receiver, span: PerLeafSpan, ) -> Result<(), Error> { @@ -570,11 +571,10 @@ impl CandidateBackingJob { Ok(()) } - #[tracing::instrument(level = "trace", skip(self, root_span, sender), fields(subsystem = LOG_TARGET))] async fn handle_validated_candidate_command( &mut self, root_span: &jaeger::Span, - sender: &mut JobSender>, + sender: &mut JobSender, command: ValidatedCandidateCommand, ) -> Result<(), Error> { let candidate_hash = command.candidate_hash(); @@ -600,14 +600,14 @@ impl CandidateBackingJob { root_span, ).await? { sender.send_message( - CandidateSelectionMessage::Seconded(self.parent, stmt).into() + CollatorProtocolMessage::Seconded(self.parent, stmt).into() ).await; } } } Err(candidate) => { sender.send_message( - CandidateSelectionMessage::Invalid(self.parent, candidate).into() + CollatorProtocolMessage::Invalid(self.parent, candidate).into() ).await; } } @@ -647,12 +647,11 @@ impl CandidateBackingJob { Ok(()) } - #[tracing::instrument(level = "trace", skip(self, sender, params), fields(subsystem = LOG_TARGET))] async fn background_validate_and_make_available( &mut self, - sender: &mut JobSender>, + sender: &mut JobSender, params: BackgroundValidationParams< - impl SubsystemSender, + impl SubsystemSender, impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Send + 'static + Sync >, ) -> Result<(), Error> { @@ -671,12 +670,11 @@ impl CandidateBackingJob { } /// Kick off background validation with intent to second. - #[tracing::instrument(level = "trace", skip(self, parent_span, sender, pov), fields(subsystem = LOG_TARGET))] async fn validate_and_second( &mut self, parent_span: &jaeger::Span, root_span: &jaeger::Span, - sender: &mut JobSender>, + sender: &mut JobSender, candidate: &CandidateReceipt, pov: Arc, ) -> Result<(), Error> { @@ -685,7 +683,7 @@ impl CandidateBackingJob { .map_or(false, |c| c != &candidate.descriptor().collator) { sender.send_message( - CandidateSelectionMessage::Invalid(self.parent, candidate.clone()).into() + CollatorProtocolMessage::Invalid(self.parent, candidate.clone()).into() ).await; return Ok(()); } @@ -727,7 +725,7 @@ impl CandidateBackingJob { async fn sign_import_and_distribute_statement( &mut self, - sender: &mut JobSender>, + sender: &mut JobSender, statement: Statement, root_span: &jaeger::Span, ) -> Result, Error> { @@ -743,8 +741,7 @@ impl CandidateBackingJob { } /// Check if there have happened any new misbehaviors and issue necessary messages. - #[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))] - async fn issue_new_misbehaviors(&mut self, sender: &mut JobSender>) { + async fn issue_new_misbehaviors(&mut self, sender: &mut JobSender) { // collect the misbehaviors to avoid double mutable self borrow issues let misbehaviors: Vec<_> = self.table.drain_misbehaviors().collect(); for (validator_id, report) in misbehaviors { @@ -758,10 +755,9 @@ impl CandidateBackingJob { } /// Import a statement into the statement table and return the summary of the import. - #[tracing::instrument(level = "trace", skip(self, sender), fields(subsystem = LOG_TARGET))] async fn import_statement( &mut self, - sender: &mut JobSender>, + sender: &mut JobSender, statement: &SignedFullStatement, root_span: &jaeger::Span, ) -> Result, Error> { @@ -828,11 +824,10 @@ impl CandidateBackingJob { Ok(summary) } - #[tracing::instrument(level = "trace", skip(self, root_span, sender), fields(subsystem = LOG_TARGET))] async fn process_msg( &mut self, root_span: &jaeger::Span, - sender: &mut JobSender>, + sender: &mut JobSender, msg: CandidateBackingMessage, ) -> Result<(), Error> { match msg { @@ -895,10 +890,9 @@ impl CandidateBackingJob { } /// Kick off validation work and distribute the result as a signed statement. - #[tracing::instrument(level = "trace", skip(self, sender, attesting, span), fields(subsystem = LOG_TARGET))] async fn kick_off_validation_work( &mut self, - sender: &mut JobSender>, + sender: &mut JobSender, attesting: AttestingData, span: Option, ) -> Result<(), Error> { @@ -951,11 +945,10 @@ impl CandidateBackingJob { } /// Import the statement and kick off validation work if it is a part of our assignment. - #[tracing::instrument(level = "trace", skip(self, root_span, sender), fields(subsystem = LOG_TARGET))] async fn maybe_validate_and_import( &mut self, root_span: &jaeger::Span, - sender: &mut JobSender>, + sender: &mut JobSender, statement: SignedFullStatement, ) -> Result<(), Error> { if let Some(summary) = self.import_statement(sender, &statement, root_span).await? { @@ -1014,7 +1007,6 @@ impl CandidateBackingJob { Ok(()) } - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] async fn sign_statement(&self, statement: Statement) -> Option { let signed = self.table_context .validator @@ -1090,7 +1082,6 @@ impl util::JobTrait for CandidateBackingJob { const NAME: &'static str = "CandidateBackingJob"; - #[tracing::instrument(skip(span, keystore, metrics, rx_to, sender), fields(subsystem = LOG_TARGET))] fn run( parent: Hash, span: Arc, @@ -1324,1558 +1315,3 @@ impl metrics::Metrics for Metrics { /// The candidate backing subsystem. pub type CandidateBackingSubsystem = polkadot_node_subsystem_util::JobSubsystem; - -#[cfg(test)] -mod tests { - use super::*; - use assert_matches::assert_matches; - use futures::{future, Future}; - use polkadot_primitives::v1::{GroupRotationInfo, HeadData, PersistedValidationData, ScheduledCore}; - use polkadot_subsystem::{ - messages::{RuntimeApiRequest, RuntimeApiMessage}, - ActiveLeavesUpdate, FromOverseer, OverseerSignal, ActivatedLeaf, LeafStatus, - }; - use polkadot_node_primitives::{InvalidCandidate, BlockData}; - use polkadot_node_subsystem_test_helpers as test_helpers; - use sp_keyring::Sr25519Keyring; - use sp_application_crypto::AppKey; - use sp_keystore::{CryptoStore, SyncCryptoStore}; - use statement_table::v1::Misbehavior; - use std::collections::HashMap; - use sp_tracing as _; - - fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() - } - - fn table_statement_to_primitive( - statement: TableStatement, - ) -> Statement { - match statement { - TableStatement::Seconded(committed_candidate_receipt) => Statement::Seconded(committed_candidate_receipt), - TableStatement::Valid(candidate_hash) => Statement::Valid(candidate_hash), - } - } - - struct TestState { - chain_ids: Vec, - keystore: SyncCryptoStorePtr, - validators: Vec, - validator_public: Vec, - validation_data: PersistedValidationData, - validator_groups: (Vec>, GroupRotationInfo), - availability_cores: Vec, - head_data: HashMap, - signing_context: SigningContext, - relay_parent: Hash, - } - - impl Default for TestState { - fn default() -> Self { - let chain_a = ParaId::from(1); - let chain_b = ParaId::from(2); - let thread_a = ParaId::from(3); - - let chain_ids = vec![chain_a, chain_b, thread_a]; - - let validators = vec![ - Sr25519Keyring::Alice, - Sr25519Keyring::Bob, - Sr25519Keyring::Charlie, - Sr25519Keyring::Dave, - Sr25519Keyring::Ferdie, - Sr25519Keyring::One, - ]; - - let keystore = Arc::new(sc_keystore::LocalKeystore::in_memory()); - // Make sure `Alice` key is in the keystore, so this mocked node will be a parachain validator. - SyncCryptoStore::sr25519_generate_new(&*keystore, ValidatorId::ID, Some(&validators[0].to_seed())) - .expect("Insert key into keystore"); - - let validator_public = validator_pubkeys(&validators); - - let validator_groups = vec![vec![2, 0, 3, 5], vec![1], vec![4]] - .into_iter().map(|g| g.into_iter().map(ValidatorIndex).collect()).collect(); - let group_rotation_info = GroupRotationInfo { - session_start_block: 0, - group_rotation_frequency: 100, - now: 1, - }; - - let thread_collator: CollatorId = Sr25519Keyring::Two.public().into(); - let availability_cores = vec![ - CoreState::Scheduled(ScheduledCore { - para_id: chain_a, - collator: None, - }), - CoreState::Scheduled(ScheduledCore { - para_id: chain_b, - collator: None, - }), - CoreState::Scheduled(ScheduledCore { - para_id: thread_a, - collator: Some(thread_collator.clone()), - }), - ]; - - let mut head_data = HashMap::new(); - head_data.insert(chain_a, HeadData(vec![4, 5, 6])); - - let relay_parent = Hash::repeat_byte(5); - - let signing_context = SigningContext { - session_index: 1, - parent_hash: relay_parent, - }; - - let validation_data = PersistedValidationData { - parent_head: HeadData(vec![7, 8, 9]), - relay_parent_number: Default::default(), - max_pov_size: 1024, - relay_parent_storage_root: Default::default(), - }; - - Self { - chain_ids, - keystore, - validators, - validator_public, - validator_groups: (validator_groups, group_rotation_info), - availability_cores, - head_data, - validation_data, - signing_context, - relay_parent, - } - } - } - - type VirtualOverseer = test_helpers::TestSubsystemContextHandle; - - fn test_harness>( - keystore: SyncCryptoStorePtr, - test: impl FnOnce(VirtualOverseer) -> T, - ) { - let pool = sp_core::testing::TaskExecutor::new(); - - let (context, virtual_overseer) = - test_helpers::make_subsystem_context(pool.clone()); - - let subsystem = CandidateBackingSubsystem::new( - pool.clone(), - keystore, - Metrics(None), - ).run(context); - - let test_fut = test(virtual_overseer); - - futures::pin_mut!(test_fut); - futures::pin_mut!(subsystem); - futures::executor::block_on(future::join(async move { - let mut virtual_overseer = test_fut.await; - virtual_overseer.send(FromOverseer::Signal( - OverseerSignal::Conclude, - )).await; - }, subsystem)); - } - - fn make_erasure_root(test: &TestState, pov: PoV) -> Hash { - let available_data = AvailableData { - validation_data: test.validation_data.clone(), - pov: Arc::new(pov), - }; - - let chunks = erasure_coding::obtain_chunks_v1(test.validators.len(), &available_data).unwrap(); - erasure_coding::branches(&chunks).root() - } - - #[derive(Default)] - struct TestCandidateBuilder { - para_id: ParaId, - head_data: HeadData, - pov_hash: Hash, - relay_parent: Hash, - erasure_root: Hash, - } - - impl TestCandidateBuilder { - fn build(self) -> CommittedCandidateReceipt { - CommittedCandidateReceipt { - descriptor: CandidateDescriptor { - para_id: self.para_id, - pov_hash: self.pov_hash, - relay_parent: self.relay_parent, - erasure_root: self.erasure_root, - ..Default::default() - }, - commitments: CandidateCommitments { - head_data: self.head_data, - ..Default::default() - }, - } - } - } - - // Tests that the subsystem performs actions that are requied on startup. - async fn test_startup( - virtual_overseer: &mut VirtualOverseer, - test_state: &TestState, - ) { - // Start work on some new parent. - virtual_overseer.send(FromOverseer::Signal( - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.relay_parent, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }))) - ).await; - - // Check that subsystem job issues a request for a validator set. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(parent, RuntimeApiRequest::Validators(tx)) - ) if parent == test_state.relay_parent => { - tx.send(Ok(test_state.validator_public.clone())).unwrap(); - } - ); - - // Check that subsystem job issues a request for the validator groups. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(parent, RuntimeApiRequest::ValidatorGroups(tx)) - ) if parent == test_state.relay_parent => { - tx.send(Ok(test_state.validator_groups.clone())).unwrap(); - } - ); - - // Check that subsystem job issues a request for the session index for child. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(parent, RuntimeApiRequest::SessionIndexForChild(tx)) - ) if parent == test_state.relay_parent => { - tx.send(Ok(test_state.signing_context.session_index)).unwrap(); - } - ); - - // Check that subsystem job issues a request for the availability cores. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(parent, RuntimeApiRequest::AvailabilityCores(tx)) - ) if parent == test_state.relay_parent => { - tx.send(Ok(test_state.availability_cores.clone())).unwrap(); - } - ); - } - - // Test that a `CandidateBackingMessage::Second` issues validation work - // and in case validation is successful issues a `StatementDistributionMessage`. - #[test] - fn backing_second_works() { - let test_state = TestState::default(); - test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { - test_startup(&mut virtual_overseer, &test_state).await; - - let pov = PoV { - block_data: BlockData(vec![42, 43, 44]), - }; - - let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); - - let pov_hash = pov.hash(); - let candidate = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash, - head_data: expected_head_data.clone(), - erasure_root: make_erasure_root(&test_state, pov.clone()), - ..Default::default() - }.build(); - - let second = CandidateBackingMessage::Second( - test_state.relay_parent, - candidate.to_plain(), - pov.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: second }).await; - - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - c, - pov, - tx, - ) - ) if pov == pov && &c == candidate.descriptor() => { - tx.send(Ok( - ValidationResult::Valid(CandidateCommitments { - head_data: expected_head_data.clone(), - horizontal_messages: Vec::new(), - upward_messages: Vec::new(), - new_validation_code: None, - processed_downward_messages: 0, - hrmp_watermark: 0, - }, test_state.validation_data), - )).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::AvailabilityStore( - AvailabilityStoreMessage::StoreAvailableData(candidate_hash, _, _, _, tx) - ) if candidate_hash == candidate.hash() => { - tx.send(Ok(())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::StatementDistribution( - StatementDistributionMessage::Share( - parent_hash, - _signed_statement, - ) - ) if parent_hash == test_state.relay_parent => {} - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateSelection(CandidateSelectionMessage::Seconded(hash, statement)) => { - assert_eq!(test_state.relay_parent, hash); - assert_matches!(statement.payload(), Statement::Seconded(_)); - } - ); - - virtual_overseer.send(FromOverseer::Signal( - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) - ).await; - virtual_overseer - }); - } - - // Test that the candidate reaches quorum succesfully. - #[test] - fn backing_works() { - let test_state = TestState::default(); - test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { - test_startup(&mut virtual_overseer, &test_state).await; - - let pov = PoV { - block_data: BlockData(vec![1, 2, 3]), - }; - - let pov_hash = pov.hash(); - - let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); - - let candidate_a = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash, - head_data: expected_head_data.clone(), - erasure_root: make_erasure_root(&test_state, pov.clone()), - ..Default::default() - }.build(); - - let candidate_a_hash = candidate_a.hash(); - let public1 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, - Some(&test_state.validators[5].to_seed()), - ).await.expect("Insert key into keystore"); - let public2 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, - Some(&test_state.validators[2].to_seed()), - ).await.expect("Insert key into keystore"); - - let signed_a = SignedFullStatement::sign( - &test_state.keystore, - Statement::Seconded(candidate_a.clone()), - &test_state.signing_context, - ValidatorIndex(2), - &public2.into(), - ).await.ok().flatten().expect("should be signed"); - - let signed_b = SignedFullStatement::sign( - &test_state.keystore, - Statement::Valid(candidate_a_hash), - &test_state.signing_context, - ValidatorIndex(5), - &public1.into(), - ).await.ok().flatten().expect("should be signed"); - - let statement = CandidateBackingMessage::Statement(test_state.relay_parent, signed_a.clone()); - - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - // Sending a `Statement::Seconded` for our assignment will start - // validation process. The first thing requested is the PoV. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::AvailabilityDistribution( - AvailabilityDistributionMessage::FetchPoV { - relay_parent, - tx, - .. - } - ) if relay_parent == test_state.relay_parent => { - tx.send(pov.clone()).unwrap(); - } - ); - - // The next step is the actual request to Validation subsystem - // to validate the `Seconded` candidate. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - c, - pov, - tx, - ) - ) if pov == pov && &c == candidate_a.descriptor() => { - tx.send(Ok( - ValidationResult::Valid(CandidateCommitments { - head_data: expected_head_data.clone(), - upward_messages: Vec::new(), - horizontal_messages: Vec::new(), - new_validation_code: None, - processed_downward_messages: 0, - hrmp_watermark: 0, - }, test_state.validation_data), - )).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::AvailabilityStore( - AvailabilityStoreMessage::StoreAvailableData(candidate_hash, _, _, _, tx) - ) if candidate_hash == candidate_a.hash() => { - tx.send(Ok(())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::StatementDistribution( - StatementDistributionMessage::Share(hash, _stmt) - ) => { - assert_eq!(test_state.relay_parent, hash); - } - ); - - let statement = CandidateBackingMessage::Statement( - test_state.relay_parent, - signed_b.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::Provisioner( - ProvisionerMessage::ProvisionableData( - _, - ProvisionableData::BackedCandidate(candidate_receipt) - ) - ) => { - assert_eq!(candidate_receipt, candidate_a.to_plain()); - } - ); - - virtual_overseer.send(FromOverseer::Signal( - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) - ).await; - virtual_overseer - }); - } - - #[test] - fn backing_works_while_validation_ongoing() { - let test_state = TestState::default(); - test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { - test_startup(&mut virtual_overseer, &test_state).await; - - let pov = PoV { - block_data: BlockData(vec![1, 2, 3]), - }; - - let pov_hash = pov.hash(); - - let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); - - let candidate_a = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash, - head_data: expected_head_data.clone(), - erasure_root: make_erasure_root(&test_state, pov.clone()), - ..Default::default() - }.build(); - - let candidate_a_hash = candidate_a.hash(); - let public1 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, - Some(&test_state.validators[5].to_seed()), - ).await.expect("Insert key into keystore"); - let public2 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, - Some(&test_state.validators[2].to_seed()), - ).await.expect("Insert key into keystore"); - let public3 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, - Some(&test_state.validators[3].to_seed()), - ).await.expect("Insert key into keystore"); - - let signed_a = SignedFullStatement::sign( - &test_state.keystore, - Statement::Seconded(candidate_a.clone()), - &test_state.signing_context, - ValidatorIndex(2), - &public2.into(), - ).await.ok().flatten().expect("should be signed"); - - let signed_b = SignedFullStatement::sign( - &test_state.keystore, - Statement::Valid(candidate_a_hash), - &test_state.signing_context, - ValidatorIndex(5), - &public1.into(), - ).await.ok().flatten().expect("should be signed"); - - let signed_c = SignedFullStatement::sign( - &test_state.keystore, - Statement::Valid(candidate_a_hash), - &test_state.signing_context, - ValidatorIndex(3), - &public3.into(), - ).await.ok().flatten().expect("should be signed"); - - let statement = CandidateBackingMessage::Statement(test_state.relay_parent, signed_a.clone()); - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - // Sending a `Statement::Seconded` for our assignment will start - // validation process. The first thing requested is PoV from the - // `PoVDistribution`. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::AvailabilityDistribution( - AvailabilityDistributionMessage::FetchPoV { - relay_parent, - tx, - .. - } - ) if relay_parent == test_state.relay_parent => { - tx.send(pov.clone()).unwrap(); - } - ); - - // The next step is the actual request to Validation subsystem - // to validate the `Seconded` candidate. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - c, - pov, - tx, - ) - ) if pov == pov && &c == candidate_a.descriptor() => { - // we never validate the candidate. our local node - // shouldn't issue any statements. - std::mem::forget(tx); - } - ); - - let statement = CandidateBackingMessage::Statement( - test_state.relay_parent, - signed_b.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - let statement = CandidateBackingMessage::Statement( - test_state.relay_parent, - signed_c.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - // Candidate gets backed entirely by other votes. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::Provisioner( - ProvisionerMessage::ProvisionableData( - _, - ProvisionableData::BackedCandidate(CandidateReceipt { - descriptor, - .. - }) - ) - ) if descriptor == candidate_a.descriptor - ); - - let (tx, rx) = oneshot::channel(); - let msg = CandidateBackingMessage::GetBackedCandidates( - test_state.relay_parent, - vec![candidate_a.hash()], - tx, - ); - - virtual_overseer.send(FromOverseer::Communication{ msg }).await; - - let candidates = rx.await.unwrap(); - assert_eq!(1, candidates.len()); - assert_eq!(candidates[0].validity_votes.len(), 3); - - assert!(candidates[0].validity_votes.contains( - &ValidityAttestation::Implicit(signed_a.signature().clone()) - )); - assert!(candidates[0].validity_votes.contains( - &ValidityAttestation::Explicit(signed_b.signature().clone()) - )); - assert!(candidates[0].validity_votes.contains( - &ValidityAttestation::Explicit(signed_c.signature().clone()) - )); - assert_eq!( - candidates[0].validator_indices, - bitvec::bitvec![bitvec::order::Lsb0, u8; 1, 0, 1, 1], - ); - - virtual_overseer.send(FromOverseer::Signal( - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) - ).await; - virtual_overseer - }); - } - - // Issuing conflicting statements on the same candidate should - // be a misbehavior. - #[test] - fn backing_misbehavior_works() { - let test_state = TestState::default(); - test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { - test_startup(&mut virtual_overseer, &test_state).await; - - let pov = PoV { - block_data: BlockData(vec![1, 2, 3]), - }; - - let pov_hash = pov.hash(); - - let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); - - let candidate_a = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash, - erasure_root: make_erasure_root(&test_state, pov.clone()), - head_data: expected_head_data.clone(), - ..Default::default() - }.build(); - - let candidate_a_hash = candidate_a.hash(); - let public2 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, Some(&test_state.validators[2].to_seed()) - ).await.expect("Insert key into keystore"); - let seconded_2 = SignedFullStatement::sign( - &test_state.keystore, - Statement::Seconded(candidate_a.clone()), - &test_state.signing_context, - ValidatorIndex(2), - &public2.into(), - ).await.ok().flatten().expect("should be signed"); - - let valid_2 = SignedFullStatement::sign( - &test_state.keystore, - Statement::Valid(candidate_a_hash), - &test_state.signing_context, - ValidatorIndex(2), - &public2.into(), - ).await.ok().flatten().expect("should be signed"); - - let statement = CandidateBackingMessage::Statement(test_state.relay_parent, seconded_2.clone()); - - virtual_overseer.send(FromOverseer::Communication { msg: statement }).await; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::AvailabilityDistribution( - AvailabilityDistributionMessage::FetchPoV { - relay_parent, - tx, - .. - } - ) if relay_parent == test_state.relay_parent => { - tx.send(pov.clone()).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - c, - pov, - tx, - ) - ) if pov == pov && &c == candidate_a.descriptor() => { - tx.send(Ok( - ValidationResult::Valid(CandidateCommitments { - head_data: expected_head_data.clone(), - upward_messages: Vec::new(), - horizontal_messages: Vec::new(), - new_validation_code: None, - processed_downward_messages: 0, - hrmp_watermark: 0, - }, test_state.validation_data), - )).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::AvailabilityStore( - AvailabilityStoreMessage::StoreAvailableData(candidate_hash, _, _, _, tx) - ) if candidate_hash == candidate_a.hash() => { - tx.send(Ok(())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::StatementDistribution( - StatementDistributionMessage::Share( - relay_parent, - signed_statement, - ) - ) if relay_parent == test_state.relay_parent => { - assert_eq!(*signed_statement.payload(), Statement::Valid(candidate_a_hash)); - } - ); - - // This `Valid` statement is redundant after the `Seconded` statement already sent. - let statement = CandidateBackingMessage::Statement(test_state.relay_parent, valid_2.clone()); - - virtual_overseer.send(FromOverseer::Communication { msg: statement }).await; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::Provisioner( - ProvisionerMessage::ProvisionableData( - _, - ProvisionableData::MisbehaviorReport( - relay_parent, - validator_index, - Misbehavior::ValidityDoubleVote(vdv), - ) - ) - ) if relay_parent == test_state.relay_parent => { - let ((t1, s1), (t2, s2)) = vdv.deconstruct::(); - let t1 = table_statement_to_primitive(t1); - let t2 = table_statement_to_primitive(t2); - - SignedFullStatement::new( - t1, - validator_index, - s1, - &test_state.signing_context, - &test_state.validator_public[validator_index.0 as usize], - ).expect("signature must be valid"); - - SignedFullStatement::new( - t2, - validator_index, - s2, - &test_state.signing_context, - &test_state.validator_public[validator_index.0 as usize], - ).expect("signature must be valid"); - } - ); - virtual_overseer - }); - } - - // Test that if we are asked to second an invalid candidate we - // can still second a valid one afterwards. - #[test] - fn backing_dont_second_invalid() { - let test_state = TestState::default(); - test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { - test_startup(&mut virtual_overseer, &test_state).await; - - let pov_block_a = PoV { - block_data: BlockData(vec![42, 43, 44]), - }; - - let pov_block_b = PoV { - block_data: BlockData(vec![45, 46, 47]), - }; - - let pov_hash_a = pov_block_a.hash(); - let pov_hash_b = pov_block_b.hash(); - - let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); - - let candidate_a = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash: pov_hash_a, - erasure_root: make_erasure_root(&test_state, pov_block_a.clone()), - ..Default::default() - }.build(); - - let candidate_b = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash: pov_hash_b, - erasure_root: make_erasure_root(&test_state, pov_block_b.clone()), - head_data: expected_head_data.clone(), - ..Default::default() - }.build(); - - let second = CandidateBackingMessage::Second( - test_state.relay_parent, - candidate_a.to_plain(), - pov_block_a.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: second }).await; - - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - c, - pov, - tx, - ) - ) if pov == pov && &c == candidate_a.descriptor() => { - tx.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateSelection( - CandidateSelectionMessage::Invalid(parent_hash, c) - ) if parent_hash == test_state.relay_parent && c == candidate_a.to_plain() - ); - - let second = CandidateBackingMessage::Second( - test_state.relay_parent, - candidate_b.to_plain(), - pov_block_b.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: second }).await; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - c, - pov, - tx, - ) - ) if pov == pov && &c == candidate_b.descriptor() => { - tx.send(Ok( - ValidationResult::Valid(CandidateCommitments { - head_data: expected_head_data.clone(), - upward_messages: Vec::new(), - horizontal_messages: Vec::new(), - new_validation_code: None, - processed_downward_messages: 0, - hrmp_watermark: 0, - }, test_state.validation_data), - )).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::AvailabilityStore( - AvailabilityStoreMessage::StoreAvailableData(candidate_hash, _, _, _, tx) - ) if candidate_hash == candidate_b.hash() => { - tx.send(Ok(())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::StatementDistribution( - StatementDistributionMessage::Share( - parent_hash, - signed_statement, - ) - ) if parent_hash == test_state.relay_parent => { - assert_eq!(*signed_statement.payload(), Statement::Seconded(candidate_b)); - } - ); - - virtual_overseer.send(FromOverseer::Signal( - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) - ).await; - virtual_overseer - }); - } - - // Test that if we have already issued a statement (in this case `Invalid`) about a - // candidate we will not be issuing a `Seconded` statement on it. - #[test] - fn backing_second_after_first_fails_works() { - let test_state = TestState::default(); - test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { - test_startup(&mut virtual_overseer, &test_state).await; - - let pov = PoV { - block_data: BlockData(vec![42, 43, 44]), - }; - - let pov_hash = pov.hash(); - - let candidate = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash, - erasure_root: make_erasure_root(&test_state, pov.clone()), - ..Default::default() - }.build(); - - let validator2 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, Some(&test_state.validators[2].to_seed()) - ).await.expect("Insert key into keystore"); - - let signed_a = SignedFullStatement::sign( - &test_state.keystore, - Statement::Seconded(candidate.clone()), - &test_state.signing_context, - ValidatorIndex(2), - &validator2.into(), - ).await.ok().flatten().expect("should be signed"); - - // Send in a `Statement` with a candidate. - let statement = CandidateBackingMessage::Statement( - test_state.relay_parent, - signed_a.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - // Subsystem requests PoV and requests validation. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::AvailabilityDistribution( - AvailabilityDistributionMessage::FetchPoV { - relay_parent, - tx, - .. - } - ) if relay_parent == test_state.relay_parent => { - tx.send(pov.clone()).unwrap(); - } - ); - - - // Tell subsystem that this candidate is invalid. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - c, - pov, - tx, - ) - ) if pov == pov && &c == candidate.descriptor() => { - tx.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); - } - ); - - // Ask subsystem to `Second` a candidate that already has a statement issued about. - // This should emit no actions from subsystem. - let second = CandidateBackingMessage::Second( - test_state.relay_parent, - candidate.to_plain(), - pov.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: second }).await; - - let pov_to_second = PoV { - block_data: BlockData(vec![3, 2, 1]), - }; - - let pov_hash = pov_to_second.hash(); - - let candidate_to_second = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash, - erasure_root: make_erasure_root(&test_state, pov_to_second.clone()), - ..Default::default() - }.build(); - - let second = CandidateBackingMessage::Second( - test_state.relay_parent, - candidate_to_second.to_plain(), - pov_to_second.clone(), - ); - - // In order to trigger _some_ actions from subsystem ask it to second another - // candidate. The only reason to do so is to make sure that no actions were - // triggered on the prev step. - virtual_overseer.send(FromOverseer::Communication{ msg: second }).await; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - _, - pov, - _, - ) - ) => { - assert_eq!(&*pov, &pov_to_second); - } - ); - virtual_overseer - }); - } - - // That that if the validation of the candidate has failed this does not stop - // the work of this subsystem and so it is not fatal to the node. - #[test] - fn backing_works_after_failed_validation() { - let test_state = TestState::default(); - test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { - test_startup(&mut virtual_overseer, &test_state).await; - - let pov = PoV { - block_data: BlockData(vec![42, 43, 44]), - }; - - let pov_hash = pov.hash(); - - let candidate = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash, - erasure_root: make_erasure_root(&test_state, pov.clone()), - ..Default::default() - }.build(); - - let public2 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, Some(&test_state.validators[2].to_seed()) - ).await.expect("Insert key into keystore"); - let signed_a = SignedFullStatement::sign( - &test_state.keystore, - Statement::Seconded(candidate.clone()), - &test_state.signing_context, - ValidatorIndex(2), - &public2.into(), - ).await.ok().flatten().expect("should be signed"); - - // Send in a `Statement` with a candidate. - let statement = CandidateBackingMessage::Statement( - test_state.relay_parent, - signed_a.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - // Subsystem requests PoV and requests validation. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::AvailabilityDistribution( - AvailabilityDistributionMessage::FetchPoV { - relay_parent, - tx, - .. - } - ) if relay_parent == test_state.relay_parent => { - tx.send(pov.clone()).unwrap(); - } - ); - - // Tell subsystem that this candidate is invalid. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - c, - pov, - tx, - ) - ) if pov == pov && &c == candidate.descriptor() => { - tx.send(Err(ValidationFailed("Internal test error".into()))).unwrap(); - } - ); - - // Try to get a set of backable candidates to trigger _some_ action in the subsystem - // and check that it is still alive. - let (tx, rx) = oneshot::channel(); - let msg = CandidateBackingMessage::GetBackedCandidates( - test_state.relay_parent, - vec![candidate.hash()], - tx, - ); - - virtual_overseer.send(FromOverseer::Communication{ msg }).await; - assert_eq!(rx.await.unwrap().len(), 0); - virtual_overseer - }); - } - - // Test that a `CandidateBackingMessage::Second` issues validation work - // and in case validation is successful issues a `StatementDistributionMessage`. - #[test] - fn backing_doesnt_second_wrong_collator() { - let mut test_state = TestState::default(); - test_state.availability_cores[0] = CoreState::Scheduled(ScheduledCore { - para_id: ParaId::from(1), - collator: Some(Sr25519Keyring::Bob.public().into()), - }); - - test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { - test_startup(&mut virtual_overseer, &test_state).await; - - let pov = PoV { - block_data: BlockData(vec![42, 43, 44]), - }; - - let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); - - let pov_hash = pov.hash(); - let candidate = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash, - head_data: expected_head_data.clone(), - erasure_root: make_erasure_root(&test_state, pov.clone()), - ..Default::default() - }.build(); - - let second = CandidateBackingMessage::Second( - test_state.relay_parent, - candidate.to_plain(), - pov.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: second }).await; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateSelection( - CandidateSelectionMessage::Invalid(parent, c) - ) if parent == test_state.relay_parent && c == candidate.to_plain() => { - } - ); - - virtual_overseer.send(FromOverseer::Signal( - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) - ).await; - virtual_overseer - }); - } - - #[test] - fn validation_work_ignores_wrong_collator() { - let mut test_state = TestState::default(); - test_state.availability_cores[0] = CoreState::Scheduled(ScheduledCore { - para_id: ParaId::from(1), - collator: Some(Sr25519Keyring::Bob.public().into()), - }); - - test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { - test_startup(&mut virtual_overseer, &test_state).await; - - let pov = PoV { - block_data: BlockData(vec![1, 2, 3]), - }; - - let pov_hash = pov.hash(); - - let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); - - let candidate_a = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash, - head_data: expected_head_data.clone(), - erasure_root: make_erasure_root(&test_state, pov.clone()), - ..Default::default() - }.build(); - - let public2 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, Some(&test_state.validators[2].to_seed()) - ).await.expect("Insert key into keystore"); - let seconding = SignedFullStatement::sign( - &test_state.keystore, - Statement::Seconded(candidate_a.clone()), - &test_state.signing_context, - ValidatorIndex(2), - &public2.into(), - ).await.ok().flatten().expect("should be signed"); - - let statement = CandidateBackingMessage::Statement( - test_state.relay_parent, - seconding.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - // The statement will be ignored because it has the wrong collator. - virtual_overseer.send(FromOverseer::Signal( - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) - ).await; - virtual_overseer - }); - } - - #[test] - fn candidate_backing_reorders_votes() { - use sp_core::Encode; - use std::convert::TryFrom; - - let para_id = ParaId::from(10); - let validators = vec![ - Sr25519Keyring::Alice, - Sr25519Keyring::Bob, - Sr25519Keyring::Charlie, - Sr25519Keyring::Dave, - Sr25519Keyring::Ferdie, - Sr25519Keyring::One, - ]; - - let validator_public = validator_pubkeys(&validators); - let validator_groups = { - let mut validator_groups = HashMap::new(); - validator_groups.insert(para_id, vec![0, 1, 2, 3, 4, 5].into_iter().map(ValidatorIndex).collect()); - validator_groups - }; - - let table_context = TableContext { - validator: None, - groups: validator_groups, - validators: validator_public.clone(), - }; - - let fake_attestation = |idx: u32| { - let candidate: CommittedCandidateReceipt = Default::default(); - let hash = candidate.hash(); - let mut data = vec![0; 64]; - data[0..32].copy_from_slice(hash.0.as_bytes()); - data[32..36].copy_from_slice(idx.encode().as_slice()); - - let sig = ValidatorSignature::try_from(data).unwrap(); - statement_table::generic::ValidityAttestation::Implicit(sig) - }; - - let attested = TableAttestedCandidate { - candidate: Default::default(), - validity_votes: vec![ - (ValidatorIndex(5), fake_attestation(5)), - (ValidatorIndex(3), fake_attestation(3)), - (ValidatorIndex(1), fake_attestation(1)), - ], - group_id: para_id, - }; - - let backed = table_attested_to_backed(attested, &table_context).unwrap(); - - let expected_bitvec = { - let mut validator_indices = BitVec::::with_capacity(6); - validator_indices.resize(6, false); - - validator_indices.set(1, true); - validator_indices.set(3, true); - validator_indices.set(5, true); - - validator_indices - }; - - // Should be in bitfield order, which is opposite to the order provided to the function. - let expected_attestations = vec![ - fake_attestation(1).into(), - fake_attestation(3).into(), - fake_attestation(5).into(), - ]; - - assert_eq!(backed.validator_indices, expected_bitvec); - assert_eq!(backed.validity_votes, expected_attestations); - } - - // Test whether we retry on failed PoV fetching. - #[test] - fn retry_works() { - // sp_tracing::try_init_simple(); - let test_state = TestState::default(); - test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { - test_startup(&mut virtual_overseer, &test_state).await; - - let pov = PoV { - block_data: BlockData(vec![42, 43, 44]), - }; - - let pov_hash = pov.hash(); - - let candidate = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash, - erasure_root: make_erasure_root(&test_state, pov.clone()), - ..Default::default() - }.build(); - - let public2 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, Some(&test_state.validators[2].to_seed()) - ).await.expect("Insert key into keystore"); - let public3 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, - Some(&test_state.validators[3].to_seed()), - ).await.expect("Insert key into keystore"); - let public5 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, - Some(&test_state.validators[5].to_seed()), - ).await.expect("Insert key into keystore"); - let signed_a = SignedFullStatement::sign( - &test_state.keystore, - Statement::Seconded(candidate.clone()), - &test_state.signing_context, - ValidatorIndex(2), - &public2.into(), - ).await.ok().flatten().expect("should be signed"); - let signed_b = SignedFullStatement::sign( - &test_state.keystore, - Statement::Valid(candidate.hash()), - &test_state.signing_context, - ValidatorIndex(3), - &public3.into(), - ).await.ok().flatten().expect("should be signed"); - let signed_c = SignedFullStatement::sign( - &test_state.keystore, - Statement::Valid(candidate.hash()), - &test_state.signing_context, - ValidatorIndex(5), - &public5.into(), - ).await.ok().flatten().expect("should be signed"); - - // Send in a `Statement` with a candidate. - let statement = CandidateBackingMessage::Statement( - test_state.relay_parent, - signed_a.clone(), - ); - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - // Subsystem requests PoV and requests validation. - // We cancel - should mean retry on next backing statement. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::AvailabilityDistribution( - AvailabilityDistributionMessage::FetchPoV { - relay_parent, - tx, - .. - } - ) if relay_parent == test_state.relay_parent => { - std::mem::drop(tx); - } - ); - - let statement = CandidateBackingMessage::Statement( - test_state.relay_parent, - signed_b.clone(), - ); - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - let statement = CandidateBackingMessage::Statement( - test_state.relay_parent, - signed_c.clone(), - ); - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - // Not deterministic which message comes first: - for _ in 0u32..2 { - match virtual_overseer.recv().await { - AllMessages::Provisioner( - ProvisionerMessage::ProvisionableData( - _, - ProvisionableData::BackedCandidate(CandidateReceipt { - descriptor, - .. - }) - ) - ) => { - assert_eq!(descriptor, candidate.descriptor); - } - // Subsystem requests PoV and requests validation. - // We cancel once more: - AllMessages::AvailabilityDistribution( - AvailabilityDistributionMessage::FetchPoV { - relay_parent, - tx, - .. - } - ) if relay_parent == test_state.relay_parent => { - std::mem::drop(tx); - } - msg => { - assert!(false, "Unexpected message: {:?}", msg); - } - } - } - - // Subsystem requests PoV and requests validation. - // Now we pass. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::AvailabilityDistribution( - AvailabilityDistributionMessage::FetchPoV { - relay_parent, - tx, - .. - } - ) if relay_parent == test_state.relay_parent => { - tx.send(pov.clone()).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - c, - pov, - _tx, - ) - ) if pov == pov && &c == candidate.descriptor() - ); - virtual_overseer - }); - } - - #[test] - fn observes_backing_even_if_not_validator() { - let test_state = TestState::default(); - let empty_keystore = Arc::new(sc_keystore::LocalKeystore::in_memory()); - test_harness(empty_keystore, |mut virtual_overseer| async move { - test_startup(&mut virtual_overseer, &test_state).await; - - let pov = PoV { - block_data: BlockData(vec![1, 2, 3]), - }; - - let pov_hash = pov.hash(); - - let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); - - let candidate_a = TestCandidateBuilder { - para_id: test_state.chain_ids[0], - relay_parent: test_state.relay_parent, - pov_hash, - head_data: expected_head_data.clone(), - erasure_root: make_erasure_root(&test_state, pov.clone()), - ..Default::default() - }.build(); - - let candidate_a_hash = candidate_a.hash(); - let public0 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, - Some(&test_state.validators[0].to_seed()), - ).await.expect("Insert key into keystore"); - let public1 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, - Some(&test_state.validators[5].to_seed()), - ).await.expect("Insert key into keystore"); - let public2 = CryptoStore::sr25519_generate_new( - &*test_state.keystore, - ValidatorId::ID, - Some(&test_state.validators[2].to_seed()), - ).await.expect("Insert key into keystore"); - - // Produce a 3-of-5 quorum on the candidate. - - let signed_a = SignedFullStatement::sign( - &test_state.keystore, - Statement::Seconded(candidate_a.clone()), - &test_state.signing_context, - ValidatorIndex(0), - &public0.into(), - ).await.ok().flatten().expect("should be signed"); - - let signed_b = SignedFullStatement::sign( - &test_state.keystore, - Statement::Valid(candidate_a_hash), - &test_state.signing_context, - ValidatorIndex(5), - &public1.into(), - ).await.ok().flatten().expect("should be signed"); - - let signed_c = SignedFullStatement::sign( - &test_state.keystore, - Statement::Valid(candidate_a_hash), - &test_state.signing_context, - ValidatorIndex(2), - &public2.into(), - ).await.ok().flatten().expect("should be signed"); - - let statement = CandidateBackingMessage::Statement(test_state.relay_parent, signed_a.clone()); - - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - let statement = CandidateBackingMessage::Statement( - test_state.relay_parent, - signed_b.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - let statement = CandidateBackingMessage::Statement( - test_state.relay_parent, - signed_c.clone(), - ); - - virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::Provisioner( - ProvisionerMessage::ProvisionableData( - _, - ProvisionableData::BackedCandidate(candidate_receipt) - ) - ) => { - assert_eq!(candidate_receipt, candidate_a.to_plain()); - } - ); - - virtual_overseer.send(FromOverseer::Signal( - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) - ).await; - virtual_overseer - }); - } -} diff --git a/node/core/backing/src/tests.rs b/node/core/backing/src/tests.rs new file mode 100644 index 000000000000..8aa4a3a9f3cc --- /dev/null +++ b/node/core/backing/src/tests.rs @@ -0,0 +1,1567 @@ +// Copyright 2020-2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; +use assert_matches::assert_matches; +use futures::{future, Future}; +use polkadot_primitives::v1::{GroupRotationInfo, HeadData, PersistedValidationData, ScheduledCore}; +use polkadot_subsystem::{ + messages::{RuntimeApiRequest, RuntimeApiMessage, CollatorProtocolMessage}, + ActiveLeavesUpdate, FromOverseer, OverseerSignal, ActivatedLeaf, LeafStatus, +}; +use polkadot_node_primitives::{InvalidCandidate, BlockData}; +use polkadot_node_subsystem_test_helpers as test_helpers; +use sp_keyring::Sr25519Keyring; +use sp_application_crypto::AppKey; +use sp_keystore::{CryptoStore, SyncCryptoStore}; +use statement_table::v1::Misbehavior; +use std::collections::HashMap; +use sp_tracing as _; + +fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { + val_ids.iter().map(|v| v.public().into()).collect() +} + +fn table_statement_to_primitive( + statement: TableStatement, +) -> Statement { + match statement { + TableStatement::Seconded(committed_candidate_receipt) => Statement::Seconded(committed_candidate_receipt), + TableStatement::Valid(candidate_hash) => Statement::Valid(candidate_hash), + } +} + +struct TestState { + chain_ids: Vec, + keystore: SyncCryptoStorePtr, + validators: Vec, + validator_public: Vec, + validation_data: PersistedValidationData, + validator_groups: (Vec>, GroupRotationInfo), + availability_cores: Vec, + head_data: HashMap, + signing_context: SigningContext, + relay_parent: Hash, +} + +impl Default for TestState { + fn default() -> Self { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + let chain_ids = vec![chain_a, chain_b, thread_a]; + + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + Sr25519Keyring::One, + ]; + + let keystore = Arc::new(sc_keystore::LocalKeystore::in_memory()); + // Make sure `Alice` key is in the keystore, so this mocked node will be a parachain validator. + SyncCryptoStore::sr25519_generate_new(&*keystore, ValidatorId::ID, Some(&validators[0].to_seed())) + .expect("Insert key into keystore"); + + let validator_public = validator_pubkeys(&validators); + + let validator_groups = vec![vec![2, 0, 3, 5], vec![1], vec![4]] + .into_iter().map(|g| g.into_iter().map(ValidatorIndex).collect()).collect(); + let group_rotation_info = GroupRotationInfo { + session_start_block: 0, + group_rotation_frequency: 100, + now: 1, + }; + + let thread_collator: CollatorId = Sr25519Keyring::Two.public().into(); + let availability_cores = vec![ + CoreState::Scheduled(ScheduledCore { + para_id: chain_a, + collator: None, + }), + CoreState::Scheduled(ScheduledCore { + para_id: chain_b, + collator: None, + }), + CoreState::Scheduled(ScheduledCore { + para_id: thread_a, + collator: Some(thread_collator.clone()), + }), + ]; + + let mut head_data = HashMap::new(); + head_data.insert(chain_a, HeadData(vec![4, 5, 6])); + + let relay_parent = Hash::repeat_byte(5); + + let signing_context = SigningContext { + session_index: 1, + parent_hash: relay_parent, + }; + + let validation_data = PersistedValidationData { + parent_head: HeadData(vec![7, 8, 9]), + relay_parent_number: Default::default(), + max_pov_size: 1024, + relay_parent_storage_root: Default::default(), + }; + + Self { + chain_ids, + keystore, + validators, + validator_public, + validator_groups: (validator_groups, group_rotation_info), + availability_cores, + head_data, + validation_data, + signing_context, + relay_parent, + } + } +} + +type VirtualOverseer = test_helpers::TestSubsystemContextHandle; + +fn test_harness>( + keystore: SyncCryptoStorePtr, + test: impl FnOnce(VirtualOverseer) -> T, +) { + let pool = sp_core::testing::TaskExecutor::new(); + + let (context, virtual_overseer) = + test_helpers::make_subsystem_context(pool.clone()); + + let subsystem = CandidateBackingSubsystem::new( + pool.clone(), + keystore, + Metrics(None), + ).run(context); + + let test_fut = test(virtual_overseer); + + futures::pin_mut!(test_fut); + futures::pin_mut!(subsystem); + futures::executor::block_on(future::join(async move { + let mut virtual_overseer = test_fut.await; + virtual_overseer.send(FromOverseer::Signal( + OverseerSignal::Conclude, + )).await; + }, subsystem)); +} + +fn make_erasure_root(test: &TestState, pov: PoV) -> Hash { + let available_data = AvailableData { + validation_data: test.validation_data.clone(), + pov: Arc::new(pov), + }; + + let chunks = erasure_coding::obtain_chunks_v1(test.validators.len(), &available_data).unwrap(); + erasure_coding::branches(&chunks).root() +} + +#[derive(Default)] +struct TestCandidateBuilder { + para_id: ParaId, + head_data: HeadData, + pov_hash: Hash, + relay_parent: Hash, + erasure_root: Hash, +} + +impl TestCandidateBuilder { + fn build(self) -> CommittedCandidateReceipt { + CommittedCandidateReceipt { + descriptor: CandidateDescriptor { + para_id: self.para_id, + pov_hash: self.pov_hash, + relay_parent: self.relay_parent, + erasure_root: self.erasure_root, + ..Default::default() + }, + commitments: CandidateCommitments { + head_data: self.head_data, + ..Default::default() + }, + } + } +} + +// Tests that the subsystem performs actions that are requied on startup. +async fn test_startup( + virtual_overseer: &mut VirtualOverseer, + test_state: &TestState, +) { + // Start work on some new parent. + virtual_overseer.send(FromOverseer::Signal( + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: test_state.relay_parent, + number: 1, + status: LeafStatus::Fresh, + span: Arc::new(jaeger::Span::Disabled), + }))) + ).await; + + // Check that subsystem job issues a request for a validator set. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::Validators(tx)) + ) if parent == test_state.relay_parent => { + tx.send(Ok(test_state.validator_public.clone())).unwrap(); + } + ); + + // Check that subsystem job issues a request for the validator groups. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::ValidatorGroups(tx)) + ) if parent == test_state.relay_parent => { + tx.send(Ok(test_state.validator_groups.clone())).unwrap(); + } + ); + + // Check that subsystem job issues a request for the session index for child. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::SessionIndexForChild(tx)) + ) if parent == test_state.relay_parent => { + tx.send(Ok(test_state.signing_context.session_index)).unwrap(); + } + ); + + // Check that subsystem job issues a request for the availability cores. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(parent, RuntimeApiRequest::AvailabilityCores(tx)) + ) if parent == test_state.relay_parent => { + tx.send(Ok(test_state.availability_cores.clone())).unwrap(); + } + ); +} + +// Test that a `CandidateBackingMessage::Second` issues validation work +// and in case validation is successful issues a `StatementDistributionMessage`. +#[test] +fn backing_second_works() { + let test_state = TestState::default(); + test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { + test_startup(&mut virtual_overseer, &test_state).await; + + let pov = PoV { + block_data: BlockData(vec![42, 43, 44]), + }; + + let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); + + let pov_hash = pov.hash(); + let candidate = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash, + head_data: expected_head_data.clone(), + erasure_root: make_erasure_root(&test_state, pov.clone()), + ..Default::default() + }.build(); + + let second = CandidateBackingMessage::Second( + test_state.relay_parent, + candidate.to_plain(), + pov.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: second }).await; + + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromChainState( + c, + pov, + tx, + ) + ) if pov == pov && &c == candidate.descriptor() => { + tx.send(Ok( + ValidationResult::Valid(CandidateCommitments { + head_data: expected_head_data.clone(), + horizontal_messages: Vec::new(), + upward_messages: Vec::new(), + new_validation_code: None, + processed_downward_messages: 0, + hrmp_watermark: 0, + }, test_state.validation_data), + )).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityStore( + AvailabilityStoreMessage::StoreAvailableData(candidate_hash, _, _, _, tx) + ) if candidate_hash == candidate.hash() => { + tx.send(Ok(())).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::Share( + parent_hash, + _signed_statement, + ) + ) if parent_hash == test_state.relay_parent => {} + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CollatorProtocol(CollatorProtocolMessage::Seconded(hash, statement)) => { + assert_eq!(test_state.relay_parent, hash); + assert_matches!(statement.payload(), Statement::Seconded(_)); + } + ); + + virtual_overseer.send(FromOverseer::Signal( + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) + ).await; + virtual_overseer + }); +} + +// Test that the candidate reaches quorum succesfully. +#[test] +fn backing_works() { + let test_state = TestState::default(); + test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { + test_startup(&mut virtual_overseer, &test_state).await; + + let pov = PoV { + block_data: BlockData(vec![1, 2, 3]), + }; + + let pov_hash = pov.hash(); + + let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); + + let candidate_a = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash, + head_data: expected_head_data.clone(), + erasure_root: make_erasure_root(&test_state, pov.clone()), + ..Default::default() + }.build(); + + let candidate_a_hash = candidate_a.hash(); + let public1 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, + Some(&test_state.validators[5].to_seed()), + ).await.expect("Insert key into keystore"); + let public2 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, + Some(&test_state.validators[2].to_seed()), + ).await.expect("Insert key into keystore"); + + let signed_a = SignedFullStatement::sign( + &test_state.keystore, + Statement::Seconded(candidate_a.clone()), + &test_state.signing_context, + ValidatorIndex(2), + &public2.into(), + ).await.ok().flatten().expect("should be signed"); + + let signed_b = SignedFullStatement::sign( + &test_state.keystore, + Statement::Valid(candidate_a_hash), + &test_state.signing_context, + ValidatorIndex(5), + &public1.into(), + ).await.ok().flatten().expect("should be signed"); + + let statement = CandidateBackingMessage::Statement(test_state.relay_parent, signed_a.clone()); + + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + // Sending a `Statement::Seconded` for our assignment will start + // validation process. The first thing requested is the PoV. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityDistribution( + AvailabilityDistributionMessage::FetchPoV { + relay_parent, + tx, + .. + } + ) if relay_parent == test_state.relay_parent => { + tx.send(pov.clone()).unwrap(); + } + ); + + // The next step is the actual request to Validation subsystem + // to validate the `Seconded` candidate. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromChainState( + c, + pov, + tx, + ) + ) if pov == pov && &c == candidate_a.descriptor() => { + tx.send(Ok( + ValidationResult::Valid(CandidateCommitments { + head_data: expected_head_data.clone(), + upward_messages: Vec::new(), + horizontal_messages: Vec::new(), + new_validation_code: None, + processed_downward_messages: 0, + hrmp_watermark: 0, + }, test_state.validation_data), + )).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityStore( + AvailabilityStoreMessage::StoreAvailableData(candidate_hash, _, _, _, tx) + ) if candidate_hash == candidate_a.hash() => { + tx.send(Ok(())).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::Share(hash, _stmt) + ) => { + assert_eq!(test_state.relay_parent, hash); + } + ); + + let statement = CandidateBackingMessage::Statement( + test_state.relay_parent, + signed_b.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::Provisioner( + ProvisionerMessage::ProvisionableData( + _, + ProvisionableData::BackedCandidate(candidate_receipt) + ) + ) => { + assert_eq!(candidate_receipt, candidate_a.to_plain()); + } + ); + + virtual_overseer.send(FromOverseer::Signal( + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) + ).await; + virtual_overseer + }); +} + +#[test] +fn backing_works_while_validation_ongoing() { + let test_state = TestState::default(); + test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { + test_startup(&mut virtual_overseer, &test_state).await; + + let pov = PoV { + block_data: BlockData(vec![1, 2, 3]), + }; + + let pov_hash = pov.hash(); + + let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); + + let candidate_a = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash, + head_data: expected_head_data.clone(), + erasure_root: make_erasure_root(&test_state, pov.clone()), + ..Default::default() + }.build(); + + let candidate_a_hash = candidate_a.hash(); + let public1 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, + Some(&test_state.validators[5].to_seed()), + ).await.expect("Insert key into keystore"); + let public2 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, + Some(&test_state.validators[2].to_seed()), + ).await.expect("Insert key into keystore"); + let public3 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, + Some(&test_state.validators[3].to_seed()), + ).await.expect("Insert key into keystore"); + + let signed_a = SignedFullStatement::sign( + &test_state.keystore, + Statement::Seconded(candidate_a.clone()), + &test_state.signing_context, + ValidatorIndex(2), + &public2.into(), + ).await.ok().flatten().expect("should be signed"); + + let signed_b = SignedFullStatement::sign( + &test_state.keystore, + Statement::Valid(candidate_a_hash), + &test_state.signing_context, + ValidatorIndex(5), + &public1.into(), + ).await.ok().flatten().expect("should be signed"); + + let signed_c = SignedFullStatement::sign( + &test_state.keystore, + Statement::Valid(candidate_a_hash), + &test_state.signing_context, + ValidatorIndex(3), + &public3.into(), + ).await.ok().flatten().expect("should be signed"); + + let statement = CandidateBackingMessage::Statement(test_state.relay_parent, signed_a.clone()); + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + // Sending a `Statement::Seconded` for our assignment will start + // validation process. The first thing requested is PoV from the + // `PoVDistribution`. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityDistribution( + AvailabilityDistributionMessage::FetchPoV { + relay_parent, + tx, + .. + } + ) if relay_parent == test_state.relay_parent => { + tx.send(pov.clone()).unwrap(); + } + ); + + // The next step is the actual request to Validation subsystem + // to validate the `Seconded` candidate. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromChainState( + c, + pov, + tx, + ) + ) if pov == pov && &c == candidate_a.descriptor() => { + // we never validate the candidate. our local node + // shouldn't issue any statements. + std::mem::forget(tx); + } + ); + + let statement = CandidateBackingMessage::Statement( + test_state.relay_parent, + signed_b.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + let statement = CandidateBackingMessage::Statement( + test_state.relay_parent, + signed_c.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + // Candidate gets backed entirely by other votes. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::Provisioner( + ProvisionerMessage::ProvisionableData( + _, + ProvisionableData::BackedCandidate(CandidateReceipt { + descriptor, + .. + }) + ) + ) if descriptor == candidate_a.descriptor + ); + + let (tx, rx) = oneshot::channel(); + let msg = CandidateBackingMessage::GetBackedCandidates( + test_state.relay_parent, + vec![candidate_a.hash()], + tx, + ); + + virtual_overseer.send(FromOverseer::Communication{ msg }).await; + + let candidates = rx.await.unwrap(); + assert_eq!(1, candidates.len()); + assert_eq!(candidates[0].validity_votes.len(), 3); + + assert!(candidates[0].validity_votes.contains( + &ValidityAttestation::Implicit(signed_a.signature().clone()) + )); + assert!(candidates[0].validity_votes.contains( + &ValidityAttestation::Explicit(signed_b.signature().clone()) + )); + assert!(candidates[0].validity_votes.contains( + &ValidityAttestation::Explicit(signed_c.signature().clone()) + )); + assert_eq!( + candidates[0].validator_indices, + bitvec::bitvec![bitvec::order::Lsb0, u8; 1, 0, 1, 1], + ); + + virtual_overseer.send(FromOverseer::Signal( + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) + ).await; + virtual_overseer + }); +} + +// Issuing conflicting statements on the same candidate should +// be a misbehavior. +#[test] +fn backing_misbehavior_works() { + let test_state = TestState::default(); + test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { + test_startup(&mut virtual_overseer, &test_state).await; + + let pov = PoV { + block_data: BlockData(vec![1, 2, 3]), + }; + + let pov_hash = pov.hash(); + + let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); + + let candidate_a = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash, + erasure_root: make_erasure_root(&test_state, pov.clone()), + head_data: expected_head_data.clone(), + ..Default::default() + }.build(); + + let candidate_a_hash = candidate_a.hash(); + let public2 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, Some(&test_state.validators[2].to_seed()) + ).await.expect("Insert key into keystore"); + let seconded_2 = SignedFullStatement::sign( + &test_state.keystore, + Statement::Seconded(candidate_a.clone()), + &test_state.signing_context, + ValidatorIndex(2), + &public2.into(), + ).await.ok().flatten().expect("should be signed"); + + let valid_2 = SignedFullStatement::sign( + &test_state.keystore, + Statement::Valid(candidate_a_hash), + &test_state.signing_context, + ValidatorIndex(2), + &public2.into(), + ).await.ok().flatten().expect("should be signed"); + + let statement = CandidateBackingMessage::Statement(test_state.relay_parent, seconded_2.clone()); + + virtual_overseer.send(FromOverseer::Communication { msg: statement }).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityDistribution( + AvailabilityDistributionMessage::FetchPoV { + relay_parent, + tx, + .. + } + ) if relay_parent == test_state.relay_parent => { + tx.send(pov.clone()).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromChainState( + c, + pov, + tx, + ) + ) if pov == pov && &c == candidate_a.descriptor() => { + tx.send(Ok( + ValidationResult::Valid(CandidateCommitments { + head_data: expected_head_data.clone(), + upward_messages: Vec::new(), + horizontal_messages: Vec::new(), + new_validation_code: None, + processed_downward_messages: 0, + hrmp_watermark: 0, + }, test_state.validation_data), + )).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityStore( + AvailabilityStoreMessage::StoreAvailableData(candidate_hash, _, _, _, tx) + ) if candidate_hash == candidate_a.hash() => { + tx.send(Ok(())).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::Share( + relay_parent, + signed_statement, + ) + ) if relay_parent == test_state.relay_parent => { + assert_eq!(*signed_statement.payload(), Statement::Valid(candidate_a_hash)); + } + ); + + // This `Valid` statement is redundant after the `Seconded` statement already sent. + let statement = CandidateBackingMessage::Statement(test_state.relay_parent, valid_2.clone()); + + virtual_overseer.send(FromOverseer::Communication { msg: statement }).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::Provisioner( + ProvisionerMessage::ProvisionableData( + _, + ProvisionableData::MisbehaviorReport( + relay_parent, + validator_index, + Misbehavior::ValidityDoubleVote(vdv), + ) + ) + ) if relay_parent == test_state.relay_parent => { + let ((t1, s1), (t2, s2)) = vdv.deconstruct::(); + let t1 = table_statement_to_primitive(t1); + let t2 = table_statement_to_primitive(t2); + + SignedFullStatement::new( + t1, + validator_index, + s1, + &test_state.signing_context, + &test_state.validator_public[validator_index.0 as usize], + ).expect("signature must be valid"); + + SignedFullStatement::new( + t2, + validator_index, + s2, + &test_state.signing_context, + &test_state.validator_public[validator_index.0 as usize], + ).expect("signature must be valid"); + } + ); + virtual_overseer + }); +} + +// Test that if we are asked to second an invalid candidate we +// can still second a valid one afterwards. +#[test] +fn backing_dont_second_invalid() { + let test_state = TestState::default(); + test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { + test_startup(&mut virtual_overseer, &test_state).await; + + let pov_block_a = PoV { + block_data: BlockData(vec![42, 43, 44]), + }; + + let pov_block_b = PoV { + block_data: BlockData(vec![45, 46, 47]), + }; + + let pov_hash_a = pov_block_a.hash(); + let pov_hash_b = pov_block_b.hash(); + + let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); + + let candidate_a = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash: pov_hash_a, + erasure_root: make_erasure_root(&test_state, pov_block_a.clone()), + ..Default::default() + }.build(); + + let candidate_b = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash: pov_hash_b, + erasure_root: make_erasure_root(&test_state, pov_block_b.clone()), + head_data: expected_head_data.clone(), + ..Default::default() + }.build(); + + let second = CandidateBackingMessage::Second( + test_state.relay_parent, + candidate_a.to_plain(), + pov_block_a.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: second }).await; + + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromChainState( + c, + pov, + tx, + ) + ) if pov == pov && &c == candidate_a.descriptor() => { + tx.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CollatorProtocol( + CollatorProtocolMessage::Invalid(parent_hash, c) + ) if parent_hash == test_state.relay_parent && c == candidate_a.to_plain() + ); + + let second = CandidateBackingMessage::Second( + test_state.relay_parent, + candidate_b.to_plain(), + pov_block_b.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: second }).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromChainState( + c, + pov, + tx, + ) + ) if pov == pov && &c == candidate_b.descriptor() => { + tx.send(Ok( + ValidationResult::Valid(CandidateCommitments { + head_data: expected_head_data.clone(), + upward_messages: Vec::new(), + horizontal_messages: Vec::new(), + new_validation_code: None, + processed_downward_messages: 0, + hrmp_watermark: 0, + }, test_state.validation_data), + )).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityStore( + AvailabilityStoreMessage::StoreAvailableData(candidate_hash, _, _, _, tx) + ) if candidate_hash == candidate_b.hash() => { + tx.send(Ok(())).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::StatementDistribution( + StatementDistributionMessage::Share( + parent_hash, + signed_statement, + ) + ) if parent_hash == test_state.relay_parent => { + assert_eq!(*signed_statement.payload(), Statement::Seconded(candidate_b)); + } + ); + + virtual_overseer.send(FromOverseer::Signal( + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) + ).await; + virtual_overseer + }); +} + +// Test that if we have already issued a statement (in this case `Invalid`) about a +// candidate we will not be issuing a `Seconded` statement on it. +#[test] +fn backing_second_after_first_fails_works() { + let test_state = TestState::default(); + test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { + test_startup(&mut virtual_overseer, &test_state).await; + + let pov = PoV { + block_data: BlockData(vec![42, 43, 44]), + }; + + let pov_hash = pov.hash(); + + let candidate = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash, + erasure_root: make_erasure_root(&test_state, pov.clone()), + ..Default::default() + }.build(); + + let validator2 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, Some(&test_state.validators[2].to_seed()) + ).await.expect("Insert key into keystore"); + + let signed_a = SignedFullStatement::sign( + &test_state.keystore, + Statement::Seconded(candidate.clone()), + &test_state.signing_context, + ValidatorIndex(2), + &validator2.into(), + ).await.ok().flatten().expect("should be signed"); + + // Send in a `Statement` with a candidate. + let statement = CandidateBackingMessage::Statement( + test_state.relay_parent, + signed_a.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + // Subsystem requests PoV and requests validation. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityDistribution( + AvailabilityDistributionMessage::FetchPoV { + relay_parent, + tx, + .. + } + ) if relay_parent == test_state.relay_parent => { + tx.send(pov.clone()).unwrap(); + } + ); + + + // Tell subsystem that this candidate is invalid. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromChainState( + c, + pov, + tx, + ) + ) if pov == pov && &c == candidate.descriptor() => { + tx.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); + } + ); + + // Ask subsystem to `Second` a candidate that already has a statement issued about. + // This should emit no actions from subsystem. + let second = CandidateBackingMessage::Second( + test_state.relay_parent, + candidate.to_plain(), + pov.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: second }).await; + + let pov_to_second = PoV { + block_data: BlockData(vec![3, 2, 1]), + }; + + let pov_hash = pov_to_second.hash(); + + let candidate_to_second = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash, + erasure_root: make_erasure_root(&test_state, pov_to_second.clone()), + ..Default::default() + }.build(); + + let second = CandidateBackingMessage::Second( + test_state.relay_parent, + candidate_to_second.to_plain(), + pov_to_second.clone(), + ); + + // In order to trigger _some_ actions from subsystem ask it to second another + // candidate. The only reason to do so is to make sure that no actions were + // triggered on the prev step. + virtual_overseer.send(FromOverseer::Communication{ msg: second }).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromChainState( + _, + pov, + _, + ) + ) => { + assert_eq!(&*pov, &pov_to_second); + } + ); + virtual_overseer + }); +} + +// That that if the validation of the candidate has failed this does not stop +// the work of this subsystem and so it is not fatal to the node. +#[test] +fn backing_works_after_failed_validation() { + let test_state = TestState::default(); + test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { + test_startup(&mut virtual_overseer, &test_state).await; + + let pov = PoV { + block_data: BlockData(vec![42, 43, 44]), + }; + + let pov_hash = pov.hash(); + + let candidate = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash, + erasure_root: make_erasure_root(&test_state, pov.clone()), + ..Default::default() + }.build(); + + let public2 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, Some(&test_state.validators[2].to_seed()) + ).await.expect("Insert key into keystore"); + let signed_a = SignedFullStatement::sign( + &test_state.keystore, + Statement::Seconded(candidate.clone()), + &test_state.signing_context, + ValidatorIndex(2), + &public2.into(), + ).await.ok().flatten().expect("should be signed"); + + // Send in a `Statement` with a candidate. + let statement = CandidateBackingMessage::Statement( + test_state.relay_parent, + signed_a.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + // Subsystem requests PoV and requests validation. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityDistribution( + AvailabilityDistributionMessage::FetchPoV { + relay_parent, + tx, + .. + } + ) if relay_parent == test_state.relay_parent => { + tx.send(pov.clone()).unwrap(); + } + ); + + // Tell subsystem that this candidate is invalid. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromChainState( + c, + pov, + tx, + ) + ) if pov == pov && &c == candidate.descriptor() => { + tx.send(Err(ValidationFailed("Internal test error".into()))).unwrap(); + } + ); + + // Try to get a set of backable candidates to trigger _some_ action in the subsystem + // and check that it is still alive. + let (tx, rx) = oneshot::channel(); + let msg = CandidateBackingMessage::GetBackedCandidates( + test_state.relay_parent, + vec![candidate.hash()], + tx, + ); + + virtual_overseer.send(FromOverseer::Communication{ msg }).await; + assert_eq!(rx.await.unwrap().len(), 0); + virtual_overseer + }); +} + +// Test that a `CandidateBackingMessage::Second` issues validation work +// and in case validation is successful issues a `StatementDistributionMessage`. +#[test] +fn backing_doesnt_second_wrong_collator() { + let mut test_state = TestState::default(); + test_state.availability_cores[0] = CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(1), + collator: Some(Sr25519Keyring::Bob.public().into()), + }); + + test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { + test_startup(&mut virtual_overseer, &test_state).await; + + let pov = PoV { + block_data: BlockData(vec![42, 43, 44]), + }; + + let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); + + let pov_hash = pov.hash(); + let candidate = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash, + head_data: expected_head_data.clone(), + erasure_root: make_erasure_root(&test_state, pov.clone()), + ..Default::default() + }.build(); + + let second = CandidateBackingMessage::Second( + test_state.relay_parent, + candidate.to_plain(), + pov.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: second }).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CollatorProtocol( + CollatorProtocolMessage::Invalid(parent, c) + ) if parent == test_state.relay_parent && c == candidate.to_plain() => { + } + ); + + virtual_overseer.send(FromOverseer::Signal( + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) + ).await; + virtual_overseer + }); +} + +#[test] +fn validation_work_ignores_wrong_collator() { + let mut test_state = TestState::default(); + test_state.availability_cores[0] = CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(1), + collator: Some(Sr25519Keyring::Bob.public().into()), + }); + + test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { + test_startup(&mut virtual_overseer, &test_state).await; + + let pov = PoV { + block_data: BlockData(vec![1, 2, 3]), + }; + + let pov_hash = pov.hash(); + + let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); + + let candidate_a = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash, + head_data: expected_head_data.clone(), + erasure_root: make_erasure_root(&test_state, pov.clone()), + ..Default::default() + }.build(); + + let public2 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, Some(&test_state.validators[2].to_seed()) + ).await.expect("Insert key into keystore"); + let seconding = SignedFullStatement::sign( + &test_state.keystore, + Statement::Seconded(candidate_a.clone()), + &test_state.signing_context, + ValidatorIndex(2), + &public2.into(), + ).await.ok().flatten().expect("should be signed"); + + let statement = CandidateBackingMessage::Statement( + test_state.relay_parent, + seconding.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + // The statement will be ignored because it has the wrong collator. + virtual_overseer.send(FromOverseer::Signal( + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) + ).await; + virtual_overseer + }); +} + +#[test] +fn candidate_backing_reorders_votes() { + use sp_core::Encode; + use std::convert::TryFrom; + + let para_id = ParaId::from(10); + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + Sr25519Keyring::One, + ]; + + let validator_public = validator_pubkeys(&validators); + let validator_groups = { + let mut validator_groups = HashMap::new(); + validator_groups.insert(para_id, vec![0, 1, 2, 3, 4, 5].into_iter().map(ValidatorIndex).collect()); + validator_groups + }; + + let table_context = TableContext { + validator: None, + groups: validator_groups, + validators: validator_public.clone(), + }; + + let fake_attestation = |idx: u32| { + let candidate: CommittedCandidateReceipt = Default::default(); + let hash = candidate.hash(); + let mut data = vec![0; 64]; + data[0..32].copy_from_slice(hash.0.as_bytes()); + data[32..36].copy_from_slice(idx.encode().as_slice()); + + let sig = ValidatorSignature::try_from(data).unwrap(); + statement_table::generic::ValidityAttestation::Implicit(sig) + }; + + let attested = TableAttestedCandidate { + candidate: Default::default(), + validity_votes: vec![ + (ValidatorIndex(5), fake_attestation(5)), + (ValidatorIndex(3), fake_attestation(3)), + (ValidatorIndex(1), fake_attestation(1)), + ], + group_id: para_id, + }; + + let backed = table_attested_to_backed(attested, &table_context).unwrap(); + + let expected_bitvec = { + let mut validator_indices = BitVec::::with_capacity(6); + validator_indices.resize(6, false); + + validator_indices.set(1, true); + validator_indices.set(3, true); + validator_indices.set(5, true); + + validator_indices + }; + + // Should be in bitfield order, which is opposite to the order provided to the function. + let expected_attestations = vec![ + fake_attestation(1).into(), + fake_attestation(3).into(), + fake_attestation(5).into(), + ]; + + assert_eq!(backed.validator_indices, expected_bitvec); + assert_eq!(backed.validity_votes, expected_attestations); +} + +// Test whether we retry on failed PoV fetching. +#[test] +fn retry_works() { + // sp_tracing::try_init_simple(); + let test_state = TestState::default(); + test_harness(test_state.keystore.clone(), |mut virtual_overseer| async move { + test_startup(&mut virtual_overseer, &test_state).await; + + let pov = PoV { + block_data: BlockData(vec![42, 43, 44]), + }; + + let pov_hash = pov.hash(); + + let candidate = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash, + erasure_root: make_erasure_root(&test_state, pov.clone()), + ..Default::default() + }.build(); + + let public2 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, Some(&test_state.validators[2].to_seed()) + ).await.expect("Insert key into keystore"); + let public3 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, + Some(&test_state.validators[3].to_seed()), + ).await.expect("Insert key into keystore"); + let public5 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, + Some(&test_state.validators[5].to_seed()), + ).await.expect("Insert key into keystore"); + let signed_a = SignedFullStatement::sign( + &test_state.keystore, + Statement::Seconded(candidate.clone()), + &test_state.signing_context, + ValidatorIndex(2), + &public2.into(), + ).await.ok().flatten().expect("should be signed"); + let signed_b = SignedFullStatement::sign( + &test_state.keystore, + Statement::Valid(candidate.hash()), + &test_state.signing_context, + ValidatorIndex(3), + &public3.into(), + ).await.ok().flatten().expect("should be signed"); + let signed_c = SignedFullStatement::sign( + &test_state.keystore, + Statement::Valid(candidate.hash()), + &test_state.signing_context, + ValidatorIndex(5), + &public5.into(), + ).await.ok().flatten().expect("should be signed"); + + // Send in a `Statement` with a candidate. + let statement = CandidateBackingMessage::Statement( + test_state.relay_parent, + signed_a.clone(), + ); + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + // Subsystem requests PoV and requests validation. + // We cancel - should mean retry on next backing statement. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityDistribution( + AvailabilityDistributionMessage::FetchPoV { + relay_parent, + tx, + .. + } + ) if relay_parent == test_state.relay_parent => { + std::mem::drop(tx); + } + ); + + let statement = CandidateBackingMessage::Statement( + test_state.relay_parent, + signed_b.clone(), + ); + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + let statement = CandidateBackingMessage::Statement( + test_state.relay_parent, + signed_c.clone(), + ); + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + // Not deterministic which message comes first: + for _ in 0u32..2 { + match virtual_overseer.recv().await { + AllMessages::Provisioner( + ProvisionerMessage::ProvisionableData( + _, + ProvisionableData::BackedCandidate(CandidateReceipt { + descriptor, + .. + }) + ) + ) => { + assert_eq!(descriptor, candidate.descriptor); + } + // Subsystem requests PoV and requests validation. + // We cancel once more: + AllMessages::AvailabilityDistribution( + AvailabilityDistributionMessage::FetchPoV { + relay_parent, + tx, + .. + } + ) if relay_parent == test_state.relay_parent => { + std::mem::drop(tx); + } + msg => { + assert!(false, "Unexpected message: {:?}", msg); + } + } + } + + // Subsystem requests PoV and requests validation. + // Now we pass. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityDistribution( + AvailabilityDistributionMessage::FetchPoV { + relay_parent, + tx, + .. + } + ) if relay_parent == test_state.relay_parent => { + tx.send(pov.clone()).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromChainState( + c, + pov, + _tx, + ) + ) if pov == pov && &c == candidate.descriptor() + ); + virtual_overseer + }); +} + +#[test] +fn observes_backing_even_if_not_validator() { + let test_state = TestState::default(); + let empty_keystore = Arc::new(sc_keystore::LocalKeystore::in_memory()); + test_harness(empty_keystore, |mut virtual_overseer| async move { + test_startup(&mut virtual_overseer, &test_state).await; + + let pov = PoV { + block_data: BlockData(vec![1, 2, 3]), + }; + + let pov_hash = pov.hash(); + + let expected_head_data = test_state.head_data.get(&test_state.chain_ids[0]).unwrap(); + + let candidate_a = TestCandidateBuilder { + para_id: test_state.chain_ids[0], + relay_parent: test_state.relay_parent, + pov_hash, + head_data: expected_head_data.clone(), + erasure_root: make_erasure_root(&test_state, pov.clone()), + ..Default::default() + }.build(); + + let candidate_a_hash = candidate_a.hash(); + let public0 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, + Some(&test_state.validators[0].to_seed()), + ).await.expect("Insert key into keystore"); + let public1 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, + Some(&test_state.validators[5].to_seed()), + ).await.expect("Insert key into keystore"); + let public2 = CryptoStore::sr25519_generate_new( + &*test_state.keystore, + ValidatorId::ID, + Some(&test_state.validators[2].to_seed()), + ).await.expect("Insert key into keystore"); + + // Produce a 3-of-5 quorum on the candidate. + + let signed_a = SignedFullStatement::sign( + &test_state.keystore, + Statement::Seconded(candidate_a.clone()), + &test_state.signing_context, + ValidatorIndex(0), + &public0.into(), + ).await.ok().flatten().expect("should be signed"); + + let signed_b = SignedFullStatement::sign( + &test_state.keystore, + Statement::Valid(candidate_a_hash), + &test_state.signing_context, + ValidatorIndex(5), + &public1.into(), + ).await.ok().flatten().expect("should be signed"); + + let signed_c = SignedFullStatement::sign( + &test_state.keystore, + Statement::Valid(candidate_a_hash), + &test_state.signing_context, + ValidatorIndex(2), + &public2.into(), + ).await.ok().flatten().expect("should be signed"); + + let statement = CandidateBackingMessage::Statement(test_state.relay_parent, signed_a.clone()); + + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + let statement = CandidateBackingMessage::Statement( + test_state.relay_parent, + signed_b.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + let statement = CandidateBackingMessage::Statement( + test_state.relay_parent, + signed_c.clone(), + ); + + virtual_overseer.send(FromOverseer::Communication{ msg: statement }).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::Provisioner( + ProvisionerMessage::ProvisionableData( + _, + ProvisionableData::BackedCandidate(candidate_receipt) + ) + ) => { + assert_eq!(candidate_receipt, candidate_a.to_plain()); + } + ); + + virtual_overseer.send(FromOverseer::Signal( + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::stop_work(test_state.relay_parent))) + ).await; + virtual_overseer + }); +} diff --git a/node/core/bitfield-signing/Cargo.toml b/node/core/bitfield-signing/Cargo.toml index de68072221f5..bf67a0f3783c 100644 --- a/node/core/bitfield-signing/Cargo.toml +++ b/node/core/bitfield-signing/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Parity Technologies "] edition = "2018" [dependencies] -futures = "0.3.12" +futures = "0.3.15" tracing = "0.1.26" polkadot-primitives = { path = "../../../primitives" } polkadot-node-subsystem = { path = "../../subsystem" } diff --git a/node/core/bitfield-signing/src/lib.rs b/node/core/bitfield-signing/src/lib.rs index adbb42fb1a5e..a36b2d8baa0b 100644 --- a/node/core/bitfield-signing/src/lib.rs +++ b/node/core/bitfield-signing/src/lib.rs @@ -70,11 +70,10 @@ pub enum Error { /// If there is a candidate pending availability, query the Availability Store /// for whether we have the availability chunk for our validator index. -#[tracing::instrument(level = "trace", skip(sender, span), fields(subsystem = LOG_TARGET))] async fn get_core_availability( core: &CoreState, validator_idx: ValidatorIndex, - sender: &Mutex<&mut impl SubsystemSender>, + sender: &Mutex<&mut impl SubsystemSender>, span: &jaeger::Span, ) -> Result { if let &CoreState::Occupied(ref core) = core { @@ -112,7 +111,7 @@ async fn get_core_availability( /// delegates to the v1 runtime API async fn get_availability_cores( relay_parent: Hash, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) -> Result, Error> { let (tx, rx) = oneshot::channel(); sender @@ -132,12 +131,11 @@ async fn get_availability_cores( /// - for each core, concurrently determine chunk availability (see `get_core_availability`) /// - return the bitfield if there were no errors at any point in this process /// (otherwise, it's prone to false negatives) -#[tracing::instrument(level = "trace", skip(sender, span), fields(subsystem = LOG_TARGET))] async fn construct_availability_bitfield( relay_parent: Hash, span: &jaeger::Span, validator_idx: ValidatorIndex, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) -> Result { // get the set of availability cores from the runtime let availability_cores = { @@ -226,7 +224,6 @@ impl JobTrait for BitfieldSigningJob { const NAME: &'static str = "BitfieldSigningJob"; /// Run a job for the parent block indicated - #[tracing::instrument(skip(span, keystore, metrics, _receiver, sender), fields(subsystem = LOG_TARGET))] fn run( relay_parent: Hash, span: Arc, @@ -319,6 +316,7 @@ mod tests { use super::*; use futures::{pin_mut, executor::block_on}; use polkadot_primitives::v1::{CandidateHash, OccupiedCore}; + use polkadot_node_subsystem::messages::AllMessages; fn occupied_core(para_id: u32, candidate_hash: CandidateHash) -> CoreState { CoreState::Occupied(OccupiedCore { diff --git a/node/core/candidate-validation/Cargo.toml b/node/core/candidate-validation/Cargo.toml index 61960f95b010..a109cb590510 100644 --- a/node/core/candidate-validation/Cargo.toml +++ b/node/core/candidate-validation/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] async-trait = "0.1.42" -futures = "0.3.12" +futures = "0.3.15" tracing = "0.1.26" sp-maybe-compressed-blob = { package = "sp-maybe-compressed-blob", git = "https://github.com/paritytech/substrate", branch = "master" } @@ -23,7 +23,7 @@ polkadot-node-core-pvf = { path = "../pvf" } [dev-dependencies] sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } -futures = { version = "0.3.12", features = ["thread-pool"] } +futures = { version = "0.3.15", features = ["thread-pool"] } assert_matches = "1.4.0" polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 1f27e1c55e8a..aabdf55517ed 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. +// Copyright 2020-2021 Parity Technologies (UK) Ltd. // This file is part of Polkadot. // Polkadot is free software: you can redistribute it and/or modify @@ -53,9 +53,13 @@ use std::path::PathBuf; use async_trait::async_trait; +#[cfg(test)] +mod tests; + const LOG_TARGET: &'static str = "parachain::candidate-validation"; /// Configuration for the candidate validation subsystem +#[derive(Clone)] pub struct Config { /// The path where candidate validation can store compiled artifacts for PVFs. pub artifacts_cache_path: PathBuf, @@ -94,7 +98,6 @@ impl Subsystem for CandidateValidationSubsystem where } } -#[tracing::instrument(skip(ctx, metrics), fields(subsystem = LOG_TARGET))] async fn run( mut ctx: impl SubsystemContext, metrics: Metrics, @@ -104,7 +107,7 @@ async fn run( let (mut validation_host, task) = polkadot_node_core_pvf::start( polkadot_node_core_pvf::Config::new(cache_path, program_path), ); - ctx.spawn_blocking("pvf-validation-host", task.boxed()).await?; + ctx.spawn_blocking("pvf-validation-host", task.boxed())?; loop { match ctx.recv().await? { @@ -194,7 +197,6 @@ enum AssumptionCheckOutcome { BadRequest, } -#[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn check_assumption_validation_data( ctx: &mut impl SubsystemContext, descriptor: &CandidateDescriptor, @@ -245,7 +247,6 @@ async fn check_assumption_validation_data( }) } -#[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))] async fn find_assumed_validation_data( ctx: &mut impl SubsystemContext, descriptor: &CandidateDescriptor, @@ -277,11 +278,6 @@ async fn find_assumed_validation_data( Ok(AssumptionCheckOutcome::DoesNotMatch) } -#[tracing::instrument( - level = "trace", - skip(ctx, validation_host, pov, metrics), - fields(subsystem = LOG_TARGET), -)] async fn spawn_validate_from_chain_state( ctx: &mut impl SubsystemContext, validation_host: &mut ValidationHost, @@ -340,11 +336,6 @@ async fn spawn_validate_from_chain_state( validation_result } -#[tracing::instrument( - level = "trace", - skip(validation_backend, validation_code, pov, metrics), - fields(subsystem = LOG_TARGET), -)] async fn validate_candidate_exhaustive( mut validation_backend: impl ValidationBackend, persisted_validation_data: PersistedValidationData, @@ -478,7 +469,6 @@ impl ValidationBackend for &'_ mut ValidationHost { /// Does basic checks of a candidate. Provide the encoded PoV-block. Returns `Ok` if basic checks /// are passed, `Err` otherwise. -#[tracing::instrument(level = "trace", skip(pov, validation_code), fields(subsystem = LOG_TARGET))] fn perform_basic_checks( candidate: &CandidateDescriptor, max_pov_size: u32, @@ -597,624 +587,3 @@ impl metrics::Metrics for Metrics { Ok(Metrics(Some(metrics))) } } - -#[cfg(test)] -mod tests { - use super::*; - use polkadot_node_subsystem_test_helpers as test_helpers; - use polkadot_primitives::v1::{HeadData, UpwardMessage}; - use sp_core::testing::TaskExecutor; - use futures::executor; - use assert_matches::assert_matches; - use sp_keyring::Sr25519Keyring; - - fn collator_sign(descriptor: &mut CandidateDescriptor, collator: Sr25519Keyring) { - descriptor.collator = collator.public().into(); - let payload = polkadot_primitives::v1::collator_signature_payload( - &descriptor.relay_parent, - &descriptor.para_id, - &descriptor.persisted_validation_data_hash, - &descriptor.pov_hash, - &descriptor.validation_code_hash, - ); - - descriptor.signature = collator.sign(&payload[..]).into(); - assert!(descriptor.check_collator_signature().is_ok()); - } - - #[test] - fn correctly_checks_included_assumption() { - let validation_data: PersistedValidationData = Default::default(); - let validation_code: ValidationCode = vec![1, 2, 3].into(); - - let persisted_validation_data_hash = validation_data.hash(); - let relay_parent = [2; 32].into(); - let para_id = 5.into(); - - let mut candidate = CandidateDescriptor::default(); - candidate.relay_parent = relay_parent; - candidate.persisted_validation_data_hash = persisted_validation_data_hash; - candidate.para_id = para_id; - - let pool = TaskExecutor::new(); - let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone()); - - let (check_fut, check_result) = check_assumption_validation_data( - &mut ctx, - &candidate, - OccupiedCoreAssumption::Included, - ).remote_handle(); - - let test_fut = async move { - assert_matches!( - ctx_handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - rp, - RuntimeApiRequest::PersistedValidationData( - p, - OccupiedCoreAssumption::Included, - tx - ), - )) => { - assert_eq!(rp, relay_parent); - assert_eq!(p, para_id); - - let _ = tx.send(Ok(Some(validation_data.clone()))); - } - ); - - assert_matches!( - ctx_handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - rp, - RuntimeApiRequest::ValidationCode(p, OccupiedCoreAssumption::Included, tx) - )) => { - assert_eq!(rp, relay_parent); - assert_eq!(p, para_id); - - let _ = tx.send(Ok(Some(validation_code.clone()))); - } - ); - - assert_matches!(check_result.await.unwrap(), AssumptionCheckOutcome::Matches(o, v) => { - assert_eq!(o, validation_data); - assert_eq!(v, validation_code); - }); - }; - - let test_fut = future::join(test_fut, check_fut); - executor::block_on(test_fut); - } - - #[test] - fn correctly_checks_timed_out_assumption() { - let validation_data: PersistedValidationData = Default::default(); - let validation_code: ValidationCode = vec![1, 2, 3].into(); - - let persisted_validation_data_hash = validation_data.hash(); - let relay_parent = [2; 32].into(); - let para_id = 5.into(); - - let mut candidate = CandidateDescriptor::default(); - candidate.relay_parent = relay_parent; - candidate.persisted_validation_data_hash = persisted_validation_data_hash; - candidate.para_id = para_id; - - let pool = TaskExecutor::new(); - let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone()); - - let (check_fut, check_result) = check_assumption_validation_data( - &mut ctx, - &candidate, - OccupiedCoreAssumption::TimedOut, - ).remote_handle(); - - let test_fut = async move { - assert_matches!( - ctx_handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - rp, - RuntimeApiRequest::PersistedValidationData( - p, - OccupiedCoreAssumption::TimedOut, - tx - ), - )) => { - assert_eq!(rp, relay_parent); - assert_eq!(p, para_id); - - let _ = tx.send(Ok(Some(validation_data.clone()))); - } - ); - - assert_matches!( - ctx_handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - rp, - RuntimeApiRequest::ValidationCode(p, OccupiedCoreAssumption::TimedOut, tx) - )) => { - assert_eq!(rp, relay_parent); - assert_eq!(p, para_id); - - let _ = tx.send(Ok(Some(validation_code.clone()))); - } - ); - - assert_matches!(check_result.await.unwrap(), AssumptionCheckOutcome::Matches(o, v) => { - assert_eq!(o, validation_data); - assert_eq!(v, validation_code); - }); - }; - - let test_fut = future::join(test_fut, check_fut); - executor::block_on(test_fut); - } - - #[test] - fn check_is_bad_request_if_no_validation_data() { - let validation_data: PersistedValidationData = Default::default(); - let persisted_validation_data_hash = validation_data.hash(); - let relay_parent = [2; 32].into(); - let para_id = 5.into(); - - let mut candidate = CandidateDescriptor::default(); - candidate.relay_parent = relay_parent; - candidate.persisted_validation_data_hash = persisted_validation_data_hash; - candidate.para_id = para_id; - - let pool = TaskExecutor::new(); - let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone()); - - let (check_fut, check_result) = check_assumption_validation_data( - &mut ctx, - &candidate, - OccupiedCoreAssumption::Included, - ).remote_handle(); - - let test_fut = async move { - assert_matches!( - ctx_handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - rp, - RuntimeApiRequest::PersistedValidationData( - p, - OccupiedCoreAssumption::Included, - tx - ), - )) => { - assert_eq!(rp, relay_parent); - assert_eq!(p, para_id); - - let _ = tx.send(Ok(None)); - } - ); - - assert_matches!(check_result.await.unwrap(), AssumptionCheckOutcome::BadRequest); - }; - - let test_fut = future::join(test_fut, check_fut); - executor::block_on(test_fut); - } - - #[test] - fn check_is_bad_request_if_no_validation_code() { - let validation_data: PersistedValidationData = Default::default(); - let persisted_validation_data_hash = validation_data.hash(); - let relay_parent = [2; 32].into(); - let para_id = 5.into(); - - let mut candidate = CandidateDescriptor::default(); - candidate.relay_parent = relay_parent; - candidate.persisted_validation_data_hash = persisted_validation_data_hash; - candidate.para_id = para_id; - - let pool = TaskExecutor::new(); - let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone()); - - let (check_fut, check_result) = check_assumption_validation_data( - &mut ctx, - &candidate, - OccupiedCoreAssumption::TimedOut, - ).remote_handle(); - - let test_fut = async move { - assert_matches!( - ctx_handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - rp, - RuntimeApiRequest::PersistedValidationData( - p, - OccupiedCoreAssumption::TimedOut, - tx - ), - )) => { - assert_eq!(rp, relay_parent); - assert_eq!(p, para_id); - - let _ = tx.send(Ok(Some(validation_data.clone()))); - } - ); - - assert_matches!( - ctx_handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - rp, - RuntimeApiRequest::ValidationCode(p, OccupiedCoreAssumption::TimedOut, tx) - )) => { - assert_eq!(rp, relay_parent); - assert_eq!(p, para_id); - - let _ = tx.send(Ok(None)); - } - ); - - assert_matches!(check_result.await.unwrap(), AssumptionCheckOutcome::BadRequest); - }; - - let test_fut = future::join(test_fut, check_fut); - executor::block_on(test_fut); - } - - #[test] - fn check_does_not_match() { - let validation_data: PersistedValidationData = Default::default(); - let relay_parent = [2; 32].into(); - let para_id = 5.into(); - - let mut candidate = CandidateDescriptor::default(); - candidate.relay_parent = relay_parent; - candidate.persisted_validation_data_hash = [3; 32].into(); - candidate.para_id = para_id; - - let pool = TaskExecutor::new(); - let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone()); - - let (check_fut, check_result) = check_assumption_validation_data( - &mut ctx, - &candidate, - OccupiedCoreAssumption::Included, - ).remote_handle(); - - let test_fut = async move { - assert_matches!( - ctx_handle.recv().await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - rp, - RuntimeApiRequest::PersistedValidationData( - p, - OccupiedCoreAssumption::Included, - tx - ), - )) => { - assert_eq!(rp, relay_parent); - assert_eq!(p, para_id); - - let _ = tx.send(Ok(Some(validation_data.clone()))); - } - ); - - assert_matches!(check_result.await.unwrap(), AssumptionCheckOutcome::DoesNotMatch); - }; - - let test_fut = future::join(test_fut, check_fut); - executor::block_on(test_fut); - } - - struct MockValidatorBackend { - result: Result, - } - - impl MockValidatorBackend { - fn with_hardcoded_result(result: Result) -> Self { - Self { - result, - } - } - } - - #[async_trait] - impl ValidationBackend for MockValidatorBackend { - async fn validate_candidate( - &mut self, - _raw_validation_code: Vec, - _params: ValidationParams - ) -> Result { - self.result.clone() - } - } - - #[test] - fn candidate_validation_ok_is_ok() { - let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; - - let pov = PoV { block_data: BlockData(vec![1; 32]) }; - let head_data = HeadData(vec![1, 1, 1]); - let validation_code = ValidationCode(vec![2; 16]); - - let mut descriptor = CandidateDescriptor::default(); - descriptor.pov_hash = pov.hash(); - descriptor.para_head = head_data.hash(); - descriptor.validation_code_hash = validation_code.hash(); - collator_sign(&mut descriptor, Sr25519Keyring::Alice); - - let check = perform_basic_checks( - &descriptor, - validation_data.max_pov_size, - &pov, - &validation_code, - ); - assert!(check.is_ok()); - - let validation_result = WasmValidationResult { - head_data, - new_validation_code: Some(vec![2, 2, 2].into()), - upward_messages: Vec::new(), - horizontal_messages: Vec::new(), - processed_downward_messages: 0, - hrmp_watermark: 0, - }; - - let v = executor::block_on(validate_candidate_exhaustive( - MockValidatorBackend::with_hardcoded_result(Ok(validation_result)), - validation_data.clone(), - validation_code, - descriptor, - Arc::new(pov), - &Default::default(), - )) - .unwrap() - .unwrap(); - - assert_matches!(v, ValidationResult::Valid(outputs, used_validation_data) => { - assert_eq!(outputs.head_data, HeadData(vec![1, 1, 1])); - assert_eq!(outputs.upward_messages, Vec::::new()); - assert_eq!(outputs.horizontal_messages, Vec::new()); - assert_eq!(outputs.new_validation_code, Some(vec![2, 2, 2].into())); - assert_eq!(outputs.hrmp_watermark, 0); - assert_eq!(used_validation_data, validation_data); - }); - } - - #[test] - fn candidate_validation_bad_return_is_invalid() { - let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; - - let pov = PoV { block_data: BlockData(vec![1; 32]) }; - let validation_code = ValidationCode(vec![2; 16]); - - let mut descriptor = CandidateDescriptor::default(); - descriptor.pov_hash = pov.hash(); - descriptor.validation_code_hash = validation_code.hash(); - collator_sign(&mut descriptor, Sr25519Keyring::Alice); - - let check = perform_basic_checks( - &descriptor, - validation_data.max_pov_size, - &pov, - &validation_code, - ); - assert!(check.is_ok()); - - let v = executor::block_on(validate_candidate_exhaustive( - MockValidatorBackend::with_hardcoded_result( - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbigiousWorkerDeath)) - ), - validation_data, - validation_code, - descriptor, - Arc::new(pov), - &Default::default(), - )) - .unwrap() - .unwrap(); - - assert_matches!(v, ValidationResult::Invalid(InvalidCandidate::ExecutionError(_))); - } - - #[test] - fn candidate_validation_timeout_is_internal_error() { - let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; - - let pov = PoV { block_data: BlockData(vec![1; 32]) }; - let validation_code = ValidationCode(vec![2; 16]); - - let mut descriptor = CandidateDescriptor::default(); - descriptor.pov_hash = pov.hash(); - descriptor.validation_code_hash = validation_code.hash(); - collator_sign(&mut descriptor, Sr25519Keyring::Alice); - - let check = perform_basic_checks( - &descriptor, - validation_data.max_pov_size, - &pov, - &validation_code, - ); - assert!(check.is_ok()); - - let v = executor::block_on(validate_candidate_exhaustive( - MockValidatorBackend::with_hardcoded_result( - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout)), - ), - validation_data, - validation_code, - descriptor, - Arc::new(pov), - &Default::default(), - )) - .unwrap(); - - assert_matches!(v, Ok(ValidationResult::Invalid(InvalidCandidate::Timeout))); - } - - #[test] - fn candidate_validation_code_mismatch_is_invalid() { - let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; - - let pov = PoV { block_data: BlockData(vec![1; 32]) }; - let validation_code = ValidationCode(vec![2; 16]); - - let mut descriptor = CandidateDescriptor::default(); - descriptor.pov_hash = pov.hash(); - descriptor.validation_code_hash = ValidationCode(vec![1; 16]).hash(); - collator_sign(&mut descriptor, Sr25519Keyring::Alice); - - let check = perform_basic_checks( - &descriptor, - validation_data.max_pov_size, - &pov, - &validation_code, - ); - assert_matches!(check, Err(InvalidCandidate::CodeHashMismatch)); - - let v = executor::block_on(validate_candidate_exhaustive( - MockValidatorBackend::with_hardcoded_result( - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout)), - ), - validation_data, - validation_code, - descriptor, - Arc::new(pov), - &Default::default(), - )) - .unwrap() - .unwrap(); - - assert_matches!(v, ValidationResult::Invalid(InvalidCandidate::CodeHashMismatch)); - } - - #[test] - fn compressed_code_works() { - let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; - let pov = PoV { block_data: BlockData(vec![1; 32]) }; - let head_data = HeadData(vec![1, 1, 1]); - - let raw_code = vec![2u8; 16]; - let validation_code = sp_maybe_compressed_blob::compress( - &raw_code, - VALIDATION_CODE_BOMB_LIMIT, - ) - .map(ValidationCode) - .unwrap(); - - let mut descriptor = CandidateDescriptor::default(); - descriptor.pov_hash = pov.hash(); - descriptor.para_head = head_data.hash(); - descriptor.validation_code_hash = validation_code.hash(); - collator_sign(&mut descriptor, Sr25519Keyring::Alice); - - let validation_result = WasmValidationResult { - head_data, - new_validation_code: None, - upward_messages: Vec::new(), - horizontal_messages: Vec::new(), - processed_downward_messages: 0, - hrmp_watermark: 0, - }; - - let v = executor::block_on(validate_candidate_exhaustive( - MockValidatorBackend::with_hardcoded_result(Ok(validation_result)), - validation_data, - validation_code, - descriptor, - Arc::new(pov), - &Default::default(), - )) - .unwrap(); - - assert_matches!(v, Ok(ValidationResult::Valid(_, _))); - } - - #[test] - fn code_decompression_failure_is_invalid() { - let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; - let pov = PoV { block_data: BlockData(vec![1; 32]) }; - let head_data = HeadData(vec![1, 1, 1]); - - let raw_code = vec![2u8; VALIDATION_CODE_BOMB_LIMIT + 1]; - let validation_code = sp_maybe_compressed_blob::compress( - &raw_code, - VALIDATION_CODE_BOMB_LIMIT + 1, - ) - .map(ValidationCode) - .unwrap(); - - let mut descriptor = CandidateDescriptor::default(); - descriptor.pov_hash = pov.hash(); - descriptor.para_head = head_data.hash(); - descriptor.validation_code_hash = validation_code.hash(); - collator_sign(&mut descriptor, Sr25519Keyring::Alice); - - let validation_result = WasmValidationResult { - head_data, - new_validation_code: None, - upward_messages: Vec::new(), - horizontal_messages: Vec::new(), - processed_downward_messages: 0, - hrmp_watermark: 0, - }; - - let v = executor::block_on(validate_candidate_exhaustive( - MockValidatorBackend::with_hardcoded_result(Ok(validation_result)), - validation_data, - validation_code, - descriptor, - Arc::new(pov), - &Default::default(), - )) - .unwrap(); - - assert_matches!( - v, - Ok(ValidationResult::Invalid(InvalidCandidate::CodeDecompressionFailure)) - ); - } - - #[test] - fn pov_decompression_failure_is_invalid() { - let validation_data = PersistedValidationData { - max_pov_size: POV_BOMB_LIMIT as u32, - ..Default::default() - }; - let head_data = HeadData(vec![1, 1, 1]); - - let raw_block_data = vec![2u8; POV_BOMB_LIMIT + 1]; - let pov = sp_maybe_compressed_blob::compress( - &raw_block_data, - POV_BOMB_LIMIT + 1, - ) - .map(|raw| PoV { block_data: BlockData(raw) }) - .unwrap(); - - let validation_code = ValidationCode(vec![2; 16]); - - let mut descriptor = CandidateDescriptor::default(); - descriptor.pov_hash = pov.hash(); - descriptor.para_head = head_data.hash(); - descriptor.validation_code_hash = validation_code.hash(); - collator_sign(&mut descriptor, Sr25519Keyring::Alice); - - let validation_result = WasmValidationResult { - head_data, - new_validation_code: None, - upward_messages: Vec::new(), - horizontal_messages: Vec::new(), - processed_downward_messages: 0, - hrmp_watermark: 0, - }; - - let v = executor::block_on(validate_candidate_exhaustive( - MockValidatorBackend::with_hardcoded_result(Ok(validation_result)), - validation_data, - validation_code, - descriptor, - Arc::new(pov), - &Default::default(), - )) - .unwrap(); - - assert_matches!( - v, - Ok(ValidationResult::Invalid(InvalidCandidate::PoVDecompressionFailure)) - ); - } -} diff --git a/node/core/candidate-validation/src/tests.rs b/node/core/candidate-validation/src/tests.rs new file mode 100644 index 000000000000..26ccdbb7cc9e --- /dev/null +++ b/node/core/candidate-validation/src/tests.rs @@ -0,0 +1,633 @@ +// Copyright 2020-2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; +use polkadot_node_subsystem_test_helpers as test_helpers; +use polkadot_primitives::v1::{HeadData, UpwardMessage}; +use sp_core::testing::TaskExecutor; +use futures::executor; +use assert_matches::assert_matches; +use sp_keyring::Sr25519Keyring; + +fn collator_sign(descriptor: &mut CandidateDescriptor, collator: Sr25519Keyring) { + descriptor.collator = collator.public().into(); + let payload = polkadot_primitives::v1::collator_signature_payload( + &descriptor.relay_parent, + &descriptor.para_id, + &descriptor.persisted_validation_data_hash, + &descriptor.pov_hash, + &descriptor.validation_code_hash, + ); + + descriptor.signature = collator.sign(&payload[..]).into(); + assert!(descriptor.check_collator_signature().is_ok()); +} + +#[test] +fn correctly_checks_included_assumption() { + let validation_data: PersistedValidationData = Default::default(); + let validation_code: ValidationCode = vec![1, 2, 3].into(); + + let persisted_validation_data_hash = validation_data.hash(); + let relay_parent = [2; 32].into(); + let para_id = 5.into(); + + let mut candidate = CandidateDescriptor::default(); + candidate.relay_parent = relay_parent; + candidate.persisted_validation_data_hash = persisted_validation_data_hash; + candidate.para_id = para_id; + + let pool = TaskExecutor::new(); + let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone()); + + let (check_fut, check_result) = check_assumption_validation_data( + &mut ctx, + &candidate, + OccupiedCoreAssumption::Included, + ).remote_handle(); + + let test_fut = async move { + assert_matches!( + ctx_handle.recv().await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + rp, + RuntimeApiRequest::PersistedValidationData( + p, + OccupiedCoreAssumption::Included, + tx + ), + )) => { + assert_eq!(rp, relay_parent); + assert_eq!(p, para_id); + + let _ = tx.send(Ok(Some(validation_data.clone()))); + } + ); + + assert_matches!( + ctx_handle.recv().await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + rp, + RuntimeApiRequest::ValidationCode(p, OccupiedCoreAssumption::Included, tx) + )) => { + assert_eq!(rp, relay_parent); + assert_eq!(p, para_id); + + let _ = tx.send(Ok(Some(validation_code.clone()))); + } + ); + + assert_matches!(check_result.await.unwrap(), AssumptionCheckOutcome::Matches(o, v) => { + assert_eq!(o, validation_data); + assert_eq!(v, validation_code); + }); + }; + + let test_fut = future::join(test_fut, check_fut); + executor::block_on(test_fut); +} + +#[test] +fn correctly_checks_timed_out_assumption() { + let validation_data: PersistedValidationData = Default::default(); + let validation_code: ValidationCode = vec![1, 2, 3].into(); + + let persisted_validation_data_hash = validation_data.hash(); + let relay_parent = [2; 32].into(); + let para_id = 5.into(); + + let mut candidate = CandidateDescriptor::default(); + candidate.relay_parent = relay_parent; + candidate.persisted_validation_data_hash = persisted_validation_data_hash; + candidate.para_id = para_id; + + let pool = TaskExecutor::new(); + let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone()); + + let (check_fut, check_result) = check_assumption_validation_data( + &mut ctx, + &candidate, + OccupiedCoreAssumption::TimedOut, + ).remote_handle(); + + let test_fut = async move { + assert_matches!( + ctx_handle.recv().await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + rp, + RuntimeApiRequest::PersistedValidationData( + p, + OccupiedCoreAssumption::TimedOut, + tx + ), + )) => { + assert_eq!(rp, relay_parent); + assert_eq!(p, para_id); + + let _ = tx.send(Ok(Some(validation_data.clone()))); + } + ); + + assert_matches!( + ctx_handle.recv().await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + rp, + RuntimeApiRequest::ValidationCode(p, OccupiedCoreAssumption::TimedOut, tx) + )) => { + assert_eq!(rp, relay_parent); + assert_eq!(p, para_id); + + let _ = tx.send(Ok(Some(validation_code.clone()))); + } + ); + + assert_matches!(check_result.await.unwrap(), AssumptionCheckOutcome::Matches(o, v) => { + assert_eq!(o, validation_data); + assert_eq!(v, validation_code); + }); + }; + + let test_fut = future::join(test_fut, check_fut); + executor::block_on(test_fut); +} + +#[test] +fn check_is_bad_request_if_no_validation_data() { + let validation_data: PersistedValidationData = Default::default(); + let persisted_validation_data_hash = validation_data.hash(); + let relay_parent = [2; 32].into(); + let para_id = 5.into(); + + let mut candidate = CandidateDescriptor::default(); + candidate.relay_parent = relay_parent; + candidate.persisted_validation_data_hash = persisted_validation_data_hash; + candidate.para_id = para_id; + + let pool = TaskExecutor::new(); + let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone()); + + let (check_fut, check_result) = check_assumption_validation_data( + &mut ctx, + &candidate, + OccupiedCoreAssumption::Included, + ).remote_handle(); + + let test_fut = async move { + assert_matches!( + ctx_handle.recv().await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + rp, + RuntimeApiRequest::PersistedValidationData( + p, + OccupiedCoreAssumption::Included, + tx + ), + )) => { + assert_eq!(rp, relay_parent); + assert_eq!(p, para_id); + + let _ = tx.send(Ok(None)); + } + ); + + assert_matches!(check_result.await.unwrap(), AssumptionCheckOutcome::BadRequest); + }; + + let test_fut = future::join(test_fut, check_fut); + executor::block_on(test_fut); +} + +#[test] +fn check_is_bad_request_if_no_validation_code() { + let validation_data: PersistedValidationData = Default::default(); + let persisted_validation_data_hash = validation_data.hash(); + let relay_parent = [2; 32].into(); + let para_id = 5.into(); + + let mut candidate = CandidateDescriptor::default(); + candidate.relay_parent = relay_parent; + candidate.persisted_validation_data_hash = persisted_validation_data_hash; + candidate.para_id = para_id; + + let pool = TaskExecutor::new(); + let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone()); + + let (check_fut, check_result) = check_assumption_validation_data( + &mut ctx, + &candidate, + OccupiedCoreAssumption::TimedOut, + ).remote_handle(); + + let test_fut = async move { + assert_matches!( + ctx_handle.recv().await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + rp, + RuntimeApiRequest::PersistedValidationData( + p, + OccupiedCoreAssumption::TimedOut, + tx + ), + )) => { + assert_eq!(rp, relay_parent); + assert_eq!(p, para_id); + + let _ = tx.send(Ok(Some(validation_data.clone()))); + } + ); + + assert_matches!( + ctx_handle.recv().await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + rp, + RuntimeApiRequest::ValidationCode(p, OccupiedCoreAssumption::TimedOut, tx) + )) => { + assert_eq!(rp, relay_parent); + assert_eq!(p, para_id); + + let _ = tx.send(Ok(None)); + } + ); + + assert_matches!(check_result.await.unwrap(), AssumptionCheckOutcome::BadRequest); + }; + + let test_fut = future::join(test_fut, check_fut); + executor::block_on(test_fut); +} + +#[test] +fn check_does_not_match() { + let validation_data: PersistedValidationData = Default::default(); + let relay_parent = [2; 32].into(); + let para_id = 5.into(); + + let mut candidate = CandidateDescriptor::default(); + candidate.relay_parent = relay_parent; + candidate.persisted_validation_data_hash = [3; 32].into(); + candidate.para_id = para_id; + + let pool = TaskExecutor::new(); + let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone()); + + let (check_fut, check_result) = check_assumption_validation_data( + &mut ctx, + &candidate, + OccupiedCoreAssumption::Included, + ).remote_handle(); + + let test_fut = async move { + assert_matches!( + ctx_handle.recv().await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + rp, + RuntimeApiRequest::PersistedValidationData( + p, + OccupiedCoreAssumption::Included, + tx + ), + )) => { + assert_eq!(rp, relay_parent); + assert_eq!(p, para_id); + + let _ = tx.send(Ok(Some(validation_data.clone()))); + } + ); + + assert_matches!(check_result.await.unwrap(), AssumptionCheckOutcome::DoesNotMatch); + }; + + let test_fut = future::join(test_fut, check_fut); + executor::block_on(test_fut); +} + +struct MockValidatorBackend { + result: Result, +} + +impl MockValidatorBackend { + fn with_hardcoded_result(result: Result) -> Self { + Self { + result, + } + } +} + +#[async_trait] +impl ValidationBackend for MockValidatorBackend { + async fn validate_candidate( + &mut self, + _raw_validation_code: Vec, + _params: ValidationParams + ) -> Result { + self.result.clone() + } +} + +#[test] +fn candidate_validation_ok_is_ok() { + let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; + + let pov = PoV { block_data: BlockData(vec![1; 32]) }; + let head_data = HeadData(vec![1, 1, 1]); + let validation_code = ValidationCode(vec![2; 16]); + + let mut descriptor = CandidateDescriptor::default(); + descriptor.pov_hash = pov.hash(); + descriptor.para_head = head_data.hash(); + descriptor.validation_code_hash = validation_code.hash(); + collator_sign(&mut descriptor, Sr25519Keyring::Alice); + + let check = perform_basic_checks( + &descriptor, + validation_data.max_pov_size, + &pov, + &validation_code, + ); + assert!(check.is_ok()); + + let validation_result = WasmValidationResult { + head_data, + new_validation_code: Some(vec![2, 2, 2].into()), + upward_messages: Vec::new(), + horizontal_messages: Vec::new(), + processed_downward_messages: 0, + hrmp_watermark: 0, + }; + + let v = executor::block_on(validate_candidate_exhaustive( + MockValidatorBackend::with_hardcoded_result(Ok(validation_result)), + validation_data.clone(), + validation_code, + descriptor, + Arc::new(pov), + &Default::default(), + )) + .unwrap() + .unwrap(); + + assert_matches!(v, ValidationResult::Valid(outputs, used_validation_data) => { + assert_eq!(outputs.head_data, HeadData(vec![1, 1, 1])); + assert_eq!(outputs.upward_messages, Vec::::new()); + assert_eq!(outputs.horizontal_messages, Vec::new()); + assert_eq!(outputs.new_validation_code, Some(vec![2, 2, 2].into())); + assert_eq!(outputs.hrmp_watermark, 0); + assert_eq!(used_validation_data, validation_data); + }); +} + +#[test] +fn candidate_validation_bad_return_is_invalid() { + let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; + + let pov = PoV { block_data: BlockData(vec![1; 32]) }; + let validation_code = ValidationCode(vec![2; 16]); + + let mut descriptor = CandidateDescriptor::default(); + descriptor.pov_hash = pov.hash(); + descriptor.validation_code_hash = validation_code.hash(); + collator_sign(&mut descriptor, Sr25519Keyring::Alice); + + let check = perform_basic_checks( + &descriptor, + validation_data.max_pov_size, + &pov, + &validation_code, + ); + assert!(check.is_ok()); + + let v = executor::block_on(validate_candidate_exhaustive( + MockValidatorBackend::with_hardcoded_result( + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbigiousWorkerDeath)) + ), + validation_data, + validation_code, + descriptor, + Arc::new(pov), + &Default::default(), + )) + .unwrap() + .unwrap(); + + assert_matches!(v, ValidationResult::Invalid(InvalidCandidate::ExecutionError(_))); +} + +#[test] +fn candidate_validation_timeout_is_internal_error() { + let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; + + let pov = PoV { block_data: BlockData(vec![1; 32]) }; + let validation_code = ValidationCode(vec![2; 16]); + + let mut descriptor = CandidateDescriptor::default(); + descriptor.pov_hash = pov.hash(); + descriptor.validation_code_hash = validation_code.hash(); + collator_sign(&mut descriptor, Sr25519Keyring::Alice); + + let check = perform_basic_checks( + &descriptor, + validation_data.max_pov_size, + &pov, + &validation_code, + ); + assert!(check.is_ok()); + + let v = executor::block_on(validate_candidate_exhaustive( + MockValidatorBackend::with_hardcoded_result( + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout)), + ), + validation_data, + validation_code, + descriptor, + Arc::new(pov), + &Default::default(), + )) + .unwrap(); + + assert_matches!(v, Ok(ValidationResult::Invalid(InvalidCandidate::Timeout))); +} + +#[test] +fn candidate_validation_code_mismatch_is_invalid() { + let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; + + let pov = PoV { block_data: BlockData(vec![1; 32]) }; + let validation_code = ValidationCode(vec![2; 16]); + + let mut descriptor = CandidateDescriptor::default(); + descriptor.pov_hash = pov.hash(); + descriptor.validation_code_hash = ValidationCode(vec![1; 16]).hash(); + collator_sign(&mut descriptor, Sr25519Keyring::Alice); + + let check = perform_basic_checks( + &descriptor, + validation_data.max_pov_size, + &pov, + &validation_code, + ); + assert_matches!(check, Err(InvalidCandidate::CodeHashMismatch)); + + let v = executor::block_on(validate_candidate_exhaustive( + MockValidatorBackend::with_hardcoded_result( + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout)), + ), + validation_data, + validation_code, + descriptor, + Arc::new(pov), + &Default::default(), + )) + .unwrap() + .unwrap(); + + assert_matches!(v, ValidationResult::Invalid(InvalidCandidate::CodeHashMismatch)); +} + +#[test] +fn compressed_code_works() { + let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; + let pov = PoV { block_data: BlockData(vec![1; 32]) }; + let head_data = HeadData(vec![1, 1, 1]); + + let raw_code = vec![2u8; 16]; + let validation_code = sp_maybe_compressed_blob::compress( + &raw_code, + VALIDATION_CODE_BOMB_LIMIT, + ) + .map(ValidationCode) + .unwrap(); + + let mut descriptor = CandidateDescriptor::default(); + descriptor.pov_hash = pov.hash(); + descriptor.para_head = head_data.hash(); + descriptor.validation_code_hash = validation_code.hash(); + collator_sign(&mut descriptor, Sr25519Keyring::Alice); + + let validation_result = WasmValidationResult { + head_data, + new_validation_code: None, + upward_messages: Vec::new(), + horizontal_messages: Vec::new(), + processed_downward_messages: 0, + hrmp_watermark: 0, + }; + + let v = executor::block_on(validate_candidate_exhaustive( + MockValidatorBackend::with_hardcoded_result(Ok(validation_result)), + validation_data, + validation_code, + descriptor, + Arc::new(pov), + &Default::default(), + )) + .unwrap(); + + assert_matches!(v, Ok(ValidationResult::Valid(_, _))); +} + +#[test] +fn code_decompression_failure_is_invalid() { + let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; + let pov = PoV { block_data: BlockData(vec![1; 32]) }; + let head_data = HeadData(vec![1, 1, 1]); + + let raw_code = vec![2u8; VALIDATION_CODE_BOMB_LIMIT + 1]; + let validation_code = sp_maybe_compressed_blob::compress( + &raw_code, + VALIDATION_CODE_BOMB_LIMIT + 1, + ) + .map(ValidationCode) + .unwrap(); + + let mut descriptor = CandidateDescriptor::default(); + descriptor.pov_hash = pov.hash(); + descriptor.para_head = head_data.hash(); + descriptor.validation_code_hash = validation_code.hash(); + collator_sign(&mut descriptor, Sr25519Keyring::Alice); + + let validation_result = WasmValidationResult { + head_data, + new_validation_code: None, + upward_messages: Vec::new(), + horizontal_messages: Vec::new(), + processed_downward_messages: 0, + hrmp_watermark: 0, + }; + + let v = executor::block_on(validate_candidate_exhaustive( + MockValidatorBackend::with_hardcoded_result(Ok(validation_result)), + validation_data, + validation_code, + descriptor, + Arc::new(pov), + &Default::default(), + )) + .unwrap(); + + assert_matches!( + v, + Ok(ValidationResult::Invalid(InvalidCandidate::CodeDecompressionFailure)) + ); +} + +#[test] +fn pov_decompression_failure_is_invalid() { + let validation_data = PersistedValidationData { + max_pov_size: POV_BOMB_LIMIT as u32, + ..Default::default() + }; + let head_data = HeadData(vec![1, 1, 1]); + + let raw_block_data = vec![2u8; POV_BOMB_LIMIT + 1]; + let pov = sp_maybe_compressed_blob::compress( + &raw_block_data, + POV_BOMB_LIMIT + 1, + ) + .map(|raw| PoV { block_data: BlockData(raw) }) + .unwrap(); + + let validation_code = ValidationCode(vec![2; 16]); + + let mut descriptor = CandidateDescriptor::default(); + descriptor.pov_hash = pov.hash(); + descriptor.para_head = head_data.hash(); + descriptor.validation_code_hash = validation_code.hash(); + collator_sign(&mut descriptor, Sr25519Keyring::Alice); + + let validation_result = WasmValidationResult { + head_data, + new_validation_code: None, + upward_messages: Vec::new(), + horizontal_messages: Vec::new(), + processed_downward_messages: 0, + hrmp_watermark: 0, + }; + + let v = executor::block_on(validate_candidate_exhaustive( + MockValidatorBackend::with_hardcoded_result(Ok(validation_result)), + validation_data, + validation_code, + descriptor, + Arc::new(pov), + &Default::default(), + )) + .unwrap(); + + assert_matches!( + v, + Ok(ValidationResult::Invalid(InvalidCandidate::PoVDecompressionFailure)) + ); +} diff --git a/node/core/chain-api/Cargo.toml b/node/core/chain-api/Cargo.toml index c146eab4222b..8f9ec9f8b33f 100644 --- a/node/core/chain-api/Cargo.toml +++ b/node/core/chain-api/Cargo.toml @@ -5,16 +5,19 @@ authors = ["Parity Technologies "] edition = "2018" [dependencies] -futures = "0.3.12" +futures = "0.3.15" tracing = "0.1.26" sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" } polkadot-primitives = { path = "../../../primitives" } -polkadot-overseer = { path = "../../overseer" } polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } +sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master" } +sc-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" } [dev-dependencies] -futures = { version = "0.3.12", features = ["thread-pool"] } +futures = { version = "0.3.15", features = ["thread-pool"] } maplit = "1.0.2" +parity-scale-codec = "2.0.0" +polkadot-node-primitives = { path = "../../primitives" } polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index ae67222e5a1e..b7c152686afa 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -23,6 +23,7 @@ //! Supported requests: //! * Block hash to number //! * Block hash to header +//! * Block weight (cumulative) //! * Finalized block number to hash //! * Last finalized block number //! * Ancestors @@ -30,22 +31,18 @@ #![deny(unused_crate_dependencies, unused_results)] #![warn(missing_docs)] -use polkadot_overseer::gen::{ - FromOverseer, SpawnedSubsystem, Subsystem, SubsystemContext, -}; -use polkadot_subsystem::{ - errors::{SubsystemResult, SubsystemError}, - OverseerSignal, - messages::ChainApiMessage, -}; -use polkadot_node_subsystem_util::{ - metrics::{self, prometheus}, -}; -use polkadot_primitives::v1::{Block, BlockId}; -use sp_blockchain::HeaderBackend; use std::sync::Arc; use futures::prelude::*; +use sc_client_api::AuxStore; +use sp_blockchain::HeaderBackend; + +use polkadot_node_subsystem_util::metrics::{self, prometheus}; +use polkadot_primitives::v1::{Block, BlockId}; +use polkadot_subsystem::{ + messages::ChainApiMessage, FromOverseer, OverseerSignal, SpawnedSubsystem, Subsystem, + SubsystemContext, SubsystemError, SubsystemResult, +}; const LOG_TARGET: &str = "parachain::chain-api"; @@ -65,11 +62,12 @@ impl ChainApiSubsystem { } } -impl Subsystem for ChainApiSubsystem where - Client: HeaderBackend + 'static, - Context: SubsystemContext, +impl Subsystem for ChainApiSubsystem +where + Client: HeaderBackend + AuxStore + 'static, + Context: SubsystemContext, { - fn start(self, ctx: Context) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run(ctx, self) .map_err(|e| SubsystemError::with_origin("chain-api", e)) .boxed(); @@ -80,13 +78,12 @@ impl Subsystem for ChainApiSubsystem( - mut ctx: impl SubsystemContext, + mut ctx: impl SubsystemContext, subsystem: ChainApiSubsystem, ) -> SubsystemResult<()> where - Client: HeaderBackend, + Client: HeaderBackend + AuxStore, { loop { match ctx.recv().await? { @@ -108,6 +105,13 @@ where subsystem.metrics.on_request(result.is_ok()); let _ = response_channel.send(result); }, + ChainApiMessage::BlockWeight(hash, response_channel) => { + let _timer = subsystem.metrics.time_block_weight(); + let result = sc_consensus_babe::block_weight(&*subsystem.client, hash) + .map_err(|e| e.to_string().into()); + subsystem.metrics.on_request(result.is_ok()); + let _ = response_channel.send(result); + } ChainApiMessage::FinalizedBlockHash(number, response_channel) => { let _timer = subsystem.metrics.time_finalized_block_hash(); // Note: we don't verify it's finalized @@ -164,6 +168,7 @@ struct MetricsInner { chain_api_requests: prometheus::CounterVec, block_number: prometheus::Histogram, block_header: prometheus::Histogram, + block_weight: prometheus::Histogram, finalized_block_hash: prometheus::Histogram, finalized_block_number: prometheus::Histogram, ancestors: prometheus::Histogram, @@ -194,6 +199,11 @@ impl Metrics { self.0.as_ref().map(|metrics| metrics.block_header.start_timer()) } + /// Provide a timer for `block_weight` which observes on drop. + fn time_block_weight(&self) -> Option { + self.0.as_ref().map(|metrics| metrics.block_weight.start_timer()) + } + /// Provide a timer for `finalized_block_hash` which observes on drop. fn time_finalized_block_hash(&self) -> Option { self.0.as_ref().map(|metrics| metrics.finalized_block_hash.start_timer()) @@ -241,6 +251,15 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + block_weight: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "parachain_chain_api_block_weight", + "Time spent within `chain_api::block_weight`", + ) + )?, + registry, + )?, finalized_block_hash: prometheus::register( prometheus::Histogram::with_opts( prometheus::HistogramOpts::new( @@ -273,15 +292,16 @@ impl metrics::Metrics for Metrics { } } - #[cfg(test)] mod tests { use super::*; use std::collections::BTreeMap; use futures::{future::BoxFuture, channel::oneshot}; + use parity_scale_codec::Encode; use polkadot_primitives::v1::{Hash, BlockNumber, BlockId, Header}; + use polkadot_node_primitives::BlockWeight; use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle}; use sp_blockchain::Info as BlockInfo; use sp_core::testing::TaskExecutor; @@ -289,6 +309,7 @@ mod tests { #[derive(Clone)] struct TestClient { blocks: BTreeMap, + block_weights: BTreeMap, finalized_blocks: BTreeMap, headers: BTreeMap, } @@ -318,6 +339,12 @@ mod tests { THREE => 3, FOUR => 4, }, + block_weights: maplit::btreemap! { + ONE => 0, + TWO => 1, + THREE => 1, + FOUR => 2, + }, finalized_blocks: maplit::btreemap! { 1 => ONE, 3 => THREE, @@ -341,7 +368,7 @@ mod tests { ERROR_PATH => Header { ..default_header() } - } + }, } } } @@ -367,6 +394,7 @@ mod tests { finalized_hash, finalized_number, number_leaves: 0, + finalized_state: None, } } fn number(&self, hash: Hash) -> sp_blockchain::Result> { @@ -406,6 +434,30 @@ mod tests { futures::executor::block_on(future::join(chain_api_task, test_task)); } + impl AuxStore for TestClient { + fn insert_aux< + 'a, + 'b: 'a, + 'c: 'a, + I: IntoIterator, + D: IntoIterator, + >( + &self, + _insert: I, + _delete: D, + ) -> sp_blockchain::Result<()> { + unimplemented!() + } + + fn get_aux(&self, key: &[u8]) -> sp_blockchain::Result>> { + Ok(self + .block_weights + .iter() + .find(|(hash, _)| sc_consensus_babe::aux_schema::block_weight_key(hash) == key) + .map(|(_, weight)| weight.encode())) + } + } + #[test] fn request_block_number() { test_harness(|client, mut sender| { @@ -454,6 +506,31 @@ mod tests { }) } + #[test] + fn request_block_weight() { + test_harness(|client, mut sender| { + async move { + const NOT_HERE: Hash = Hash::repeat_byte(0x5); + let test_cases = [ + (TWO, sc_consensus_babe::block_weight(&*client, TWO).unwrap()), + (FOUR, sc_consensus_babe::block_weight(&*client, FOUR).unwrap()), + (NOT_HERE, sc_consensus_babe::block_weight(&*client, NOT_HERE).unwrap()), + ]; + for (hash, expected) in &test_cases { + let (tx, rx) = oneshot::channel(); + + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::BlockWeight(*hash, tx), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), *expected); + } + + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) + } + #[test] fn request_finalized_hash() { test_harness(|client, mut sender| { diff --git a/node/core/chain-selection/Cargo.toml b/node/core/chain-selection/Cargo.toml new file mode 100644 index 000000000000..ee498427ea0d --- /dev/null +++ b/node/core/chain-selection/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "polkadot-node-core-chain-selection" +description = "Chain Selection Subsystem" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" + +[dependencies] +futures = "0.3.15" +tracing = "0.1.26" +polkadot-primitives = { path = "../../../primitives" } +polkadot-node-primitives = { path = "../../primitives" } +polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } +polkadot-node-subsystem-util = { path = "../../subsystem-util" } +kvdb = "0.9.0" +thiserror = "1.0.23" +parity-scale-codec = "2" + +[dev-dependencies] +polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } +sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } +parking_lot = "0.11" +assert_matches = "1" diff --git a/node/core/chain-selection/src/backend.rs b/node/core/chain-selection/src/backend.rs new file mode 100644 index 000000000000..160825b757e7 --- /dev/null +++ b/node/core/chain-selection/src/backend.rs @@ -0,0 +1,235 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! An abstraction over storage used by the chain selection subsystem. +//! +//! This provides both a [`Backend`] trait and an [`OverlayedBackend`] +//! struct which allows in-memory changes to be applied on top of a +//! [`Backend`], maintaining consistency between queries and temporary writes, +//! before any commit to the underlying storage is made. + +use polkadot_primitives::v1::{BlockNumber, Hash}; + +use std::collections::HashMap; + +use crate::{Error, LeafEntrySet, BlockEntry, Timestamp}; + +pub(super) enum BackendWriteOp { + WriteBlockEntry(BlockEntry), + WriteBlocksByNumber(BlockNumber, Vec), + WriteViableLeaves(LeafEntrySet), + WriteStagnantAt(Timestamp, Vec), + DeleteBlocksByNumber(BlockNumber), + DeleteBlockEntry(Hash), + DeleteStagnantAt(Timestamp), +} + +/// An abstraction over backend storage for the logic of this subsystem. +pub(super) trait Backend { + /// Load a block entry from the DB. + fn load_block_entry(&self, hash: &Hash) -> Result, Error>; + /// Load the active-leaves set. + fn load_leaves(&self) -> Result; + /// Load the stagnant list at the given timestamp. + fn load_stagnant_at(&self, timestamp: Timestamp) -> Result, Error>; + /// Load all stagnant lists up to and including the given unix timestamp + /// in ascending order. + fn load_stagnant_at_up_to(&self, up_to: Timestamp) + -> Result)>, Error>; + /// Load the earliest kept block number. + fn load_first_block_number(&self) -> Result, Error>; + /// Load blocks by number. + fn load_blocks_by_number(&self, number: BlockNumber) -> Result, Error>; + + /// Atomically write the list of operations, with later operations taking precedence over prior. + fn write(&mut self, ops: I) -> Result<(), Error> + where I: IntoIterator; +} + +/// An in-memory overlay over the backend. +/// +/// This maintains read-only access to the underlying backend, but can be +/// converted into a set of write operations which will, when written to +/// the underlying backend, give the same view as the state of the overlay. +pub(super) struct OverlayedBackend<'a, B: 'a> { + inner: &'a B, + + // `None` means 'deleted', missing means query inner. + block_entries: HashMap>, + // `None` means 'deleted', missing means query inner. + blocks_by_number: HashMap>>, + // 'None' means 'deleted', missing means query inner. + stagnant_at: HashMap>>, + // 'None' means query inner. + leaves: Option, +} + +impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> { + pub(super) fn new(backend: &'a B) -> Self { + OverlayedBackend { + inner: backend, + block_entries: HashMap::new(), + blocks_by_number: HashMap::new(), + stagnant_at: HashMap::new(), + leaves: None, + } + } + + pub(super) fn load_block_entry(&self, hash: &Hash) -> Result, Error> { + if let Some(val) = self.block_entries.get(&hash) { + return Ok(val.clone()) + } + + self.inner.load_block_entry(hash) + } + + pub(super) fn load_blocks_by_number(&self, number: BlockNumber) -> Result, Error> { + if let Some(val) = self.blocks_by_number.get(&number) { + return Ok(val.as_ref().map_or(Vec::new(), Clone::clone)); + } + + self.inner.load_blocks_by_number(number) + } + + pub(super) fn load_leaves(&self) -> Result { + if let Some(ref set) = self.leaves { + return Ok(set.clone()) + } + + self.inner.load_leaves() + } + + pub(super) fn load_stagnant_at(&self, timestamp: Timestamp) -> Result, Error> { + if let Some(val) = self.stagnant_at.get(×tamp) { + return Ok(val.as_ref().map_or(Vec::new(), Clone::clone)); + } + + self.inner.load_stagnant_at(timestamp) + } + + pub(super) fn write_block_entry(&mut self, entry: BlockEntry) { + self.block_entries.insert(entry.block_hash, Some(entry)); + } + + pub(super) fn delete_block_entry(&mut self, hash: &Hash) { + self.block_entries.insert(*hash, None); + } + + pub(super) fn write_blocks_by_number(&mut self, number: BlockNumber, blocks: Vec) { + if blocks.is_empty() { + self.blocks_by_number.insert(number, None); + } else { + self.blocks_by_number.insert(number, Some(blocks)); + } + } + + pub(super) fn delete_blocks_by_number(&mut self, number: BlockNumber) { + self.blocks_by_number.insert(number, None); + } + + pub(super) fn write_leaves(&mut self, leaves: LeafEntrySet) { + self.leaves = Some(leaves); + } + + pub(super) fn write_stagnant_at(&mut self, timestamp: Timestamp, hashes: Vec) { + self.stagnant_at.insert(timestamp, Some(hashes)); + } + + pub(super) fn delete_stagnant_at(&mut self, timestamp: Timestamp) { + self.stagnant_at.insert(timestamp, None); + } + + /// Transform this backend into a set of write-ops to be written to the + /// inner backend. + pub(super) fn into_write_ops(self) -> impl Iterator { + let block_entry_ops = self.block_entries.into_iter().map(|(h, v)| match v { + Some(v) => BackendWriteOp::WriteBlockEntry(v), + None => BackendWriteOp::DeleteBlockEntry(h), + }); + + let blocks_by_number_ops = self.blocks_by_number.into_iter().map(|(n, v)| match v { + Some(v) => BackendWriteOp::WriteBlocksByNumber(n, v), + None => BackendWriteOp::DeleteBlocksByNumber(n), + }); + + let leaf_ops = self.leaves.into_iter().map(BackendWriteOp::WriteViableLeaves); + + let stagnant_at_ops = self.stagnant_at.into_iter().map(|(n, v)| match v { + Some(v) => BackendWriteOp::WriteStagnantAt(n, v), + None => BackendWriteOp::DeleteStagnantAt(n), + }); + + block_entry_ops + .chain(blocks_by_number_ops) + .chain(leaf_ops) + .chain(stagnant_at_ops) + } +} + +/// Attempt to find the given ancestor in the chain with given head. +/// +/// If the ancestor is the most recently finalized block, and the `head` is +/// a known unfinalized block, this will return `true`. +/// +/// If the ancestor is an unfinalized block and `head` is known, this will +/// return true if `ancestor` is in `head`'s chain. +/// +/// If the ancestor is an older finalized block, this will return `false`. +fn contains_ancestor( + backend: &impl Backend, + head: Hash, + ancestor: Hash, +) -> Result { + let mut current_hash = head; + loop { + if current_hash == ancestor { return Ok(true) } + match backend.load_block_entry(¤t_hash)? { + Some(e) => { current_hash = e.parent_hash } + None => break + } + } + + Ok(false) +} + +/// This returns the best unfinalized leaf containing the required block. +/// +/// If the required block is finalized but not the most recent finalized block, +/// this will return `None`. +/// +/// If the required block is unfinalized but not an ancestor of any viable leaf, +/// this will return `None`. +// +// Note: this is O(N^2) in the depth of `required` and the number of leaves. +// We expect the number of unfinalized blocks to be small, as in, to not exceed +// single digits in practice, and exceedingly unlikely to surpass 1000. +// +// However, if we need to, we could implement some type of skip-list for +// fast ancestry checks. +pub(super) fn find_best_leaf_containing( + backend: &impl Backend, + required: Hash, +) -> Result, Error> { + let leaves = backend.load_leaves()?; + for leaf in leaves.into_hashes_descending() { + if contains_ancestor(backend, leaf, required)? { + return Ok(Some(leaf)) + } + } + + // If there are no viable leaves containing the ancestor + Ok(None) +} diff --git a/node/core/chain-selection/src/lib.rs b/node/core/chain-selection/src/lib.rs new file mode 100644 index 000000000000..dddfc2590d33 --- /dev/null +++ b/node/core/chain-selection/src/lib.rs @@ -0,0 +1,574 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Implements the Chain Selection Subsystem. + +use polkadot_primitives::v1::{BlockNumber, Hash, Header, ConsensusLog}; +use polkadot_node_primitives::BlockWeight; +use polkadot_subsystem::{ + Subsystem, SubsystemContext, SubsystemError, SpawnedSubsystem, + OverseerSignal, FromOverseer, + messages::{ChainSelectionMessage, ChainApiMessage}, + errors::ChainApiError, +}; + +use parity_scale_codec::Error as CodecError; +use futures::channel::oneshot; +use futures::prelude::*; + +use std::time::{UNIX_EPOCH, SystemTime}; + +use crate::backend::{Backend, OverlayedBackend, BackendWriteOp}; + +mod backend; +mod tree; + +#[cfg(test)] +mod tests; + +const LOG_TARGET: &str = "parachain::chain-selection"; +/// Timestamp based on the 1 Jan 1970 UNIX base, which is persistent across node restarts and OS reboots. +type Timestamp = u64; + +#[derive(Debug, Clone)] +enum Approval { + // Approved + Approved, + // Unapproved but not stagnant + Unapproved, + // Unapproved and stagnant. + Stagnant, +} + +impl Approval { + fn is_stagnant(&self) -> bool { + matches!(*self, Approval::Stagnant) + } +} + +#[derive(Debug, Clone)] +struct ViabilityCriteria { + // Whether this block has been explicitly reverted by one of its descendants. + explicitly_reverted: bool, + // The approval state of this block specifically. + approval: Approval, + // The earliest unviable ancestor - the hash of the earliest unfinalized + // block in the ancestry which is explicitly reverted or stagnant. + earliest_unviable_ancestor: Option, +} + +impl ViabilityCriteria { + fn is_viable(&self) -> bool { + self.is_parent_viable() && self.is_explicitly_viable() + } + + // Whether the current block is explicitly viable. + // That is, whether the current block is neither reverted nor stagnant. + fn is_explicitly_viable(&self) -> bool { + !self.explicitly_reverted && !self.approval.is_stagnant() + } + + // Whether the parent is viable. This assumes that the parent + // descends from the finalized chain. + fn is_parent_viable(&self) -> bool { + self.earliest_unviable_ancestor.is_none() + } +} + +// Light entries describing leaves of the chain. +// +// These are ordered first by weight and then by block number. +#[derive(Debug, Clone, PartialEq)] +struct LeafEntry { + weight: BlockWeight, + block_number: BlockNumber, + block_hash: Hash, +} + +impl PartialOrd for LeafEntry { + fn partial_cmp(&self, other: &Self) -> Option { + let ord = self.weight.cmp(&other.weight) + .then(self.block_number.cmp(&other.block_number)); + + if !matches!(ord, std::cmp::Ordering::Equal) { Some(ord) } else { None } + } +} + +#[derive(Debug, Default, Clone)] +struct LeafEntrySet { + inner: Vec +} + +impl LeafEntrySet { + fn remove(&mut self, hash: &Hash) -> bool { + match self.inner.iter().position(|e| &e.block_hash == hash) { + None => false, + Some(i) => { + self.inner.remove(i); + true + } + } + } + + fn insert(&mut self, new: LeafEntry) { + let mut pos = None; + for (i, e) in self.inner.iter().enumerate() { + if e == &new { return } + if e < &new { + pos = Some(i); + break + } + } + + match pos { + None => self.inner.push(new), + Some(i) => self.inner.insert(i, new), + } + } + + fn into_hashes_descending(self) -> impl Iterator { + self.inner.into_iter().map(|e| e.block_hash) + } +} + +#[derive(Debug, Clone)] +struct BlockEntry { + block_hash: Hash, + block_number: BlockNumber, + parent_hash: Hash, + children: Vec, + viability: ViabilityCriteria, + weight: BlockWeight, +} + +impl BlockEntry { + fn leaf_entry(&self) -> LeafEntry { + LeafEntry { + block_hash: self.block_hash, + block_number: self.block_number, + weight: self.weight, + } + } + + fn non_viable_ancestor_for_child(&self) -> Option { + if self.viability.is_viable() { + None + } else { + self.viability.earliest_unviable_ancestor.or(Some(self.block_hash)) + } + } +} + +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] +pub enum Error { + #[error(transparent)] + ChainApi(#[from] ChainApiError), + + #[error(transparent)] + Io(#[from] std::io::Error), + + #[error(transparent)] + Oneshot(#[from] oneshot::Canceled), + + #[error(transparent)] + Subsystem(#[from] SubsystemError), + + #[error(transparent)] + Codec(#[from] CodecError), +} + +impl Error { + fn trace(&self) { + match self { + // don't spam the log with spurious errors + Self::Oneshot(_) => tracing::debug!(target: LOG_TARGET, err = ?self), + // it's worth reporting otherwise + _ => tracing::warn!(target: LOG_TARGET, err = ?self), + } + } +} + +fn timestamp_now() -> Timestamp { + // `SystemTime` is notoriously non-monotonic, so our timers might not work + // exactly as expected. Regardless, stagnation is detected on the order of minutes, + // and slippage of a few seconds in either direction won't cause any major harm. + // + // The exact time that a block becomes stagnant in the local node is always expected + // to differ from other nodes due to network asynchrony and delays in block propagation. + // Non-monotonicity exarcerbates that somewhat, but not meaningfully. + + match SystemTime::now().duration_since(UNIX_EPOCH) { + Ok(d) => d.as_secs(), + Err(e) => { + tracing::warn!( + target: LOG_TARGET, + err = ?e, + "Current time is before unix epoch. Validation will not work correctly." + ); + + 0 + } + } +} + +fn stagnant_timeout_from_now() -> Timestamp { + // If a block isn't approved in 120 seconds, nodes will abandon it + // and begin building on another chain. + const STAGNANT_TIMEOUT: Timestamp = 120; + + timestamp_now() + STAGNANT_TIMEOUT +} + +// TODO https://github.com/paritytech/polkadot/issues/3293: +// +// This is used just so we can have a public function that calls +// `run` and eliminates all the unused errors. +// +// Should be removed when the real implementation is done. +struct VoidBackend; + +impl Backend for VoidBackend { + fn load_block_entry(&self, _: &Hash) -> Result, Error> { + Ok(None) + } + fn load_leaves(&self) -> Result { + Ok(LeafEntrySet::default()) + } + fn load_stagnant_at(&self, _: Timestamp) -> Result, Error> { + Ok(Vec::new()) + } + fn load_stagnant_at_up_to(&self, _: Timestamp) + -> Result)>, Error> + { + Ok(Vec::new()) + } + fn load_first_block_number(&self) -> Result, Error> { + Ok(None) + } + fn load_blocks_by_number(&self, _: BlockNumber) -> Result, Error> { + Ok(Vec::new()) + } + + fn write(&mut self, _: I) -> Result<(), Error> + where I: IntoIterator + { + Ok(()) + } +} + +/// The chain selection subsystem. +pub struct ChainSelectionSubsystem; + +impl Subsystem for ChainSelectionSubsystem + where Context: SubsystemContext +{ + fn start(self, ctx: Context) -> SpawnedSubsystem { + let backend = VoidBackend; + SpawnedSubsystem { + future: run(ctx, backend).map(|()| Ok(())).boxed(), + name: "chain-selection-subsystem", + } + } +} + +async fn run(mut ctx: Context, mut backend: B) + where + Context: SubsystemContext, + B: Backend, +{ + loop { + let res = run_iteration(&mut ctx, &mut backend).await; + match res { + Err(e) => { + e.trace(); + + if let Error::Subsystem(SubsystemError::Context(_)) = e { + break; + } + } + Ok(()) => { + tracing::info!(target: LOG_TARGET, "received `Conclude` signal, exiting"); + break; + } + } + } +} + +// Run the subsystem until an error is encountered or a `conclude` signal is received. +// Most errors are non-fatal and should lead to another call to this function. +// +// A return value of `Ok` indicates that an exit should be made, while non-fatal errors +// lead to another call to this function. +async fn run_iteration(ctx: &mut Context, backend: &mut B) + -> Result<(), Error> + where + Context: SubsystemContext, + B: Backend, +{ + // TODO https://github.com/paritytech/polkadot/issues/3293: Add stagnant checking timer loop. + loop { + match ctx.recv().await? { + FromOverseer::Signal(OverseerSignal::Conclude) => { + return Ok(()) + } + FromOverseer::Signal(OverseerSignal::ActiveLeaves(update)) => { + for leaf in update.activated { + let write_ops = handle_active_leaf( + ctx, + &*backend, + leaf.hash, + ).await?; + + backend.write(write_ops)?; + } + } + FromOverseer::Signal(OverseerSignal::BlockFinalized(h, n)) => { + handle_finalized_block(backend, h, n)? + } + FromOverseer::Communication { msg } => match msg { + ChainSelectionMessage::Approved(hash) => { + handle_approved_block(backend, hash)? + } + ChainSelectionMessage::Leaves(tx) => { + let leaves = load_leaves(ctx, &*backend).await?; + let _ = tx.send(leaves); + } + ChainSelectionMessage::BestLeafContaining(required, tx) => { + let best_containing = crate::backend::find_best_leaf_containing( + &*backend, + required, + )?; + + // note - this may be none if the finalized block is + // a leaf. this is fine according to the expected usage of the + // function. `None` responses should just `unwrap_or(required)`, + // so if the required block is the finalized block, then voilá. + + let _ = tx.send(best_containing); + } + } + }; + } +} + +async fn fetch_finalized( + ctx: &mut impl SubsystemContext, +) -> Result, Error> { + let (number_tx, number_rx) = oneshot::channel(); + let (hash_tx, hash_rx) = oneshot::channel(); + + ctx.send_message(ChainApiMessage::FinalizedBlockNumber(number_tx).into()).await; + + let number = number_rx.await??; + + ctx.send_message(ChainApiMessage::FinalizedBlockHash(number, hash_tx).into()).await; + + match hash_rx.await?? { + None => { + tracing::warn!( + target: LOG_TARGET, + number, + "Missing hash for finalized block number" + ); + + return Ok(None) + } + Some(h) => Ok(Some((h, number))) + } +} + +async fn fetch_header( + ctx: &mut impl SubsystemContext, + hash: Hash, +) -> Result, Error> { + let (h_tx, h_rx) = oneshot::channel(); + ctx.send_message(ChainApiMessage::BlockHeader(hash, h_tx).into()).await; + + h_rx.await?.map_err(Into::into) +} + +async fn fetch_block_weight( + ctx: &mut impl SubsystemContext, + hash: Hash, +) -> Result, Error> { + let (tx, rx) = oneshot::channel(); + ctx.send_message(ChainApiMessage::BlockWeight(hash, tx).into()).await; + + rx.await?.map_err(Into::into) +} + +// Handle a new active leaf. +async fn handle_active_leaf( + ctx: &mut impl SubsystemContext, + backend: &impl Backend, + hash: Hash, +) -> Result, Error> { + let lower_bound = match backend.load_first_block_number()? { + Some(l) => { + // We want to iterate back to finalized, and first block number + // is assumed to be 1 above finalized - the implicit root of the + // tree. + l.saturating_sub(1) + }, + None => fetch_finalized(ctx).await?.map_or(1, |(_, n)| n), + }; + + let header = match fetch_header(ctx, hash).await? { + None => { + tracing::warn!( + target: LOG_TARGET, + ?hash, + "Missing header for new head", + ); + return Ok(Vec::new()) + } + Some(h) => h, + }; + + let new_blocks = polkadot_node_subsystem_util::determine_new_blocks( + ctx.sender(), + |h| backend.load_block_entry(h).map(|b| b.is_some()), + hash, + &header, + lower_bound, + ).await?; + + let mut overlay = OverlayedBackend::new(backend); + + // determine_new_blocks gives blocks in descending order. + // for this, we want ascending order. + for (hash, header) in new_blocks.into_iter().rev() { + let weight = match fetch_block_weight(ctx, hash).await? { + None => { + tracing::warn!( + target: LOG_TARGET, + ?hash, + "Missing block weight for new head. Skipping chain.", + ); + + // If we don't know the weight, we can't import the block. + // And none of its descendents either. + break; + } + Some(w) => w, + }; + + let reversion_logs = extract_reversion_logs(&header); + crate::tree::import_block( + &mut overlay, + hash, + header.number, + header.parent_hash, + reversion_logs, + weight, + )?; + } + + Ok(overlay.into_write_ops().collect()) +} + +// Extract all reversion logs from a header in ascending order. +// +// Ignores logs with number >= the block header number. +fn extract_reversion_logs(header: &Header) -> Vec { + let number = header.number; + let mut logs = header.digest.logs() + .iter() + .enumerate() + .filter_map(|(i, d)| match ConsensusLog::from_digest_item(d) { + Err(e) => { + tracing::warn!( + target: LOG_TARGET, + err = ?e, + index = i, + block_hash = ?header.hash(), + "Digest item failed to encode" + ); + + None + } + Ok(Some(ConsensusLog::Revert(b))) if b < number => Some(b), + Ok(Some(ConsensusLog::Revert(b))) => { + tracing::warn!( + target: LOG_TARGET, + revert_target = b, + block_number = number, + block_hash = ?header.hash(), + "Block issued invalid revert digest targeting itself or future" + ); + + None + } + Ok(_) => None, + }) + .collect::>(); + + logs.sort(); + + logs +} + +// Handle a finalized block event. +fn handle_finalized_block( + backend: &mut impl Backend, + finalized_hash: Hash, + finalized_number: BlockNumber, +) -> Result<(), Error> { + let ops = crate::tree::finalize_block( + &*backend, + finalized_hash, + finalized_number, + )?.into_write_ops(); + + backend.write(ops) +} + +// Handle an approved block event. +fn handle_approved_block( + backend: &mut impl Backend, + approved_block: Hash, +) -> Result<(), Error> { + let ops = { + let mut overlay = OverlayedBackend::new(&*backend); + + crate::tree::approve_block( + &mut overlay, + approved_block, + )?; + + overlay.into_write_ops() + }; + + backend.write(ops) +} + +// Load the leaves from the backend. If there are no leaves, then return +// the finalized block. +async fn load_leaves( + ctx: &mut impl SubsystemContext, + backend: &impl Backend, +) -> Result, Error> { + let leaves: Vec<_> = backend.load_leaves()? + .into_hashes_descending() + .collect(); + + if leaves.is_empty() { + Ok(fetch_finalized(ctx).await?.map_or(Vec::new(), |(h, _)| vec![h])) + } else { + Ok(leaves) + } +} diff --git a/node/core/chain-selection/src/tests.rs b/node/core/chain-selection/src/tests.rs new file mode 100644 index 000000000000..945578a47e6e --- /dev/null +++ b/node/core/chain-selection/src/tests.rs @@ -0,0 +1,1909 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Tests for the subsystem. +//! +//! These primarily revolve around having a backend which is shared between +//! both the test code and the tested subsystem, and which also gives the +//! test code the ability to wait for write operations to occur. + +use super::*; +use std::collections::{HashMap, HashSet, BTreeMap}; +use std::sync::Arc; + +use futures::channel::oneshot; +use parity_scale_codec::Encode; +use parking_lot::Mutex; +use sp_core::testing::TaskExecutor; +use assert_matches::assert_matches; + +use polkadot_primitives::v1::{BlakeTwo256, HashT, ConsensusLog}; +use polkadot_subsystem::{jaeger, ActiveLeavesUpdate, ActivatedLeaf, LeafStatus}; +use polkadot_subsystem::messages::AllMessages; +use polkadot_node_subsystem_test_helpers as test_helpers; + +#[derive(Default)] +struct TestBackendInner { + leaves: LeafEntrySet, + block_entries: HashMap, + blocks_by_number: BTreeMap>, + stagnant_at: BTreeMap>, + // earlier wakers at the back. + write_wakers: Vec>, +} + +#[derive(Clone)] +struct TestBackend { + inner: Arc>, +} + +impl TestBackend { + // Yields a receiver which will be woken up on some future write + // to the backend along with its position (starting at 0) in the + // queue. + // + // Our tests assume that there is only one task calling this function + // and the index is useful to get a waker that will trigger after + // some known amount of writes to the backend that happen internally + // inside the subsystem. + // + // It's important to call this function at points where no writes + // are pending to the backend. This requires knowing some details + // about the internals of the subsystem, so the abstraction leaks + // somewhat, but this is acceptable enough. + fn await_next_write(&self) -> (usize, oneshot::Receiver<()>) { + let (tx, rx) = oneshot::channel(); + + let mut inner = self.inner.lock(); + let pos = inner.write_wakers.len(); + inner.write_wakers.insert(0, tx); + + (pos, rx) + } + + // Assert the backend contains only the given blocks and no others. + // This does not check the stagnant_at mapping because that is + // pruned lazily by the subsystem as opposed to eagerly. + fn assert_contains_only( + &self, + blocks: Vec<(BlockNumber, Hash)>, + ) { + let hashes: Vec<_> = blocks.iter().map(|(_, h)| *h).collect(); + let mut by_number: HashMap<_, HashSet<_>> = HashMap::new(); + + for (number, hash) in blocks { + by_number.entry(number).or_default().insert(hash); + } + + let inner = self.inner.lock(); + assert_eq!(inner.block_entries.len(), hashes.len()); + assert_eq!(inner.blocks_by_number.len(), by_number.len()); + + for leaf in inner.leaves.clone().into_hashes_descending() { + assert!(hashes.contains(&leaf)); + } + + for (number, hashes_at_number) in by_number { + let at = inner.blocks_by_number.get(&number).unwrap(); + for hash in at { + assert!(hashes_at_number.contains(&hash)); + } + } + } +} + +impl Default for TestBackend { + fn default() -> Self { + TestBackend { + inner: Default::default(), + } + } +} + +impl Backend for TestBackend { + fn load_block_entry(&self, hash: &Hash) -> Result, Error> { + Ok(self.inner.lock().block_entries.get(hash).map(|e| e.clone())) + } + fn load_leaves(&self) -> Result { + Ok(self.inner.lock().leaves.clone()) + } + fn load_stagnant_at(&self, timestamp: Timestamp) -> Result, Error> { + Ok(self.inner.lock().stagnant_at.get(×tamp).map_or(Vec::new(), |s| s.clone())) + } + fn load_stagnant_at_up_to(&self, up_to: Timestamp) + -> Result)>, Error> + { + Ok(self.inner.lock().stagnant_at.range(..=up_to).map(|(t, v)| (*t, v.clone())).collect()) + } + fn load_first_block_number(&self) -> Result, Error> { + Ok(self.inner.lock().blocks_by_number.range(..).map(|(k, _)| *k).next()) + } + fn load_blocks_by_number(&self, number: BlockNumber) -> Result, Error> { + Ok(self.inner.lock().blocks_by_number.get(&number).map_or(Vec::new(), |v| v.clone())) + } + + fn write(&mut self, ops: I) -> Result<(), Error> + where I: IntoIterator + { + let mut inner = self.inner.lock(); + + for op in ops { + match op { + BackendWriteOp::WriteBlockEntry(entry) => { + inner.block_entries.insert(entry.block_hash, entry); + } + BackendWriteOp::WriteBlocksByNumber(number, hashes) => { + inner.blocks_by_number.insert(number, hashes); + } + BackendWriteOp::WriteViableLeaves(leaves) => { + inner.leaves = leaves; + } + BackendWriteOp::WriteStagnantAt(time, hashes) => { + inner.stagnant_at.insert(time, hashes); + } + BackendWriteOp::DeleteBlocksByNumber(number) => { + inner.blocks_by_number.remove(&number); + } + BackendWriteOp::DeleteBlockEntry(hash) => { + inner.block_entries.remove(&hash); + } + BackendWriteOp::DeleteStagnantAt(time) => { + inner.stagnant_at.remove(&time); + } + } + } + + if let Some(waker) = inner.write_wakers.pop() { + let _ = waker.send(()); + } + Ok(()) + } +} + +type VirtualOverseer = test_helpers::TestSubsystemContextHandle; + +fn test_harness>( + test: impl FnOnce(TestBackend, VirtualOverseer) -> T +) { + let pool = TaskExecutor::new(); + let (context, virtual_overseer) = test_helpers::make_subsystem_context(pool); + + let backend = TestBackend::default(); + let subsystem = crate::run(context, backend.clone()); + + let test_fut = test(backend, virtual_overseer); + let test_and_conclude = async move { + let mut virtual_overseer = test_fut.await; + virtual_overseer.send(OverseerSignal::Conclude.into()).await; + + // Ensure no messages are pending when the subsystem shuts down. + assert!(virtual_overseer.try_recv().await.is_none()); + }; + futures::executor::block_on(futures::future::join(subsystem, test_and_conclude)); +} + +// Answer requests from the subsystem about the finalized block. +async fn answer_finalized_block_info( + overseer: &mut VirtualOverseer, + finalized_number: BlockNumber, + finalized_hash: Hash, +) { + assert_matches!( + overseer.recv().await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(tx)) => { + let _ = tx.send(Ok(finalized_number)); + } + ); + + assert_matches!( + overseer.recv().await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockHash(n, tx)) => { + assert_eq!(n, finalized_number); + let _ = tx.send(Ok(Some(finalized_hash))); + } + ); +} + +async fn answer_header_request( + overseer: &mut VirtualOverseer, + maybe_header: impl Into>, +) { + assert_matches!( + overseer.recv().await, + AllMessages::ChainApi(ChainApiMessage::BlockHeader(hash, tx)) => { + let maybe_header = maybe_header.into(); + assert!(maybe_header.as_ref().map_or(true, |h| h.hash() == hash)); + let _ = tx.send(Ok(maybe_header)); + } + ) +} + +async fn answer_weight_request( + overseer: &mut VirtualOverseer, + hash: Hash, + weight: impl Into>, +) { + assert_matches!( + overseer.recv().await, + AllMessages::ChainApi(ChainApiMessage::BlockWeight(h, tx)) => { + assert_eq!(h, hash); + let _ = tx.send(Ok(weight.into())); + } + ) +} + +fn child_header(parent_number: BlockNumber, parent_hash: Hash) -> Header { + Header { + parent_hash, + number: parent_number + 1, + state_root: Default::default(), + extrinsics_root: Default::default(), + digest: Default::default() + } +} + +fn salt_header(header: &mut Header, salt: impl Encode) { + header.state_root = BlakeTwo256::hash_of(&salt) +} + +fn add_reversions( + header: &mut Header, + reversions: impl IntoIterator, +) { + for log in reversions.into_iter().map(ConsensusLog::Revert) { + header.digest.logs.push(log.into()) + } +} + +// Builds a chain on top of the given base, with one block for each +// provided weight. +fn construct_chain_on_base( + weights: impl IntoIterator, + base_number: BlockNumber, + base_hash: Hash, + mut mutate: impl FnMut(&mut Header), +) -> (Hash, Vec<(Header, BlockWeight)>) { + let mut parent_number = base_number; + let mut parent_hash = base_hash; + + let mut chain = Vec::new(); + for weight in weights { + let mut header = child_header(parent_number, parent_hash); + mutate(&mut header); + + parent_number = header.number; + parent_hash = header.hash(); + chain.push((header, weight)); + } + + (parent_hash, chain) +} + +// import blocks 1-by-1. If `finalized_base` is supplied, +// it will be answered before the first block in `answers. +async fn import_blocks_into( + virtual_overseer: &mut VirtualOverseer, + backend: &TestBackend, + mut finalized_base: Option<(BlockNumber, Hash)>, + blocks: Vec<(Header, BlockWeight)>, +) { + for (header, weight) in blocks { + let (_, write_rx) = backend.await_next_write(); + + let hash = header.hash(); + virtual_overseer.send(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work( + ActivatedLeaf { + hash, + number: header.number, + status: LeafStatus::Fresh, + span: Arc::new(jaeger::Span::Disabled), + } + )).into()).await; + + if let Some((f_n, f_h)) = finalized_base.take() { + answer_finalized_block_info(virtual_overseer, f_n, f_h).await; + } + + answer_header_request(virtual_overseer, header.clone()).await; + answer_weight_request(virtual_overseer, hash, weight).await; + + write_rx.await.unwrap(); + } +} + +async fn import_chains_into_empty( + virtual_overseer: &mut VirtualOverseer, + backend: &TestBackend, + finalized_number: BlockNumber, + finalized_hash: Hash, + chains: Vec>, +) { + for (i, chain)in chains.into_iter().enumerate() { + let finalized_base = Some((finalized_number, finalized_hash)).filter(|_| i == 0); + import_blocks_into( + virtual_overseer, + backend, + finalized_base, + chain, + ).await; + } +} + +// Import blocks all at once. This assumes that the ancestor is known/finalized +// but none of the other blocks. +// import blocks 1-by-1. If `finalized_base` is supplied, +// it will be answered before the first block. +// +// some pre-blocks may need to be supplied to answer ancestry requests +// that gather batches beyond the beginning of the new chain. +// pre-blocks are those already known by the subsystem, however, +// the subsystem has no way of knowin that until requesting ancestry. +async fn import_all_blocks_into( + virtual_overseer: &mut VirtualOverseer, + backend: &TestBackend, + finalized_base: Option<(BlockNumber, Hash)>, + pre_blocks: Vec
, + blocks: Vec<(Header, BlockWeight)>, +) { + assert!(blocks.len() > 1, "gap only makes sense if importing multiple blocks"); + + let head = blocks.last().unwrap().0.clone(); + let head_hash = head.hash(); + + let (_, write_rx) = backend.await_next_write(); + virtual_overseer.send(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work( + ActivatedLeaf { + hash: head_hash, + number: head.number, + status: LeafStatus::Fresh, + span: Arc::new(jaeger::Span::Disabled), + } + )).into()).await; + + if let Some((f_n, f_h)) = finalized_base { + answer_finalized_block_info(virtual_overseer, f_n, f_h).await; + } + + // Head is always fetched first. + answer_header_request(virtual_overseer, head).await; + + // Answer header and ancestry requests until the parent of head + // is imported. + { + let find_block_header = |expected_hash| { + pre_blocks.iter().cloned() + .chain(blocks.iter().map(|(h, _)| h.clone())) + .find(|hdr| hdr.hash() == expected_hash) + .unwrap() + }; + + let mut behind_head = 0; + loop { + let nth_ancestor_of_head = |n: usize| { + // blocks: [d, e, f, head] + // pre: [a, b, c] + // + // [a, b, c, d, e, f, head] + // [6, 5, 4, 3, 2, 1, 0] + + let new_ancestry_end = blocks.len() - 1; + if n > new_ancestry_end { + // [6, 5, 4] -> [2, 1, 0] + let n_in_pre = n - blocks.len(); + let pre_blocks_end = pre_blocks.len() - 1; + pre_blocks[pre_blocks_end - n_in_pre].clone() + } else { + let blocks_end = blocks.len() - 1; + blocks[blocks_end - n].0.clone() + } + }; + + match virtual_overseer.recv().await { + AllMessages::ChainApi(ChainApiMessage::Ancestors { + hash: h, + k, + response_channel: tx, + }) => { + let prev_response = nth_ancestor_of_head(behind_head); + assert_eq!(h, prev_response.hash()); + + let _ = tx.send(Ok( + (0..k as usize).map(|n| n + behind_head + 1) + .map(nth_ancestor_of_head) + .map(|h| h.hash()) + .collect() + )); + + for _ in 0..k { + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ChainApi(ChainApiMessage::BlockHeader(h, tx)) => { + let header = find_block_header(h); + let _ = tx.send(Ok(Some(header))); + } + ) + } + + behind_head = behind_head + k as usize; + } + AllMessages::ChainApi(ChainApiMessage::BlockHeader(h, tx)) => { + let header = find_block_header(h); + let _ = tx.send(Ok(Some(header))); + + // Assuming that `determine_new_blocks` uses these + // instead of ancestry: 1. + behind_head += 1; + } + AllMessages::ChainApi(ChainApiMessage::BlockWeight(h, tx)) => { + let (_, weight) = blocks.iter().find(|(hdr, _)| hdr.hash() == h).unwrap(); + let _ = tx.send(Ok(Some(*weight))); + + // Last weight has been returned. Time to go. + if h == head_hash { break } + } + _ => panic!("unexpected message"), + } + } + } + write_rx.await.unwrap(); +} + +async fn finalize_block( + virtual_overseer: &mut VirtualOverseer, + backend: &TestBackend, + block_number: BlockNumber, + block_hash: Hash, +) { + let (_, write_tx) = backend.await_next_write(); + + virtual_overseer.send( + OverseerSignal::BlockFinalized(block_hash, block_number).into() + ).await; + + write_tx.await.unwrap(); +} + +fn extract_info_from_chain(i: usize, chain: &[(Header, BlockWeight)]) + -> (BlockNumber, Hash, BlockWeight) +{ + let &(ref header, weight) = &chain[i]; + + (header.number, header.hash(), weight) +} + +fn assert_backend_contains<'a>( + backend: &TestBackend, + headers: impl IntoIterator, +) { + for header in headers { + let hash = header.hash(); + assert!( + backend.load_blocks_by_number(header.number).unwrap().contains(&hash), + "blocks at {} does not contain {}", + header.number, + hash, + ); + assert!( + backend.load_block_entry(&hash).unwrap().is_some(), + "no entry found for {}", + hash, + ); + } +} + +fn assert_backend_contains_chains( + backend: &TestBackend, + chains: Vec>, +) { + for chain in chains { + assert_backend_contains( + backend, + chain.iter().map(|&(ref hdr, _)| hdr) + ) + } +} + +fn assert_leaves( + backend: &TestBackend, + leaves: Vec, +) { + assert_eq!( + backend.load_leaves().unwrap().into_hashes_descending().into_iter().collect::>(), + leaves, + ); +} + +async fn assert_leaves_query( + virtual_overseer: &mut VirtualOverseer, + leaves: Vec, +) { + assert!(!leaves.is_empty(), "empty leaves impossible. answer finalized query"); + + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: ChainSelectionMessage::Leaves(tx) + }).await; + + assert_eq!(rx.await.unwrap(), leaves); +} + +async fn assert_finalized_leaves_query( + virtual_overseer: &mut VirtualOverseer, + finalized_number: BlockNumber, + finalized_hash: Hash, +) { + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: ChainSelectionMessage::Leaves(tx) + }).await; + + answer_finalized_block_info(virtual_overseer, finalized_number, finalized_hash).await; + + assert_eq!(rx.await.unwrap(), vec![finalized_hash]); +} + +async fn best_leaf_containing( + virtual_overseer: &mut VirtualOverseer, + required: Hash, +) -> Option { + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: ChainSelectionMessage::BestLeafContaining(required, tx) + }).await; + + rx.await.unwrap() +} + +async fn approve_block( + virtual_overseer: &mut VirtualOverseer, + backend: &TestBackend, + approved: Hash, +) { + let (_, write_rx) = backend.await_next_write(); + virtual_overseer.send(FromOverseer::Communication { + msg: ChainSelectionMessage::Approved(approved) + }).await; + + write_rx.await.unwrap() +} + +#[test] +fn no_op_subsystem_run() { + test_harness(|_, virtual_overseer| async move { virtual_overseer }); +} + +#[test] +fn import_direct_child_of_finalized_on_empty() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + let child = child_header(finalized_number, finalized_hash); + let child_hash = child.hash(); + let child_weight = 1; + let child_number = child.number; + + import_blocks_into( + &mut virtual_overseer, + &backend, + Some((finalized_number, finalized_hash)), + vec![(child.clone(), child_weight)], + ).await; + + assert_eq!(backend.load_first_block_number().unwrap().unwrap(), child_number); + assert_backend_contains(&backend, &[child]); + assert_leaves(&backend, vec![child_hash]); + assert_leaves_query(&mut virtual_overseer, vec![child_hash]).await; + + virtual_overseer + }) +} + +#[test] +fn import_chain_on_finalized_incrementally() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + let (head_hash, chain) = construct_chain_on_base( + vec![1, 2, 3, 4, 5], + finalized_number, + finalized_hash, + |_| {} + ); + + import_blocks_into( + &mut virtual_overseer, + &backend, + Some((finalized_number, finalized_hash)), + chain.clone(), + ).await; + + assert_eq!(backend.load_first_block_number().unwrap().unwrap(), 1); + assert_backend_contains(&backend, chain.iter().map(|&(ref h, _)| h)); + assert_leaves(&backend, vec![head_hash]); + assert_leaves_query(&mut virtual_overseer, vec![head_hash]).await; + + virtual_overseer + }) +} + +#[test] +fn import_two_subtrees_on_finalized() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + let (a_hash, chain_a) = construct_chain_on_base( + vec![1], + finalized_number, + finalized_hash, + |_| {} + ); + + let (b_hash, chain_b) = construct_chain_on_base( + vec![2], + finalized_number, + finalized_hash, + |h| salt_header(h, b"b"), + ); + + import_blocks_into( + &mut virtual_overseer, + &backend, + Some((finalized_number, finalized_hash)), + chain_a.clone(), + ).await; + + import_blocks_into( + &mut virtual_overseer, + &backend, + None, + chain_b.clone(), + ).await; + + assert_eq!(backend.load_first_block_number().unwrap().unwrap(), 1); + assert_backend_contains(&backend, chain_a.iter().map(|&(ref h, _)| h)); + assert_backend_contains(&backend, chain_b.iter().map(|&(ref h, _)| h)); + assert_leaves(&backend, vec![b_hash, a_hash]); + assert_leaves_query(&mut virtual_overseer, vec![b_hash, a_hash]).await; + + virtual_overseer + }) +} + +#[test] +fn import_two_subtrees_on_nonzero_finalized() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 100; + let finalized_hash = Hash::repeat_byte(0); + + let (a_hash, chain_a) = construct_chain_on_base( + vec![1], + finalized_number, + finalized_hash, + |_| {} + ); + + let (b_hash, chain_b) = construct_chain_on_base( + vec![2], + finalized_number, + finalized_hash, + |h| salt_header(h, b"b"), + ); + + import_blocks_into( + &mut virtual_overseer, + &backend, + Some((finalized_number, finalized_hash)), + chain_a.clone(), + ).await; + + import_blocks_into( + &mut virtual_overseer, + &backend, + None, + chain_b.clone(), + ).await; + + assert_eq!(backend.load_first_block_number().unwrap().unwrap(), 101); + assert_backend_contains(&backend, chain_a.iter().map(|&(ref h, _)| h)); + assert_backend_contains(&backend, chain_b.iter().map(|&(ref h, _)| h)); + assert_leaves(&backend, vec![b_hash, a_hash]); + assert_leaves_query(&mut virtual_overseer, vec![b_hash, a_hash]).await; + + virtual_overseer + }) +} + +#[test] +fn leaves_ordered_by_weight_and_then_number() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3 + // A1 <- B2 + // F <- C1 <- C2 + // + // expected_leaves: [(C2, 3), (A3, 2), (B2, 2)] + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 1, 2], + finalized_number, + finalized_hash, + |_| {} + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + + let (b2_hash, chain_b) = construct_chain_on_base( + vec![2], + 1, + a1_hash, + |h| salt_header(h, b"b"), + ); + + let (c2_hash, chain_c) = construct_chain_on_base( + vec![1, 3], + finalized_number, + finalized_hash, + |h| salt_header(h, b"c"), + ); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone(), chain_b.clone(), chain_c.clone()], + ).await; + + assert_eq!(backend.load_first_block_number().unwrap().unwrap(), 1); + assert_backend_contains(&backend, chain_a.iter().map(|&(ref h, _)| h)); + assert_backend_contains(&backend, chain_b.iter().map(|&(ref h, _)| h)); + assert_backend_contains(&backend, chain_c.iter().map(|&(ref h, _)| h)); + assert_leaves(&backend, vec![c2_hash, a3_hash, b2_hash]); + assert_leaves_query(&mut virtual_overseer, vec![c2_hash, a3_hash, b2_hash]).await; + virtual_overseer + }); +} + +#[test] +fn subtrees_imported_even_with_gaps() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3 + // A2 <- B3 <- B4 <- B5 + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 3], + finalized_number, + finalized_hash, + |_| {} + ); + + let (_, a2_hash, _) = extract_info_from_chain(1, &chain_a); + + let (b5_hash, chain_b) = construct_chain_on_base( + vec![4, 4, 5], + 2, + a2_hash, + |h| salt_header(h, b"b"), + ); + + import_all_blocks_into( + &mut virtual_overseer, + &backend, + Some((finalized_number, finalized_hash)), + Vec::new(), + chain_a.clone(), + ).await; + + import_all_blocks_into( + &mut virtual_overseer, + &backend, + None, + vec![chain_a[0].0.clone(), chain_a[1].0.clone()], + chain_b.clone(), + ).await; + + assert_eq!(backend.load_first_block_number().unwrap().unwrap(), 1); + assert_backend_contains(&backend, chain_a.iter().map(|&(ref h, _)| h)); + assert_backend_contains(&backend, chain_b.iter().map(|&(ref h, _)| h)); + assert_leaves(&backend, vec![b5_hash, a3_hash]); + assert_leaves_query(&mut virtual_overseer, vec![b5_hash, a3_hash]).await; + + virtual_overseer + }); +} + +#[test] +fn reversion_removes_viability_of_chain() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3. + // + // A3 reverts A1 + + let (_a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 3], + finalized_number, + finalized_hash, + |h| if h.number == 3 { add_reversions(h, Some(1)) } + ); + + import_blocks_into( + &mut virtual_overseer, + &backend, + Some((finalized_number, finalized_hash)), + chain_a.clone(), + ).await; + + assert_backend_contains(&backend, chain_a.iter().map(|&(ref h, _)| h)); + assert_leaves(&backend, vec![]); + assert_finalized_leaves_query( + &mut virtual_overseer, + finalized_number, + finalized_hash, + ).await; + + virtual_overseer + }); +} + +#[test] +fn reversion_removes_viability_and_finds_ancestor_as_leaf() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3. + // + // A3 reverts A2 + + let (_a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 3], + finalized_number, + finalized_hash, + |h| if h.number == 3 { add_reversions(h, Some(2)) } + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + + import_blocks_into( + &mut virtual_overseer, + &backend, + Some((finalized_number, finalized_hash)), + chain_a.clone(), + ).await; + + assert_backend_contains(&backend, chain_a.iter().map(|&(ref h, _)| h)); + assert_leaves(&backend, vec![a1_hash]); + assert_leaves_query(&mut virtual_overseer, vec![a1_hash]).await; + + virtual_overseer + }); +} + +#[test] +fn ancestor_of_unviable_is_not_leaf_if_has_children() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3. + // A1 <- B2 + // + // A3 reverts A2 + + let (a2_hash, chain_a) = construct_chain_on_base( + vec![1, 2], + finalized_number, + finalized_hash, + |_| {} + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + + let (_a3_hash, chain_a_ext) = construct_chain_on_base( + vec![3], + 2, + a2_hash, + |h| add_reversions(h, Some(2)), + ); + + let (b2_hash, chain_b) = construct_chain_on_base( + vec![1], + 1, + a1_hash, + |h| salt_header(h, b"b") + ); + + import_blocks_into( + &mut virtual_overseer, + &backend, + Some((finalized_number, finalized_hash)), + chain_a.clone(), + ).await; + + import_blocks_into( + &mut virtual_overseer, + &backend, + None, + chain_b.clone(), + ).await; + + assert_backend_contains(&backend, chain_a.iter().map(|&(ref h, _)| h)); + assert_backend_contains(&backend, chain_b.iter().map(|&(ref h, _)| h)); + assert_leaves(&backend, vec![a2_hash, b2_hash]); + + import_blocks_into( + &mut virtual_overseer, + &backend, + None, + chain_a_ext.clone(), + ).await; + + assert_backend_contains(&backend, chain_a.iter().map(|&(ref h, _)| h)); + assert_backend_contains(&backend, chain_a_ext.iter().map(|&(ref h, _)| h)); + assert_backend_contains(&backend, chain_b.iter().map(|&(ref h, _)| h)); + assert_leaves(&backend, vec![b2_hash]); + assert_leaves_query(&mut virtual_overseer, vec![b2_hash]).await; + + virtual_overseer + }); +} + +#[test] +fn self_and_future_reversions_are_ignored() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3. + // + // A3 reverts itself and future blocks. ignored. + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 3], + finalized_number, + finalized_hash, + |h| if h.number == 3 { add_reversions(h, vec![3, 4, 100]) } + ); + + import_blocks_into( + &mut virtual_overseer, + &backend, + Some((finalized_number, finalized_hash)), + chain_a.clone(), + ).await; + + assert_backend_contains(&backend, chain_a.iter().map(|&(ref h, _)| h)); + assert_leaves(&backend, vec![a3_hash]); + assert_leaves_query(&mut virtual_overseer, vec![a3_hash]).await; + + virtual_overseer + }); +} + +#[test] +fn revert_finalized_is_ignored() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 10; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3. + // + // A3 reverts itself and future blocks. ignored. + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 3], + finalized_number, + finalized_hash, + |h| if h.number == 13 { add_reversions(h, vec![10, 9, 8, 0, 1]) } + ); + + import_blocks_into( + &mut virtual_overseer, + &backend, + Some((finalized_number, finalized_hash)), + chain_a.clone(), + ).await; + + assert_backend_contains(&backend, chain_a.iter().map(|&(ref h, _)| h)); + assert_leaves(&backend, vec![a3_hash]); + assert_leaves_query(&mut virtual_overseer, vec![a3_hash]).await; + + virtual_overseer + }); +} + +#[test] +fn reversion_affects_viability_of_all_subtrees() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3. + // A2 <- B3 <- B4 + // + // B4 reverts A2. + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 3], + finalized_number, + finalized_hash, + |_| {} + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + let (_, a2_hash, _) = extract_info_from_chain(1, &chain_a); + + let (_b4_hash, chain_b) = construct_chain_on_base( + vec![3, 4], + 2, + a2_hash, + |h| { + salt_header(h, b"b"); + if h.number == 4 { + add_reversions(h, Some(2)); + } + } + ); + + import_blocks_into( + &mut virtual_overseer, + &backend, + Some((finalized_number, finalized_hash)), + chain_a.clone(), + ).await; + + assert_leaves(&backend, vec![a3_hash]); + + import_blocks_into( + &mut virtual_overseer, + &backend, + None, + chain_b.clone(), + ).await; + + assert_backend_contains(&backend, chain_a.iter().map(|&(ref h, _)| h)); + assert_backend_contains(&backend, chain_b.iter().map(|&(ref h, _)| h)); + assert_leaves(&backend, vec![a1_hash]); + assert_leaves_query(&mut virtual_overseer, vec![a1_hash]).await; + + virtual_overseer + }); +} + +#[test] +fn finalize_viable_prunes_subtrees() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // A2 <- X3 + // F <- A1 <- A2 <- A3. + // A1 <- B2 + // F <- C1 <- C2 <- C3 + // C2 <- D3 + // + // Finalize A2. Only A2, A3, and X3 should remain. + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 10], + finalized_number, + finalized_hash, + |h| salt_header(h, b"a"), + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + let (_, a2_hash, _) = extract_info_from_chain(1, &chain_a); + + let (x3_hash, chain_x) = construct_chain_on_base( + vec![3], + 2, + a2_hash, + |h| salt_header(h, b"x"), + ); + + let (b2_hash, chain_b) = construct_chain_on_base( + vec![6], + 1, + a1_hash, + |h| salt_header(h, b"b"), + ); + + let (c3_hash, chain_c) = construct_chain_on_base( + vec![1, 2, 8], + finalized_number, + finalized_hash, + |h| salt_header(h, b"c"), + ); + let (_, c2_hash, _) = extract_info_from_chain(1, &chain_c); + + let (d3_hash, chain_d) = construct_chain_on_base( + vec![7], + 2, + c2_hash, + |h| salt_header(h, b"d"), + ); + + let all_chains = vec![ + chain_a.clone(), + chain_x.clone(), + chain_b.clone(), + chain_c.clone(), + chain_d.clone(), + ]; + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + all_chains.clone(), + ).await; + + assert_backend_contains_chains( + &backend, + all_chains.clone(), + ); + assert_leaves(&backend, vec![a3_hash, c3_hash, d3_hash, b2_hash, x3_hash]); + + // Finalize block A2. Now lots of blocks should go missing. + finalize_block( + &mut virtual_overseer, + &backend, + 2, + a2_hash, + ).await; + + // A2 <- A3 + // A2 <- X3 + + backend.assert_contains_only(vec![ + (3, a3_hash), + (3, x3_hash), + ]); + + assert_leaves(&backend, vec![a3_hash, x3_hash]); + assert_leaves_query(&mut virtual_overseer, vec![a3_hash, x3_hash]).await; + + assert_eq!( + backend.load_first_block_number().unwrap().unwrap(), + 3, + ); + + assert_eq!( + backend.load_blocks_by_number(3).unwrap(), + vec![a3_hash, x3_hash], + ); + + virtual_overseer + }); +} + +#[test] +fn finalization_does_not_clobber_unviability() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3 + // A3 reverts A2. + // Finalize A1. + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 10], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + if h.number == 3 { + add_reversions(h, Some(2)); + } + } + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + let (_, a2_hash, _) = extract_info_from_chain(1, &chain_a); + + import_blocks_into( + &mut virtual_overseer, + &backend, + Some((finalized_number, finalized_hash)), + chain_a.clone(), + ).await; + + finalize_block( + &mut virtual_overseer, + &backend, + 1, + a1_hash, + ).await; + + assert_leaves(&backend, vec![]); + assert_finalized_leaves_query( + &mut virtual_overseer, + 1, + a1_hash, + ).await; + backend.assert_contains_only(vec![ + (3, a3_hash), + (2, a2_hash), + ]); + + virtual_overseer + }); +} + +#[test] +fn finalization_erases_unviable() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3 + // A1 <- B2 + // + // A2 reverts A1. + // Finalize A1. + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 3], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + if h.number == 2 { + add_reversions(h, Some(1)); + } + } + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + let (_, a2_hash, _) = extract_info_from_chain(1, &chain_a); + + let (b2_hash, chain_b) = construct_chain_on_base( + vec![1], + 1, + a1_hash, + |h| salt_header(h, b"b"), + ); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone(), chain_b.clone()], + ).await; + + assert_leaves(&backend, vec![]); + + finalize_block( + &mut virtual_overseer, + &backend, + 1, + a1_hash, + ).await; + + assert_leaves(&backend, vec![a3_hash, b2_hash]); + assert_leaves_query(&mut virtual_overseer, vec![a3_hash, b2_hash]).await; + + backend.assert_contains_only(vec![ + (3, a3_hash), + (2, a2_hash), + (2, b2_hash), + ]); + + virtual_overseer + }); +} + +#[test] +fn finalize_erases_unviable_but_keeps_later_unviability() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3 + // A1 <- B2 + // + // A2 reverts A1. + // A3 reverts A2. + // Finalize A1. A2 is stil unviable, but B2 is viable. + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 3], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + if h.number == 2 { + add_reversions(h, Some(1)); + } + if h.number == 3 { + add_reversions(h, Some(2)); + } + } + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + let (_, a2_hash, _) = extract_info_from_chain(1, &chain_a); + + let (b2_hash, chain_b) = construct_chain_on_base( + vec![1], + 1, + a1_hash, + |h| salt_header(h, b"b"), + ); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone(), chain_b.clone()], + ).await; + + assert_leaves(&backend, vec![]); + + finalize_block( + &mut virtual_overseer, + &backend, + 1, + a1_hash, + ).await; + + assert_leaves(&backend, vec![b2_hash]); + assert_leaves_query(&mut virtual_overseer, vec![b2_hash]).await; + + backend.assert_contains_only(vec![ + (3, a3_hash), + (2, a2_hash), + (2, b2_hash), + ]); + + virtual_overseer + }); +} + +#[test] +fn finalize_erases_unviable_from_one_but_not_all_reverts() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3 + // + // A3 reverts A2 and A1. + // Finalize A1. A2 is stil unviable. + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 3], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + if h.number == 3 { + add_reversions(h, Some(1)); + add_reversions(h, Some(2)); + } + } + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + let (_, a2_hash, _) = extract_info_from_chain(1, &chain_a); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone()], + ).await; + + assert_leaves(&backend, vec![]); + + finalize_block( + &mut virtual_overseer, + &backend, + 1, + a1_hash, + ).await; + + assert_leaves(&backend, vec![]); + assert_finalized_leaves_query( + &mut virtual_overseer, + 1, + a1_hash, + ).await; + + backend.assert_contains_only(vec![ + (3, a3_hash), + (2, a2_hash), + ]); + + virtual_overseer + }); +} + +#[test] +fn finalize_triggers_viability_search() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3 + // A2 <- B3 + // A2 <- C3 + // A3 reverts A1. + // Finalize A1. A3, B3, and C3 are all viable now. + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 3], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + if h.number == 3 { + add_reversions(h, Some(1)); + } + } + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + let (_, a2_hash, _) = extract_info_from_chain(1, &chain_a); + + let (b3_hash, chain_b) = construct_chain_on_base( + vec![4], + 2, + a2_hash, + |h| salt_header(h, b"b"), + ); + + let (c3_hash, chain_c) = construct_chain_on_base( + vec![5], + 2, + a2_hash, + |h| salt_header(h, b"c"), + ); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone(), chain_b.clone(), chain_c.clone()], + ).await; + + assert_leaves(&backend, vec![]); + + finalize_block( + &mut virtual_overseer, + &backend, + 1, + a1_hash, + ).await; + + assert_leaves(&backend, vec![c3_hash, b3_hash, a3_hash]); + assert_leaves_query(&mut virtual_overseer, vec![c3_hash, b3_hash, a3_hash]).await; + + backend.assert_contains_only(vec![ + (3, a3_hash), + (3, b3_hash), + (3, c3_hash), + (2, a2_hash), + ]); + + virtual_overseer + }); +} + +#[test] +fn best_leaf_none_with_empty_db() { + test_harness(|_backend, mut virtual_overseer| async move { + let required = Hash::repeat_byte(1); + let best_leaf = best_leaf_containing(&mut virtual_overseer, required).await; + assert!(best_leaf.is_none()); + + virtual_overseer + }) +} + +#[test] +fn best_leaf_none_with_no_viable_leaves() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 + // + // A2 reverts A1. + + let (a2_hash, chain_a) = construct_chain_on_base( + vec![1, 2], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + if h.number == 2 { + add_reversions(h, Some(1)); + } + } + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone()], + ).await; + + let best_leaf = best_leaf_containing(&mut virtual_overseer, a2_hash).await; + assert!(best_leaf.is_none()); + + let best_leaf = best_leaf_containing(&mut virtual_overseer, a1_hash).await; + assert!(best_leaf.is_none()); + + virtual_overseer + }) +} + +#[test] +fn best_leaf_none_with_unknown_required() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 + + let (_a2_hash, chain_a) = construct_chain_on_base( + vec![1, 2], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + } + ); + + let unknown_hash = Hash::repeat_byte(0x69); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone()], + ).await; + + let best_leaf = best_leaf_containing(&mut virtual_overseer, unknown_hash).await; + assert!(best_leaf.is_none()); + + virtual_overseer + }) +} + +#[test] +fn best_leaf_none_with_unviable_required() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 + // F <- B1 <- B2 + // + // A2 reverts A1. + + let (a2_hash, chain_a) = construct_chain_on_base( + vec![1, 2], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + if h.number == 2 { + add_reversions(h, Some(1)); + } + } + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + + let (_b2_hash, chain_b) = construct_chain_on_base( + vec![1, 2], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"b"); + } + ); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone(), chain_b.clone()], + ).await; + + let best_leaf = best_leaf_containing(&mut virtual_overseer, a2_hash).await; + assert!(best_leaf.is_none()); + + let best_leaf = best_leaf_containing(&mut virtual_overseer, a1_hash).await; + assert!(best_leaf.is_none()); + + virtual_overseer + }) +} + +#[test] +fn best_leaf_with_finalized_required() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 + // F <- B1 <- B2 + // + // B2 > A2 + + let (_a2_hash, chain_a) = construct_chain_on_base( + vec![1, 1], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + } + ); + + let (b2_hash, chain_b) = construct_chain_on_base( + vec![1, 2], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"b"); + } + ); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone(), chain_b.clone()], + ).await; + + let best_leaf = best_leaf_containing(&mut virtual_overseer, finalized_hash).await; + assert_eq!(best_leaf, Some(b2_hash)); + + virtual_overseer + }) +} + +#[test] +fn best_leaf_with_unfinalized_required() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 + // F <- B1 <- B2 + // + // B2 > A2 + + let (a2_hash, chain_a) = construct_chain_on_base( + vec![1, 1], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + } + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + + let (_b2_hash, chain_b) = construct_chain_on_base( + vec![1, 2], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"b"); + } + ); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone(), chain_b.clone()], + ).await; + + let best_leaf = best_leaf_containing(&mut virtual_overseer, a1_hash).await; + assert_eq!(best_leaf, Some(a2_hash)); + + virtual_overseer + }) +} + +#[test] +fn best_leaf_ancestor_of_all_leaves() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3 + // A1 <- B2 <- B3 + // B2 <- C3 + // + // C3 > B3 > A3 + + let (_a3_hash, chain_a) = construct_chain_on_base( + vec![1, 1, 2], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + } + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + + let (_b3_hash, chain_b) = construct_chain_on_base( + vec![2, 3], + 1, + a1_hash, + |h| { + salt_header(h, b"b"); + } + ); + + let (_, b2_hash, _) = extract_info_from_chain(0, &chain_b); + + let (c3_hash, chain_c) = construct_chain_on_base( + vec![4], + 2, + b2_hash, + |h| { + salt_header(h, b"c"); + } + ); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone(), chain_b.clone(), chain_c.clone()], + ).await; + + let best_leaf = best_leaf_containing(&mut virtual_overseer, a1_hash).await; + assert_eq!(best_leaf, Some(c3_hash)); + + virtual_overseer + }) +} + +#[test] +fn approve_message_approves_block_entry() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3 + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 3], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + } + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + let (_, a2_hash, _) = extract_info_from_chain(1, &chain_a); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone()], + ).await; + + approve_block(&mut virtual_overseer, &backend, a3_hash).await; + + // a3 is approved, but not a1 or a2. + assert_matches!( + backend.load_block_entry(&a3_hash).unwrap().unwrap().viability.approval, + Approval::Approved + ); + + assert_matches!( + backend.load_block_entry(&a2_hash).unwrap().unwrap().viability.approval, + Approval::Unapproved + ); + + assert_matches!( + backend.load_block_entry(&a1_hash).unwrap().unwrap().viability.approval, + Approval::Unapproved + ); + + virtual_overseer + }) +} + +#[test] +fn approve_nonexistent_has_no_effect() { + test_harness(|backend, mut virtual_overseer| async move { + let finalized_number = 0; + let finalized_hash = Hash::repeat_byte(0); + + // F <- A1 <- A2 <- A3 + + let (a3_hash, chain_a) = construct_chain_on_base( + vec![1, 2, 3], + finalized_number, + finalized_hash, + |h| { + salt_header(h, b"a"); + } + ); + + let (_, a1_hash, _) = extract_info_from_chain(0, &chain_a); + let (_, a2_hash, _) = extract_info_from_chain(1, &chain_a); + + import_chains_into_empty( + &mut virtual_overseer, + &backend, + finalized_number, + finalized_hash, + vec![chain_a.clone()], + ).await; + + let nonexistent = Hash::repeat_byte(1); + approve_block(&mut virtual_overseer, &backend, nonexistent).await; + + // a3 is approved, but not a1 or a2. + assert_matches!( + backend.load_block_entry(&a3_hash).unwrap().unwrap().viability.approval, + Approval::Unapproved + ); + + assert_matches!( + backend.load_block_entry(&a2_hash).unwrap().unwrap().viability.approval, + Approval::Unapproved + ); + + assert_matches!( + backend.load_block_entry(&a1_hash).unwrap().unwrap().viability.approval, + Approval::Unapproved + ); + + virtual_overseer + }) +} diff --git a/node/core/chain-selection/src/tree.rs b/node/core/chain-selection/src/tree.rs new file mode 100644 index 000000000000..a10f0d0c5ad5 --- /dev/null +++ b/node/core/chain-selection/src/tree.rs @@ -0,0 +1,584 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Implements the tree-view over the data backend which we use to determine +//! viable leaves. +//! +//! The metadata is structured as a tree, with the root implicitly being the +//! finalized block, which is not stored as part of the tree. +//! +//! Each direct descendant of the finalized block acts as its own sub-tree, +//! and as the finalized block advances, orphaned sub-trees are entirely pruned. + +use polkadot_primitives::v1::{BlockNumber, Hash}; +use polkadot_node_primitives::BlockWeight; + + +use std::collections::HashMap; + +use super::{ + LOG_TARGET, + Approval, BlockEntry, Error, LeafEntry, ViabilityCriteria, + Timestamp, +}; +use crate::backend::{Backend, OverlayedBackend}; + +// A viability update to be applied to a block. +struct ViabilityUpdate(Option); + +impl ViabilityUpdate { + // Apply the viability update to a single block, yielding the updated + // block entry along with a vector of children and the updates to apply + // to them. + fn apply(self, mut entry: BlockEntry) -> ( + BlockEntry, + Vec<(Hash, ViabilityUpdate)> + ) { + // 1. When an ancestor has changed from unviable to viable, + // we erase the `earliest_unviable_ancestor` of all descendants + // until encountering a explicitly unviable descendant D. + // + // We then update the `earliest_unviable_ancestor` for all + // descendants of D to be equal to D. + // + // 2. When an ancestor A has changed from viable to unviable, + // we update the `earliest_unviable_ancestor` for all blocks + // to A. + // + // The following algorithm covers both cases. + // + // Furthermore, if there has been any change in viability, + // it is necessary to visit every single descendant of the root + // block. + // + // If a block B was unviable and is now viable, then every descendant + // has an `earliest_unviable_ancestor` which must be updated either + // to nothing or to the new earliest unviable ancestor. + // + // If a block B was viable and is now unviable, then every descendant + // has an `earliest_unviable_ancestor` which needs to be set to B. + + let maybe_earliest_unviable = self.0; + let next_earliest_unviable = { + if maybe_earliest_unviable.is_none() && !entry.viability.is_explicitly_viable() { + Some(entry.block_hash) + } else { + maybe_earliest_unviable + } + }; + entry.viability.earliest_unviable_ancestor = maybe_earliest_unviable; + + let recurse = entry.children.iter() + .cloned() + .map(move |c| (c, ViabilityUpdate(next_earliest_unviable))) + .collect(); + + (entry, recurse) + } +} + +// Propagate viability update to descendants of the given block. This writes +// the `base` entry as well as all descendants. If the parent of the block +// entry is not viable, this wlil not affect any descendants. +// +// If the block entry provided is self-unviable, then it's assumed that an +// unviability update needs to be propagated to descendants. +// +// If the block entry provided is self-viable, then it's assumed that a +// viability update needs to be propagated to descendants. +fn propagate_viability_update( + backend: &mut OverlayedBackend, + base: BlockEntry, +) -> Result<(), Error> { + enum BlockEntryRef { + Explicit(BlockEntry), + Hash(Hash), + } + + if !base.viability.is_parent_viable() { + // If the parent of the block is still unviable, + // then the `earliest_viable_ancestor` will not change + // regardless of the change in the block here. + // + // Furthermore, in such cases, the set of viable leaves + // does not change at all. + backend.write_block_entry(base); + return Ok(()) + } + + let mut viable_leaves = backend.load_leaves()?; + + // A mapping of Block Hash -> number + // Where the hash is the hash of a viable block which has + // at least 1 unviable child. + // + // The number is the number of known unviable children which is known + // as the pivot count. + let mut viability_pivots = HashMap::new(); + + // If the base block is itself explicitly unviable, + // this will change to a `Some(base_hash)` after the first + // invocation. + let viability_update = ViabilityUpdate(None); + + // Recursively apply update to tree. + // + // As we go, we remove any blocks from the leaves which are no longer viable + // leaves. We also add blocks to the leaves-set which are obviously viable leaves. + // And we build up a frontier of blocks which may either be viable leaves or + // the ancestors of one. + let mut tree_frontier = vec![(BlockEntryRef::Explicit(base), viability_update)]; + while let Some((entry_ref, update)) = tree_frontier.pop() { + let entry = match entry_ref { + BlockEntryRef::Explicit(entry) => entry, + BlockEntryRef::Hash(hash) => match backend.load_block_entry(&hash)? { + None => { + tracing::warn!( + target: LOG_TARGET, + block_hash = ?hash, + "Missing expected block entry" + ); + + continue; + } + Some(entry) => entry, + } + }; + + let (new_entry, children) = update.apply(entry); + + if new_entry.viability.is_viable() { + // A block which is viable has a parent which is obviously not + // in the viable leaves set. + viable_leaves.remove(&new_entry.parent_hash); + + // Furthermore, if the block is viable and has no children, + // it is viable by definition. + if new_entry.children.is_empty() { + viable_leaves.insert(new_entry.leaf_entry()); + } + } else { + // A block which is not viable is certainly not a viable leaf. + viable_leaves.remove(&new_entry.block_hash); + + // When the parent is viable but the entry itself is not, that means + // that the parent is a viability pivot. As we visit the children + // of a viability pivot, we build up an exhaustive pivot count. + if new_entry.viability.is_parent_viable() { + *viability_pivots.entry(new_entry.parent_hash).or_insert(0) += 1; + } + } + + backend.write_block_entry(new_entry); + + tree_frontier.extend( + children.into_iter().map(|(h, update)| (BlockEntryRef::Hash(h), update)) + ); + } + + // Revisit the viability pivots now that we've traversed the entire subtree. + // After this point, the viable leaves set is fully updated. A proof follows. + // + // If the base has become unviable, then we've iterated into all descendants, + // made them unviable and removed them from the set. We know that the parent is + // viable as this function is a no-op otherwise, so we need to see if the parent + // has other children or not. + // + // If the base has become viable, then we've iterated into all descendants, + // and found all blocks which are viable and have no children. We've already added + // those blocks to the leaf set, but what we haven't detected + // is blocks which are viable and have children, but all of the children are + // unviable. + // + // The solution of viability pivots addresses both of these: + // + // When the base has become unviable, the parent's viability is unchanged and therefore + // any leaves descending from parent but not base are still in the viable leaves set. + // If the parent has only one child which is the base, the parent is now a viable leaf. + // We've already visited the base in recursive search so the set of pivots should + // contain only a single entry `(parent, 1)`. qed. + // + // When the base has become viable, we've already iterated into every descendant + // of the base and thus have collected a set of pivots whose corresponding pivot + // counts have already been exhaustively computed from their children. qed. + for (pivot, pivot_count) in viability_pivots { + match backend.load_block_entry(&pivot)? { + None => { + // This means the block is finalized. We might reach this + // code path when the base is a child of the finalized block + // and has become unviable. + // + // Each such child is the root of its own tree + // which, as an invariant, does not depend on the viability + // of the finalized block. So no siblings need to be inspected + // and we can ignore it safely. + // + // Furthermore, if the set of viable leaves is empty, the + // finalized block is implicitly the viable leaf. + continue + } + Some(entry) => { + if entry.children.len() == pivot_count { + viable_leaves.insert(entry.leaf_entry()); + } + } + } + } + + backend.write_leaves(viable_leaves); + + Ok(()) +} + +/// Imports a new block and applies any reversions to ancestors. +pub(crate) fn import_block( + backend: &mut OverlayedBackend, + block_hash: Hash, + block_number: BlockNumber, + parent_hash: Hash, + reversion_logs: Vec, + weight: BlockWeight, +) -> Result<(), Error> { + add_block(backend, block_hash, block_number, parent_hash, weight)?; + apply_reversions( + backend, + block_hash, + block_number, + reversion_logs, + )?; + + Ok(()) +} + +// Load the given ancestor's block entry, in descending order from the `block_hash`. +// The ancestor_number must be at least one block less than the `block_number`. +// +// The returned entry will be `None` if the range is invalid or any block in the path had +// no entry present. If any block entry was missing, it can safely be assumed to +// be finalized. +fn load_ancestor( + backend: &mut OverlayedBackend, + block_hash: Hash, + block_number: BlockNumber, + ancestor_number: BlockNumber, +) -> Result, Error> { + if block_number <= ancestor_number { return Ok(None) } + + let mut current_hash = block_hash; + let mut current_entry = None; + + let segment_length = (block_number - ancestor_number) + 1; + for _ in 0..segment_length { + match backend.load_block_entry(¤t_hash)? { + None => return Ok(None), + Some(entry) => { + let parent_hash = entry.parent_hash; + current_entry = Some(entry); + current_hash = parent_hash; + } + } + } + + // Current entry should always be `Some` here. + Ok(current_entry) +} + +// Add a new block to the tree, which is assumed to be unreverted and unapproved, +// but not stagnant. It inherits viability from its parent, if any. +// +// This updates the parent entry, if any, and updates the viable leaves set accordingly. +// This also schedules a stagnation-check update and adds the block to the blocks-by-number +// mapping. +fn add_block( + backend: &mut OverlayedBackend, + block_hash: Hash, + block_number: BlockNumber, + parent_hash: Hash, + weight: BlockWeight, +) -> Result<(), Error> { + let mut leaves = backend.load_leaves()?; + let parent_entry = backend.load_block_entry(&parent_hash)?; + + let inherited_viability = parent_entry.as_ref() + .and_then(|parent| parent.non_viable_ancestor_for_child()); + + // 1. Add the block to the DB assuming it's not reverted. + backend.write_block_entry( + BlockEntry { + block_hash, + block_number, + parent_hash, + children: Vec::new(), + viability: ViabilityCriteria { + earliest_unviable_ancestor: inherited_viability, + explicitly_reverted: false, + approval: Approval::Unapproved, + }, + weight, + } + ); + + // 2. Update leaves if inherited viability is fine. + if inherited_viability.is_none() { + leaves.remove(&parent_hash); + leaves.insert(LeafEntry { block_hash, block_number, weight }); + backend.write_leaves(leaves); + } + + // 3. Update and write the parent + if let Some(mut parent_entry) = parent_entry { + parent_entry.children.push(block_hash); + backend.write_block_entry(parent_entry); + } + + // 4. Add to blocks-by-number. + let mut blocks_by_number = backend.load_blocks_by_number(block_number)?; + blocks_by_number.push(block_hash); + backend.write_blocks_by_number(block_number, blocks_by_number); + + // 5. Add stagnation timeout. + let stagnant_at = crate::stagnant_timeout_from_now(); + let mut stagnant_at_list = backend.load_stagnant_at(stagnant_at)?; + stagnant_at_list.push(block_hash); + backend.write_stagnant_at(stagnant_at, stagnant_at_list); + + Ok(()) +} + +// Assuming that a block is already imported, accepts the number of the block +// as well as a list of reversions triggered by the block in ascending order. +fn apply_reversions( + backend: &mut OverlayedBackend, + block_hash: Hash, + block_number: BlockNumber, + reversions: Vec, +) -> Result<(), Error> { + // Note: since revert numbers are in ascending order, the expensive propagation + // of unviability is only heavy on the first log. + for revert_number in reversions { + let mut ancestor_entry = match load_ancestor( + backend, + block_hash, + block_number, + revert_number, + )? { + None => { + tracing::warn!( + target: LOG_TARGET, + ?block_hash, + block_number, + revert_target = revert_number, + "The hammer has dropped. \ + A block has indicated that its finalized ancestor be reverted. \ + Please inform an adult.", + ); + + continue + } + Some(ancestor_entry) => { + tracing::info!( + target: LOG_TARGET, + ?block_hash, + block_number, + revert_target = revert_number, + revert_hash = ?ancestor_entry.block_hash, + "A block has signaled that its ancestor be reverted due to a bad parachain block.", + ); + + ancestor_entry + } + }; + + ancestor_entry.viability.explicitly_reverted = true; + propagate_viability_update(backend, ancestor_entry)?; + } + + Ok(()) +} + +/// Finalize a block with the given number and hash. +/// +/// This will prune all sub-trees not descending from the given block, +/// all block entries at or before the given height, +/// and will update the viability of all sub-trees descending from the given +/// block if the finalized block was not viable. +/// +/// This is assumed to start with a fresh backend, and will produce +/// an overlay over the backend with all the changes applied. +pub(super) fn finalize_block<'a, B: Backend + 'a>( + backend: &'a B, + finalized_hash: Hash, + finalized_number: BlockNumber, +) -> Result, Error> { + let earliest_stored_number = backend.load_first_block_number()?; + let mut backend = OverlayedBackend::new(backend); + + let earliest_stored_number = match earliest_stored_number { + None => { + // This implies that there are no unfinalized blocks and hence nothing + // to update. + return Ok(backend); + } + Some(e) => e, + }; + + let mut viable_leaves = backend.load_leaves()?; + + // Walk all numbers up to the finalized number and remove those entries. + for number in earliest_stored_number..finalized_number { + let blocks_at = backend.load_blocks_by_number(number)?; + backend.delete_blocks_by_number(number); + + for block in blocks_at { + viable_leaves.remove(&block); + backend.delete_block_entry(&block); + } + } + + // Remove all blocks at the finalized height, with the exception of the finalized block, + // and their descendants, recursively. + { + let blocks_at_finalized_height = backend.load_blocks_by_number(finalized_number)?; + backend.delete_blocks_by_number(finalized_number); + + let mut frontier: Vec<_> = blocks_at_finalized_height + .into_iter() + .filter(|h| h != &finalized_hash) + .map(|h| (h, finalized_number)) + .collect(); + + while let Some((dead_hash, dead_number)) = frontier.pop() { + let entry = backend.load_block_entry(&dead_hash)?; + backend.delete_block_entry(&dead_hash); + viable_leaves.remove(&dead_hash); + + // This does a few extra `clone`s but is unlikely to be + // a bottleneck. Code complexity is very low as a result. + let mut blocks_at_height = backend.load_blocks_by_number(dead_number)?; + blocks_at_height.retain(|h| h != &dead_hash); + backend.write_blocks_by_number(dead_number, blocks_at_height); + + // Add all children to the frontier. + let next_height = dead_number + 1; + frontier.extend( + entry.into_iter().flat_map(|e| e.children).map(|h| (h, next_height)) + ); + } + } + + // Visit and remove the finalized block, fetching its children. + let children_of_finalized = { + let finalized_entry = backend.load_block_entry(&finalized_hash)?; + backend.delete_block_entry(&finalized_hash); + viable_leaves.remove(&finalized_hash); + + finalized_entry.into_iter().flat_map(|e| e.children) + }; + + backend.write_leaves(viable_leaves); + + // Update the viability of each child. + for child in children_of_finalized { + if let Some(mut child) = backend.load_block_entry(&child)? { + // Finalized blocks are always viable. + child.viability.earliest_unviable_ancestor = None; + + propagate_viability_update(&mut backend, child)?; + } else { + tracing::debug!( + target: LOG_TARGET, + ?finalized_hash, + finalized_number, + child_hash = ?child, + "Missing child of finalized block", + ); + + // No need to do anything, but this is an inconsistent state. + } + } + + Ok(backend) +} + +/// Mark a block as approved and update the viability of itself and its +/// descendants accordingly. +pub(super) fn approve_block( + backend: &mut OverlayedBackend, + approved_hash: Hash, +) -> Result<(), Error> { + if let Some(mut entry) = backend.load_block_entry(&approved_hash)? { + let was_viable = entry.viability.is_viable(); + entry.viability.approval = Approval::Approved; + let is_viable = entry.viability.is_viable(); + + // Approval can change the viability in only one direction. + // If the viability has changed, then we propagate that to children + // and recalculate the viable leaf set. + if !was_viable && is_viable { + propagate_viability_update(backend, entry)?; + } else { + backend.write_block_entry(entry); + } + + } else { + tracing::debug!( + target: LOG_TARGET, + block_hash = ?approved_hash, + "Missing entry for freshly-approved block. Ignoring" + ); + } + + Ok(()) +} + +/// Check whether any blocks up to the given timestamp are stagnant and update +/// accordingly. +/// +/// This accepts a fresh backend and returns an overlay on top of it representing +/// all changes made. +// TODO https://github.com/paritytech/polkadot/issues/3293:: remove allow +#[allow(unused)] +pub(super) fn detect_stagnant<'a, B: 'a + Backend>( + backend: &'a B, + up_to: Timestamp, +) -> Result, Error> { + let stagnant_up_to = backend.load_stagnant_at_up_to(up_to)?; + let mut backend = OverlayedBackend::new(backend); + + // As this is in ascending order, only the earliest stagnant + // blocks will involve heavy viability propagations. + for (timestamp, maybe_stagnant) in stagnant_up_to { + backend.delete_stagnant_at(timestamp); + + for block_hash in maybe_stagnant { + if let Some(mut entry) = backend.load_block_entry(&block_hash)? { + let was_viable = entry.viability.is_viable(); + if let Approval::Unapproved = entry.viability.approval { + entry.viability.approval = Approval::Stagnant; + } + let is_viable = entry.viability.is_viable(); + + if was_viable && !is_viable { + propagate_viability_update(&mut backend, entry)?; + } else { + backend.write_block_entry(entry); + } + } + } + } + + Ok(backend) +} diff --git a/node/core/dispute-coordinator/Cargo.toml b/node/core/dispute-coordinator/Cargo.toml new file mode 100644 index 000000000000..f2ba0491005f --- /dev/null +++ b/node/core/dispute-coordinator/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "polkadot-node-core-dispute-coordinator" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" + +[dependencies] +bitvec = { version = "0.20.1", default-features = false, features = ["alloc"] } +futures = "0.3.12" +tracing = "0.1.26" +parity-scale-codec = "2" +kvdb = "0.9.0" +derive_more = "0.99.14" +thiserror = "1.0.23" + +polkadot-primitives = { path = "../../../primitives" } +polkadot-node-primitives = { path = "../../primitives" } +polkadot-node-subsystem = { path = "../../subsystem" } +polkadot-node-subsystem-util = { path = "../../subsystem-util" } + +sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } + +[dev-dependencies] +kvdb-memorydb = "0.9.0" +polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers"} +sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } +assert_matches = "1.4.0" diff --git a/node/core/dispute-coordinator/src/db/mod.rs b/node/core/dispute-coordinator/src/db/mod.rs new file mode 100644 index 000000000000..9b79bd5bc74c --- /dev/null +++ b/node/core/dispute-coordinator/src/db/mod.rs @@ -0,0 +1,19 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Database component for the dispute coordinator. + +pub(super) mod v1; diff --git a/node/core/dispute-coordinator/src/db/v1.rs b/node/core/dispute-coordinator/src/db/v1.rs new file mode 100644 index 000000000000..2253b83c6192 --- /dev/null +++ b/node/core/dispute-coordinator/src/db/v1.rs @@ -0,0 +1,585 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! V1 database for the dispute coordinator. + +use polkadot_primitives::v1::{ + CandidateReceipt, ValidDisputeStatementKind, InvalidDisputeStatementKind, ValidatorIndex, + ValidatorSignature, SessionIndex, CandidateHash, +}; + +use kvdb::{KeyValueDB, DBTransaction}; +use parity_scale_codec::{Encode, Decode}; + +use crate::DISPUTE_WINDOW; + +const ACTIVE_DISPUTES_KEY: &[u8; 15] = b"active-disputes"; +const EARLIEST_SESSION_KEY: &[u8; 16] = b"earliest-session"; +const CANDIDATE_VOTES_SUBKEY: &[u8; 15] = b"candidate-votes"; + +fn candidate_votes_key(session: SessionIndex, candidate_hash: &CandidateHash) -> [u8; 15 + 4 + 32] { + let mut buf = [0u8; 15 + 4 + 32]; + buf[..15].copy_from_slice(CANDIDATE_VOTES_SUBKEY); + + // big-endian encoding is used to ensure lexicographic ordering. + buf[15..][..4].copy_from_slice(&session.to_be_bytes()); + candidate_hash.using_encoded(|s| buf[(15 + 4)..].copy_from_slice(s)); + + buf +} + +// Computes the upper lexicographic bound on DB keys for candidate votes with a given +// upper-exclusive bound on sessions. +fn candidate_votes_range_upper_bound(upper_exclusive: SessionIndex) -> [u8; 15 + 4] { + let mut buf = [0; 15 + 4]; + buf[..15].copy_from_slice(CANDIDATE_VOTES_SUBKEY); + // big-endian encoding is used to ensure lexicographic ordering. + buf[15..][..4].copy_from_slice(&upper_exclusive.to_be_bytes()); + + buf +} + +fn decode_candidate_votes_key(key: &[u8]) -> Option<(SessionIndex, CandidateHash)> { + if key.len() != 15 + 4 + 32 { + return None; + } + + let mut session_buf = [0; 4]; + session_buf.copy_from_slice(&key[15..][..4]); + let session = SessionIndex::from_be_bytes(session_buf); + + CandidateHash::decode(&mut &key[(15 + 4)..]).ok().map(|hash| (session, hash)) +} + +/// Column configuration information for the DB. +#[derive(Debug, Clone)] +pub struct ColumnConfiguration { + /// The column in the key-value DB where data is stored. + pub col_data: u32, +} + +/// Tracked votes on candidates, for the purposes of dispute resolution. +#[derive(Debug, Clone, Encode, Decode)] +pub struct CandidateVotes { + /// The receipt of the candidate itself. + pub candidate_receipt: CandidateReceipt, + /// Votes of validity, sorted by validator index. + pub valid: Vec<(ValidDisputeStatementKind, ValidatorIndex, ValidatorSignature)>, + /// Votes of invalidity, sorted by validator index. + pub invalid: Vec<(InvalidDisputeStatementKind, ValidatorIndex, ValidatorSignature)>, +} + +impl From for polkadot_node_primitives::CandidateVotes { + fn from(db_votes: CandidateVotes) -> polkadot_node_primitives::CandidateVotes { + polkadot_node_primitives::CandidateVotes { + candidate_receipt: db_votes.candidate_receipt, + valid: db_votes.valid, + invalid: db_votes.invalid, + } + } +} + +impl From for CandidateVotes { + fn from(primitive_votes: polkadot_node_primitives::CandidateVotes) -> CandidateVotes { + CandidateVotes { + candidate_receipt: primitive_votes.candidate_receipt, + valid: primitive_votes.valid, + invalid: primitive_votes.invalid, + } + } +} + +/// Meta-key for tracking active disputes. +#[derive(Debug, Default, Clone, Encode, Decode, PartialEq)] +pub struct ActiveDisputes { + /// All disputed candidates, sorted by session index and then by candidate hash. + pub disputed: Vec<(SessionIndex, CandidateHash)>, +} + +impl ActiveDisputes { + /// Whether the set of active disputes contains the given candidate. + pub(crate) fn contains( + &self, + session: SessionIndex, + candidate_hash: CandidateHash, + ) -> bool { + self.disputed.contains(&(session, candidate_hash)) + } + + /// Insert the session and candidate hash from the set of active disputes. + /// Returns 'true' if the entry was not already in the set. + pub(crate) fn insert( + &mut self, + session: SessionIndex, + candidate_hash: CandidateHash, + ) -> bool { + let new_entry = (session, candidate_hash); + + let pos = self.disputed.iter() + .take_while(|&e| &new_entry < e) + .count(); + if self.disputed.get(pos).map_or(false, |&e| new_entry == e) { + false + } else { + self.disputed.insert(pos, new_entry); + true + } + } + + /// Delete the session and candidate hash from the set of active disputes. + /// Returns 'true' if the entry was present. + pub(crate) fn delete( + &mut self, + session: SessionIndex, + candidate_hash: CandidateHash, + ) -> bool { + let new_entry = (session, candidate_hash); + + match self.disputed.iter().position(|e| &new_entry == e) { + None => false, + Some(pos) => { + self.disputed.remove(pos); + true + } + } + } +} + +/// Errors while accessing things from the DB. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Codec(#[from] parity_scale_codec::Error), +} + +/// Result alias for DB errors. +pub type Result = std::result::Result; + +fn load_decode(db: &dyn KeyValueDB, col_data: u32, key: &[u8]) + -> Result> +{ + match db.get(col_data, key)? { + None => Ok(None), + Some(raw) => D::decode(&mut &raw[..]) + .map(Some) + .map_err(Into::into), + } +} + +/// Load the candidate votes for the identified candidate under the given hash. +pub(crate) fn load_candidate_votes( + db: &dyn KeyValueDB, + config: &ColumnConfiguration, + session: SessionIndex, + candidate_hash: &CandidateHash, +) -> Result> { + load_decode(db, config.col_data, &candidate_votes_key(session, candidate_hash)) +} + +/// Load the earliest session, if any. +pub(crate) fn load_earliest_session( + db: &dyn KeyValueDB, + config: &ColumnConfiguration, +) -> Result> { + load_decode(db, config.col_data, EARLIEST_SESSION_KEY) +} + +/// Load the active disputes, if any. +pub(crate) fn load_active_disputes( + db: &dyn KeyValueDB, + config: &ColumnConfiguration, +) -> Result> { + load_decode(db, config.col_data, ACTIVE_DISPUTES_KEY) +} + +/// An atomic transaction to be commited to the underlying DB. +#[derive(Debug, Default, Clone)] +pub(crate) struct Transaction { + earliest_session: Option, + active_disputes: Option, + write_candidate_votes: Vec<(SessionIndex, CandidateHash, CandidateVotes)>, + delete_candidate_votes: Vec<(SessionIndex, CandidateHash)>, +} + +impl Transaction { + /// Prepare a write to the 'earliest session' field of the DB. + /// + /// Later calls to this function will override earlier ones. + pub(crate) fn put_earliest_session(&mut self, session: SessionIndex) { + self.earliest_session = Some(session); + } + + /// Prepare a write to the active disputes stored in the DB. + /// + /// Later calls to this function will override earlier ones. + pub(crate) fn put_active_disputes(&mut self, active: ActiveDisputes) { + self.active_disputes = Some(active); + } + + + /// Prepare a write of the candidate votes under the indicated candidate. + /// + /// Later calls to this function for the same candidate will override earlier ones. + /// Any calls to this function will be overridden by deletions of the same candidate. + pub(crate) fn put_candidate_votes( + &mut self, + session: SessionIndex, + candidate_hash: CandidateHash, + votes: CandidateVotes, + ) { + self.write_candidate_votes.push((session, candidate_hash, votes)) + } + + /// Prepare a deletion of the candidate votes under the indicated candidate. + /// + /// Any calls to this function will override writes to the same candidate. + pub(crate) fn delete_candidate_votes( + &mut self, + session: SessionIndex, + candidate_hash: CandidateHash, + ) { + self.delete_candidate_votes.push((session, candidate_hash)) + } + + /// Write the transaction atomically to the DB. + pub(crate) fn write(self, db: &dyn KeyValueDB, config: &ColumnConfiguration) -> Result<()> { + let mut tx = DBTransaction::new(); + + if let Some(s) = self.earliest_session { + tx.put_vec(config.col_data, EARLIEST_SESSION_KEY, s.encode()); + } + + if let Some(a) = self.active_disputes { + tx.put_vec(config.col_data, ACTIVE_DISPUTES_KEY, a.encode()); + } + + for (session, candidate_hash, votes) in self.write_candidate_votes { + tx.put_vec(config.col_data, &candidate_votes_key(session, &candidate_hash), votes.encode()); + } + + for (session, candidate_hash) in self.delete_candidate_votes { + tx.delete(config.col_data, &candidate_votes_key(session, &candidate_hash)); + } + + db.write(tx).map_err(Into::into) + } +} + +/// Maybe prune data in the DB based on the provided session index. +/// +/// This is intended to be called on every block, and as such will be used to populate the DB on +/// first launch. If the on-disk data does not need to be pruned, only a single storage read +/// will be performed. +/// +/// If one or more ancient sessions are pruned, all metadata on candidates within the ancient +/// session will be deleted. +pub(crate) fn note_current_session( + store: &dyn KeyValueDB, + config: &ColumnConfiguration, + current_session: SessionIndex, +) -> Result<()> { + let new_earliest = current_session.saturating_sub(DISPUTE_WINDOW); + let mut tx = Transaction::default(); + + match load_earliest_session(store, config)? { + None => { + // First launch - write new-earliest. + tx.put_earliest_session(new_earliest); + } + Some(prev_earliest) if new_earliest > prev_earliest => { + // Prune all data in the outdated sessions. + tx.put_earliest_session(new_earliest); + + // Clear active disputes metadata. + { + let mut active_disputes = load_active_disputes(store, config)?.unwrap_or_default(); + let prune_up_to = active_disputes.disputed.iter() + .take_while(|s| s.0 < new_earliest) + .count(); + + if prune_up_to > 0 { + let _ = active_disputes.disputed.drain(..prune_up_to); + tx.put_active_disputes(active_disputes); + } + } + + // Clear all candidate data with session less than the new earliest kept. + { + let end_prefix = candidate_votes_range_upper_bound(new_earliest); + + store.iter_with_prefix(config.col_data, CANDIDATE_VOTES_SUBKEY) + .take_while(|(k, _)| &k[..] < &end_prefix[..]) + .filter_map(|(k, _)| decode_candidate_votes_key(&k[..])) + .for_each(|(session, candidate_hash)| { + tx.delete_candidate_votes(session, candidate_hash); + }); + } + } + Some(_) => { + // nothing to do. + } + }; + + tx.write(store, config) +} + +#[cfg(test)] +mod tests { + use super::*; + use polkadot_primitives::v1::{Hash, Id as ParaId}; + + #[test] + fn candidate_votes_key_works() { + let session = 4; + let candidate = CandidateHash(Hash::repeat_byte(0x01)); + + let key = candidate_votes_key(session, &candidate); + + assert_eq!(&key[0..15], CANDIDATE_VOTES_SUBKEY); + assert_eq!(&key[15..19], &[0x00, 0x00, 0x00, 0x04]); + assert_eq!(&key[19..51], candidate.0.as_bytes()); + + assert_eq!( + decode_candidate_votes_key(&key[..]), + Some((session, candidate)), + ); + } + + #[test] + fn db_transaction() { + let store = kvdb_memorydb::create(1); + let config = ColumnConfiguration { col_data: 0 }; + + { + let mut tx = Transaction::default(); + + tx.put_earliest_session(0); + tx.put_earliest_session(1); + + tx.put_active_disputes(ActiveDisputes { + disputed: vec![ + (0, CandidateHash(Hash::repeat_byte(0))), + ], + }); + + tx.put_active_disputes(ActiveDisputes { + disputed: vec![ + (1, CandidateHash(Hash::repeat_byte(1))), + ], + }); + + tx.put_candidate_votes( + 1, + CandidateHash(Hash::repeat_byte(1)), + CandidateVotes { + candidate_receipt: Default::default(), + valid: Vec::new(), + invalid: Vec::new(), + }, + ); + tx.put_candidate_votes( + 1, + CandidateHash(Hash::repeat_byte(1)), + CandidateVotes { + candidate_receipt: { + let mut receipt = CandidateReceipt::default(); + receipt.descriptor.para_id = 5.into(); + + receipt + }, + valid: Vec::new(), + invalid: Vec::new(), + }, + ); + + tx.write(&store, &config).unwrap(); + } + + // Test that subsequent writes were written. + { + assert_eq!( + load_earliest_session(&store, &config).unwrap().unwrap(), + 1, + ); + + assert_eq!( + load_active_disputes(&store, &config).unwrap().unwrap(), + ActiveDisputes { + disputed: vec![ + (1, CandidateHash(Hash::repeat_byte(1))), + ], + }, + ); + + assert_eq!( + load_candidate_votes( + &store, + &config, + 1, + &CandidateHash(Hash::repeat_byte(1)) + ).unwrap().unwrap().candidate_receipt.descriptor.para_id, + ParaId::from(5), + ); + } + } + + #[test] + fn db_deletes_supersede_writes() { + let store = kvdb_memorydb::create(1); + let config = ColumnConfiguration { col_data: 0 }; + + { + let mut tx = Transaction::default(); + tx.put_candidate_votes( + 1, + CandidateHash(Hash::repeat_byte(1)), + CandidateVotes { + candidate_receipt: Default::default(), + valid: Vec::new(), + invalid: Vec::new(), + } + ); + + tx.write(&store, &config).unwrap(); + } + + assert_eq!( + load_candidate_votes( + &store, + &config, + 1, + &CandidateHash(Hash::repeat_byte(1)) + ).unwrap().unwrap().candidate_receipt.descriptor.para_id, + ParaId::from(0), + ); + + { + let mut tx = Transaction::default(); + tx.put_candidate_votes( + 1, + CandidateHash(Hash::repeat_byte(1)), + CandidateVotes { + candidate_receipt: { + let mut receipt = CandidateReceipt::default(); + receipt.descriptor.para_id = 5.into(); + + receipt + }, + valid: Vec::new(), + invalid: Vec::new(), + } + ); + + tx.delete_candidate_votes(1, CandidateHash(Hash::repeat_byte(1))); + + tx.write(&store, &config).unwrap(); + } + + assert!( + load_candidate_votes( + &store, + &config, + 1, + &CandidateHash(Hash::repeat_byte(1)) + ).unwrap().is_none() + ); + } + + #[test] + fn note_current_session_prunes_old() { + let store = kvdb_memorydb::create(1); + let config = ColumnConfiguration { col_data: 0 }; + + let hash_a = CandidateHash(Hash::repeat_byte(0x0a)); + let hash_b = CandidateHash(Hash::repeat_byte(0x0b)); + let hash_c = CandidateHash(Hash::repeat_byte(0x0c)); + let hash_d = CandidateHash(Hash::repeat_byte(0x0d)); + + let prev_earliest_session = 0; + let new_earliest_session = 5; + let current_session = 5 + DISPUTE_WINDOW; + + let very_old = 3; + let slightly_old = 4; + let very_recent = current_session - 1; + + let blank_candidate_votes = || CandidateVotes { + candidate_receipt: Default::default(), + valid: Vec::new(), + invalid: Vec::new(), + }; + + { + let mut tx = Transaction::default(); + tx.put_earliest_session(prev_earliest_session); + tx.put_active_disputes(ActiveDisputes { + disputed: vec![ + (very_old, hash_a), + (slightly_old, hash_b), + (new_earliest_session, hash_c), + (very_recent, hash_d), + ], + }); + + tx.put_candidate_votes( + very_old, + hash_a, + blank_candidate_votes(), + ); + + tx.put_candidate_votes( + slightly_old, + hash_b, + blank_candidate_votes(), + ); + + tx.put_candidate_votes( + new_earliest_session, + hash_c, + blank_candidate_votes(), + ); + + tx.put_candidate_votes( + very_recent, + hash_d, + blank_candidate_votes(), + ); + + tx.write(&store, &config).unwrap(); + } + + note_current_session(&store, &config, current_session).unwrap(); + + assert_eq!( + load_earliest_session(&store, &config).unwrap(), + Some(new_earliest_session), + ); + + assert_eq!( + load_active_disputes(&store, &config).unwrap().unwrap(), + ActiveDisputes { + disputed: vec![(new_earliest_session, hash_c), (very_recent, hash_d)], + }, + ); + + assert!(load_candidate_votes(&store, &config, very_old, &hash_a).unwrap().is_none()); + assert!(load_candidate_votes(&store, &config, slightly_old, &hash_b).unwrap().is_none()); + assert!(load_candidate_votes(&store, &config, new_earliest_session, &hash_c).unwrap().is_some()); + assert!(load_candidate_votes(&store, &config, very_recent, &hash_d).unwrap().is_some()); + } +} diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs new file mode 100644 index 000000000000..9d4daeeba7e4 --- /dev/null +++ b/node/core/dispute-coordinator/src/lib.rs @@ -0,0 +1,647 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Implements the dispute coordinator subsystem. +//! +//! This is the central subsystem of the node-side components which participate in disputes. +//! This subsystem wraps a database which tracks all statements observed by all validators over some window of sessions. +//! Votes older than this session window are pruned. +//! +//! This subsystem will be the point which produce dispute votes, either positive or negative, based on locally-observed +//! validation results as well as a sink for votes received by other subsystems. When importing a dispute vote from +//! another node, this will trigger the dispute participation subsystem to recover and validate the block and call +//! back to this subsystem. + +use std::collections::HashSet; +use std::sync::Arc; + +use polkadot_node_primitives::{CandidateVotes, SignedDisputeStatement}; +use polkadot_node_subsystem::{ + messages::{ + DisputeCoordinatorMessage, ChainApiMessage, DisputeParticipationMessage, + }, + Subsystem, SubsystemContext, FromOverseer, OverseerSignal, SpawnedSubsystem, + SubsystemError, + errors::{ChainApiError, RuntimeApiError}, +}; +use polkadot_node_subsystem_util::rolling_session_window::{ + RollingSessionWindow, SessionWindowUpdate, +}; +use polkadot_primitives::v1::{ + SessionIndex, CandidateHash, Hash, CandidateReceipt, DisputeStatement, ValidatorIndex, + ValidatorSignature, BlockNumber, ValidatorPair, +}; + +use futures::prelude::*; +use futures::channel::oneshot; +use kvdb::KeyValueDB; +use parity_scale_codec::Error as CodecError; +use sc_keystore::LocalKeystore; + +use db::v1::ActiveDisputes; + +mod db; + +#[cfg(test)] +mod tests; + +const LOG_TARGET: &str = "parachain::dispute-coordinator"; + +// It would be nice to draw this from the chain state, but we have no tools for it right now. +// On Polkadot this is 1 day, and on Kusama it's 6 hours. +const DISPUTE_WINDOW: SessionIndex = 6; + +struct State { + keystore: Arc, + highest_session: Option, + rolling_session_window: RollingSessionWindow, +} + +/// Configuration for the dispute coordinator subsystem. +#[derive(Debug, Clone, Copy)] +pub struct Config { + /// The data column in the store to use for dispute data. + pub col_data: u32, +} + +impl Config { + fn column_config(&self) -> db::v1::ColumnConfiguration { + db::v1::ColumnConfiguration { col_data: self.col_data } + } +} + +/// An implementation of the dispute coordinator subsystem. +pub struct DisputeCoordinatorSubsystem { + config: Config, + store: Arc, + keystore: Arc, +} + +impl DisputeCoordinatorSubsystem { + /// Create a new instance of the subsystem. + pub fn new( + store: Arc, + config: Config, + keystore: Arc, + ) -> Self { + DisputeCoordinatorSubsystem { store, config, keystore } + } +} + +impl Subsystem for DisputeCoordinatorSubsystem + where Context: SubsystemContext +{ + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = run(self, ctx) + .map(|_| Ok(())) + .boxed(); + + SpawnedSubsystem { + name: "dispute-coordinator-subsystem", + future, + } + } +} + +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] +pub enum Error { + #[error(transparent)] + RuntimeApi(#[from] RuntimeApiError), + + #[error(transparent)] + ChainApi(#[from] ChainApiError), + + #[error(transparent)] + Io(#[from] std::io::Error), + + #[error(transparent)] + Oneshot(#[from] oneshot::Canceled), + + #[error(transparent)] + Subsystem(#[from] SubsystemError), + + #[error(transparent)] + Codec(#[from] CodecError), +} + +impl From for Error { + fn from(err: db::v1::Error) -> Self { + match err { + db::v1::Error::Io(io) => Self::Io(io), + db::v1::Error::Codec(e) => Self::Codec(e), + } + } +} + +impl Error { + fn trace(&self) { + match self { + // don't spam the log with spurious errors + Self::RuntimeApi(_) | + Self::Oneshot(_) => tracing::debug!(target: LOG_TARGET, err = ?self), + // it's worth reporting otherwise + _ => tracing::warn!(target: LOG_TARGET, err = ?self), + } + } +} + +async fn run(subsystem: DisputeCoordinatorSubsystem, mut ctx: Context) + where Context: SubsystemContext +{ + loop { + let res = run_iteration(&mut ctx, &subsystem).await; + match res { + Err(e) => { + e.trace(); + + if let Error::Subsystem(SubsystemError::Context(_)) = e { + break; + } + } + Ok(()) => { + tracing::info!(target: LOG_TARGET, "received `Conclude` signal, exiting"); + break; + } + } + } +} + +// Run the subsystem until an error is encountered or a `conclude` signal is received. +// Most errors are non-fatal and should lead to another call to this function. +// +// A return value of `Ok` indicates that an exit should be made, while non-fatal errors +// lead to another call to this function. +async fn run_iteration(ctx: &mut Context, subsystem: &DisputeCoordinatorSubsystem) + -> Result<(), Error> + where Context: SubsystemContext +{ + let DisputeCoordinatorSubsystem { ref store, ref keystore, ref config } = *subsystem; + let mut state = State { + keystore: keystore.clone(), + highest_session: None, + rolling_session_window: RollingSessionWindow::new(DISPUTE_WINDOW), + }; + + loop { + match ctx.recv().await? { + FromOverseer::Signal(OverseerSignal::Conclude) => { + return Ok(()) + } + FromOverseer::Signal(OverseerSignal::ActiveLeaves(update)) => { + handle_new_activations( + ctx, + &**store, + &mut state, + config, + update.activated.into_iter().map(|a| a.hash), + ).await? + } + FromOverseer::Signal(OverseerSignal::BlockFinalized(_, _)) => {}, + FromOverseer::Communication { msg } => { + handle_incoming( + ctx, + &**store, + &mut state, + config, + msg, + ).await? + } + } + } +} + +async fn handle_new_activations( + ctx: &mut impl SubsystemContext, + store: &dyn KeyValueDB, + state: &mut State, + config: &Config, + new_activations: impl IntoIterator, +) -> Result<(), Error> { + for new_leaf in new_activations { + let block_header = { + let (tx, rx) = oneshot::channel(); + + ctx.send_message( + ChainApiMessage::BlockHeader(new_leaf, tx).into() + ).await; + + match rx.await?? { + None => continue, + Some(header) => header, + } + }; + + match state.rolling_session_window.cache_session_info_for_head( + ctx, + new_leaf, + &block_header, + ).await { + Err(e) => { + tracing::warn!( + target: LOG_TARGET, + err = ?e, + "Failed to update session cache for disputes", + ); + + continue + } + Ok(SessionWindowUpdate::Initialized { window_end, .. }) + | Ok(SessionWindowUpdate::Advanced { new_window_end: window_end, .. }) + => { + let session = window_end; + if state.highest_session.map_or(true, |s| s < session) { + tracing::trace!( + target: LOG_TARGET, + session, + "Observed new session. Pruning", + ); + + state.highest_session = Some(session); + + db::v1::note_current_session( + store, + &config.column_config(), + session, + )?; + } + } + _ => {} + } + + // TODO [after https://github.com/paritytech/polkadot/issues/3160]: chain rollbacks + } + + Ok(()) +} + +async fn handle_incoming( + ctx: &mut impl SubsystemContext, + store: &dyn KeyValueDB, + state: &mut State, + config: &Config, + message: DisputeCoordinatorMessage, +) -> Result<(), Error> { + match message { + DisputeCoordinatorMessage::ImportStatements { + candidate_hash, + candidate_receipt, + session, + statements, + } => { + handle_import_statements( + ctx, + store, + state, + config, + candidate_hash, + candidate_receipt, + session, + statements, + ).await?; + } + DisputeCoordinatorMessage::ActiveDisputes(rx) => { + let active_disputes = db::v1::load_active_disputes(store, &config.column_config())? + .map(|d| d.disputed) + .unwrap_or_default(); + + let _ = rx.send(active_disputes); + } + DisputeCoordinatorMessage::QueryCandidateVotes( + session, + candidate_hash, + rx + ) => { + let candidate_votes = db::v1::load_candidate_votes( + store, + &config.column_config(), + session, + &candidate_hash, + )?; + + let _ = rx.send(candidate_votes.map(Into::into)); + } + DisputeCoordinatorMessage::IssueLocalStatement( + session, + candidate_hash, + candidate_receipt, + valid, + ) => { + issue_local_statement( + ctx, + state, + store, + config, + candidate_hash, + candidate_receipt, + session, + valid, + ).await?; + } + DisputeCoordinatorMessage::DetermineUndisputedChain { + base_number, + block_descriptions, + tx, + } => { + let undisputed_chain = determine_undisputed_chain( + store, + &config, + base_number, + block_descriptions + )?; + + let _ = tx.send(undisputed_chain); + } + } + + Ok(()) +} + +fn insert_into_statement_vec( + vec: &mut Vec<(T, ValidatorIndex, ValidatorSignature)>, + tag: T, + val_index: ValidatorIndex, + val_signature: ValidatorSignature, +) { + let pos = match vec.binary_search_by_key(&val_index, |x| x.1) { + Ok(_) => return, // no duplicates needed. + Err(p) => p, + }; + + vec.insert(pos, (tag, val_index, val_signature)); +} + +async fn handle_import_statements( + ctx: &mut impl SubsystemContext, + store: &dyn KeyValueDB, + state: &mut State, + config: &Config, + candidate_hash: CandidateHash, + candidate_receipt: CandidateReceipt, + session: SessionIndex, + statements: Vec<(SignedDisputeStatement, ValidatorIndex)>, +) -> Result<(), Error> { + if state.highest_session.map_or(true, |h| session + DISPUTE_WINDOW < h) { + return Ok(()); + } + + let validators = match state.rolling_session_window.session_info(session) { + None => { + tracing::warn!( + target: LOG_TARGET, + session, + "Missing info for session which has an active dispute", + ); + + return Ok(()) + } + Some(info) => info.validators.clone(), + }; + + let n_validators = validators.len(); + + let supermajority_threshold = polkadot_primitives::v1::supermajority_threshold(n_validators); + + let mut votes = db::v1::load_candidate_votes( + store, + &config.column_config(), + session, + &candidate_hash + )? + .map(CandidateVotes::from) + .unwrap_or_else(|| CandidateVotes { + candidate_receipt: candidate_receipt.clone(), + valid: Vec::new(), + invalid: Vec::new(), + }); + + let was_undisputed = votes.valid.is_empty() || votes.invalid.is_empty(); + + // Update candidate votes. + for (statement, val_index) in statements { + if validators.get(val_index.0 as usize) + .map_or(true, |v| v != statement.validator_public()) + { + tracing::debug!( + target: LOG_TARGET, + ?val_index, + session, + claimed_key = ?statement.validator_public(), + "Validator index doesn't match claimed key", + ); + + continue + } + + match statement.statement().clone() { + DisputeStatement::Valid(valid_kind) => { + insert_into_statement_vec( + &mut votes.valid, + valid_kind, + val_index, + statement.validator_signature().clone(), + ); + } + DisputeStatement::Invalid(invalid_kind) => { + insert_into_statement_vec( + &mut votes.invalid, + invalid_kind, + val_index, + statement.validator_signature().clone(), + ); + } + } + } + + // Check if newly disputed. + let is_disputed = !votes.valid.is_empty() && !votes.invalid.is_empty(); + let freshly_disputed = is_disputed && was_undisputed; + let already_disputed = is_disputed && !was_undisputed; + let concluded_valid = votes.valid.len() >= supermajority_threshold; + + let mut tx = db::v1::Transaction::default(); + + if freshly_disputed && !concluded_valid { + // add to active disputes and begin local participation. + update_active_disputes( + store, + config, + &mut tx, + |active| active.insert(session, candidate_hash), + )?; + + ctx.send_message(DisputeParticipationMessage::Participate { + candidate_hash, + candidate_receipt, + session, + n_validators: n_validators as u32, + }.into()).await; + } + + if concluded_valid && already_disputed { + // remove from active disputes. + update_active_disputes( + store, + config, + &mut tx, + |active| active.delete(session, candidate_hash), + )?; + } + + tx.put_candidate_votes(session, candidate_hash, votes.into()); + tx.write(store, &config.column_config())?; + + Ok(()) +} + +fn update_active_disputes( + store: &dyn KeyValueDB, + config: &Config, + tx: &mut db::v1::Transaction, + with_active: impl FnOnce(&mut ActiveDisputes) -> bool, +) -> Result<(), Error> { + let mut active_disputes = db::v1::load_active_disputes(store, &config.column_config())? + .unwrap_or_default(); + + if with_active(&mut active_disputes) { + tx.put_active_disputes(active_disputes); + } + + Ok(()) +} + +async fn issue_local_statement( + ctx: &mut impl SubsystemContext, + state: &mut State, + store: &dyn KeyValueDB, + config: &Config, + candidate_hash: CandidateHash, + candidate_receipt: CandidateReceipt, + session: SessionIndex, + valid: bool, +) -> Result<(), Error> { + // Load session info. + let validators = match state.rolling_session_window.session_info(session) { + None => { + tracing::warn!( + target: LOG_TARGET, + session, + "Missing info for session which has an active dispute", + ); + + return Ok(()) + } + Some(info) => info.validators.clone(), + }; + + let votes = db::v1::load_candidate_votes( + store, + &config.column_config(), + session, + &candidate_hash + )? + .map(CandidateVotes::from) + .unwrap_or_else(|| CandidateVotes { + candidate_receipt: candidate_receipt.clone(), + valid: Vec::new(), + invalid: Vec::new(), + }); + + // Sign a statement for each validator index we control which has + // not already voted. This should generally be maximum 1 statement. + let voted_indices = votes.voted_indices(); + let mut statements = Vec::new(); + + let voted_indices: HashSet<_> = voted_indices.into_iter().collect(); + for (index, validator) in validators.iter().enumerate() { + let index = ValidatorIndex(index as _); + if voted_indices.contains(&index) { continue } + if state.keystore.key_pair::(validator).ok().flatten().is_none() { + continue + } + + let keystore = state.keystore.clone() as Arc<_>; + let res = SignedDisputeStatement::sign_explicit( + &keystore, + valid, + candidate_hash, + session, + validator.clone(), + ).await; + + match res { + Ok(Some(signed_dispute_statement)) => { + statements.push((signed_dispute_statement, index)); + } + Ok(None) => {} + Err(e) => { + tracing::error!( + target: LOG_TARGET, + err = ?e, + "Encountered keystore error while signing dispute statement", + ); + } + } + } + + // Do import + if !statements.is_empty() { + handle_import_statements( + ctx, + store, + state, + config, + candidate_hash, + candidate_receipt, + session, + statements, + ).await?; + } + + Ok(()) +} + +fn determine_undisputed_chain( + store: &dyn KeyValueDB, + config: &Config, + base_number: BlockNumber, + block_descriptions: Vec<(Hash, SessionIndex, Vec)>, +) -> Result, Error> { + let last = block_descriptions.last() + .map(|e| (base_number + block_descriptions.len() as BlockNumber, e.0)); + + // Fast path for no disputes. + let active_disputes = match db::v1::load_active_disputes(store, &config.column_config())? { + None => return Ok(last), + Some(a) if a.disputed.is_empty() => return Ok(last), + Some(a) => a, + }; + + for (i, (_, session, candidates)) in block_descriptions.iter().enumerate() { + if candidates.iter().any(|c| active_disputes.contains(*session, *c)) { + if i == 0 { + return Ok(None); + } else { + return Ok(Some(( + base_number + i as BlockNumber, + block_descriptions[i - 1].0, + ))); + } + } + } + + Ok(last) +} diff --git a/node/core/dispute-coordinator/src/tests.rs b/node/core/dispute-coordinator/src/tests.rs new file mode 100644 index 000000000000..dc8c93de968d --- /dev/null +++ b/node/core/dispute-coordinator/src/tests.rs @@ -0,0 +1,706 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + + +use super::*; +use polkadot_primitives::v1::{BlakeTwo256, HashT, ValidatorId, Header, SessionInfo}; +use polkadot_node_subsystem::{jaeger, ActiveLeavesUpdate, ActivatedLeaf, LeafStatus}; +use polkadot_node_subsystem::messages::{ + AllMessages, ChainApiMessage, RuntimeApiMessage, RuntimeApiRequest, +}; +use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle}; +use sp_core::testing::TaskExecutor; +use sp_keyring::Sr25519Keyring; +use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; +use futures::future::{self, BoxFuture}; +use parity_scale_codec::Encode; +use assert_matches::assert_matches; + +// sets up a keystore with the given keyring accounts. +fn make_keystore(accounts: &[Sr25519Keyring]) -> LocalKeystore { + let store = LocalKeystore::in_memory(); + + for s in accounts.iter().copied().map(|k| k.to_seed()) { + store.sr25519_generate_new( + polkadot_primitives::v1::PARACHAIN_KEY_TYPE_ID, + Some(s.as_str()), + ).unwrap(); + } + + store +} + +fn session_to_hash(session: SessionIndex, extra: impl Encode) -> Hash { + BlakeTwo256::hash_of(&(session, extra)) +} + +type VirtualOverseer = TestSubsystemContextHandle; + +struct TestState { + validators: Vec, + validator_public: Vec, + validator_groups: Vec>, + master_keystore: Arc, + subsystem_keystore: Arc, + db: Arc, + config: Config, +} + +impl Default for TestState { + fn default() -> TestState { + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Eve, + Sr25519Keyring::One, + ]; + + let validator_public = validators.iter() + .map(|k| ValidatorId::from(k.public())) + .collect(); + + let validator_groups = vec![ + vec![ValidatorIndex(0), ValidatorIndex(1)], + vec![ValidatorIndex(2), ValidatorIndex(3)], + vec![ValidatorIndex(4), ValidatorIndex(5)], + ]; + + let master_keystore = make_keystore(&validators).into(); + let subsystem_keystore = make_keystore(&[Sr25519Keyring::Alice]).into(); + + let db = Arc::new(kvdb_memorydb::create(1)); + let config = Config { + col_data: 0, + }; + + TestState { + validators, + validator_public, + validator_groups, + master_keystore, + subsystem_keystore, + db, + config, + } + } +} + +impl TestState { + async fn activate_leaf_at_session( + &self, + virtual_overseer: &mut VirtualOverseer, + session: SessionIndex, + block_number: BlockNumber, + ) { + assert!(block_number > 0); + + let parent_hash = session_to_hash(session, b"parent"); + let block_header = Header { + parent_hash, + number: block_number, + digest: Default::default(), + state_root: Default::default(), + extrinsics_root: Default::default(), + }; + let block_hash = block_header.hash(); + + virtual_overseer.send(FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: block_hash, + span: Arc::new(jaeger::Span::Disabled), + number: block_number, + status: LeafStatus::Fresh, + }) + ))).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::ChainApi(ChainApiMessage::BlockHeader(h, tx)) => { + assert_eq!(h, block_hash); + let _ = tx.send(Ok(Some(block_header))); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + h, + RuntimeApiRequest::SessionIndexForChild(tx), + )) => { + assert_eq!(h, parent_hash); + let _ = tx.send(Ok(session)); + } + ); + + loop { + // answer session info queries until the current session is reached. + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + h, + RuntimeApiRequest::SessionInfo(session_index, tx), + )) => { + assert_eq!(h, block_hash); + + let _ = tx.send(Ok(Some(self.session_info()))); + if session_index == session { break } + } + ) + } + } + + fn session_info(&self) -> SessionInfo { + let discovery_keys = self.validators.iter() + .map(|k| <_>::from(k.public())) + .collect(); + + let assignment_keys = self.validators.iter() + .map(|k| <_>::from(k.public())) + .collect(); + + SessionInfo { + validators: self.validator_public.clone(), + discovery_keys, + assignment_keys, + validator_groups: self.validator_groups.clone(), + n_cores: self.validator_groups.len() as _, + zeroth_delay_tranche_width: 0, + relay_vrf_modulo_samples: 1, + n_delay_tranches: 100, + no_show_slots: 1, + needed_approvals: 10, + } + } + + async fn issue_statement_with_index( + &self, + index: usize, + candidate_hash: CandidateHash, + session: SessionIndex, + valid: bool, + ) -> SignedDisputeStatement { + let public = self.validator_public[index].clone(); + + let keystore = self.master_keystore.clone() as SyncCryptoStorePtr; + + SignedDisputeStatement::sign_explicit( + &keystore, + valid, + candidate_hash, + session, + public, + ).await.unwrap().unwrap() + } +} + +fn test_harness(test: F) + where F: FnOnce(TestState, VirtualOverseer) -> BoxFuture<'static, ()> +{ + let (ctx, ctx_handle) = make_subsystem_context(TaskExecutor::new()); + + let state = TestState::default(); + let subsystem = DisputeCoordinatorSubsystem::new( + state.db.clone(), + state.config.clone(), + state.subsystem_keystore.clone(), + ); + + let subsystem_task = run(subsystem, ctx); + let test_task = test(state, ctx_handle); + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn conflicting_votes_lead_to_dispute_participation() { + test_harness(|test_state, mut virtual_overseer| Box::pin(async move { + let session = 1; + + let candidate_receipt = CandidateReceipt::default(); + let candidate_hash = candidate_receipt.hash(); + + test_state.activate_leaf_at_session( + &mut virtual_overseer, + session, + 1, + ).await; + + let valid_vote = test_state.issue_statement_with_index( + 0, + candidate_hash, + session, + true, + ).await; + + let invalid_vote = test_state.issue_statement_with_index( + 1, + candidate_hash, + session, + false, + ).await; + + let invalid_vote_2 = test_state.issue_statement_with_index( + 2, + candidate_hash, + session, + false, + ).await; + + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ImportStatements { + candidate_hash, + candidate_receipt: candidate_receipt.clone(), + session, + statements: vec![ + (valid_vote, ValidatorIndex(0)), + (invalid_vote, ValidatorIndex(1)), + ], + }, + }).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::DisputeParticipation(DisputeParticipationMessage::Participate { + candidate_hash: c_hash, + candidate_receipt: c_receipt, + session: s, + n_validators, + }) => { + assert_eq!(c_hash, candidate_hash); + assert_eq!(c_receipt, candidate_receipt); + assert_eq!(s, session); + assert_eq!(n_validators, test_state.validators.len() as u32); + } + ); + + { + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ActiveDisputes(tx), + }).await; + + assert_eq!(rx.await.unwrap(), vec![(session, candidate_hash)]); + + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::QueryCandidateVotes( + session, + candidate_hash, + tx, + ), + }).await; + + let votes = rx.await.unwrap().unwrap(); + assert_eq!(votes.valid.len(), 1); + assert_eq!(votes.invalid.len(), 1); + } + + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ImportStatements { + candidate_hash, + candidate_receipt: candidate_receipt.clone(), + session, + statements: vec![ + (invalid_vote_2, ValidatorIndex(2)), + ], + }, + }).await; + + { + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::QueryCandidateVotes( + session, + candidate_hash, + tx, + ), + }).await; + + let votes = rx.await.unwrap().unwrap(); + assert_eq!(votes.valid.len(), 1); + assert_eq!(votes.invalid.len(), 2); + } + + virtual_overseer.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + + // This confirms that the second vote doesn't lead to participation again. + assert!(virtual_overseer.try_recv().await.is_none()); + })); +} + +#[test] +fn positive_votes_dont_trigger_participation() { + test_harness(|test_state, mut virtual_overseer| Box::pin(async move { + let session = 1; + + let candidate_receipt = CandidateReceipt::default(); + let candidate_hash = candidate_receipt.hash(); + + test_state.activate_leaf_at_session( + &mut virtual_overseer, + session, + 1, + ).await; + + let valid_vote = test_state.issue_statement_with_index( + 0, + candidate_hash, + session, + true, + ).await; + + let valid_vote_2 = test_state.issue_statement_with_index( + 1, + candidate_hash, + session, + true, + ).await; + + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ImportStatements { + candidate_hash, + candidate_receipt: candidate_receipt.clone(), + session, + statements: vec![ + (valid_vote, ValidatorIndex(0)), + ], + }, + }).await; + + { + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ActiveDisputes(tx), + }).await; + + assert!(rx.await.unwrap().is_empty()); + + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::QueryCandidateVotes( + session, + candidate_hash, + tx, + ), + }).await; + + let votes = rx.await.unwrap().unwrap(); + assert_eq!(votes.valid.len(), 1); + assert!(votes.invalid.is_empty()); + } + + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ImportStatements { + candidate_hash, + candidate_receipt: candidate_receipt.clone(), + session, + statements: vec![ + (valid_vote_2, ValidatorIndex(1)), + ], + }, + }).await; + + { + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ActiveDisputes(tx), + }).await; + + assert!(rx.await.unwrap().is_empty()); + + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::QueryCandidateVotes( + session, + candidate_hash, + tx, + ), + }).await; + + let votes = rx.await.unwrap().unwrap(); + assert_eq!(votes.valid.len(), 2); + assert!(votes.invalid.is_empty()); + } + + virtual_overseer.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + + // This confirms that no participation request is made. + assert!(virtual_overseer.try_recv().await.is_none()); + })); +} + +#[test] +fn wrong_validator_index_is_ignored() { + test_harness(|test_state, mut virtual_overseer| Box::pin(async move { + let session = 1; + + let candidate_receipt = CandidateReceipt::default(); + let candidate_hash = candidate_receipt.hash(); + + test_state.activate_leaf_at_session( + &mut virtual_overseer, + session, + 1, + ).await; + + let valid_vote = test_state.issue_statement_with_index( + 0, + candidate_hash, + session, + true, + ).await; + + let invalid_vote = test_state.issue_statement_with_index( + 1, + candidate_hash, + session, + false, + ).await; + + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ImportStatements { + candidate_hash, + candidate_receipt: candidate_receipt.clone(), + session, + statements: vec![ + (valid_vote, ValidatorIndex(1)), + (invalid_vote, ValidatorIndex(0)), + ], + }, + }).await; + + { + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ActiveDisputes(tx), + }).await; + + assert!(rx.await.unwrap().is_empty()); + + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::QueryCandidateVotes( + session, + candidate_hash, + tx, + ), + }).await; + + let votes = rx.await.unwrap().unwrap(); + assert!(votes.valid.is_empty()); + assert!(votes.invalid.is_empty()); + } + + virtual_overseer.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + + // This confirms that no participation request is made. + assert!(virtual_overseer.try_recv().await.is_none()); + })); +} + +#[test] +fn finality_votes_ignore_disputed_candidates() { + test_harness(|test_state, mut virtual_overseer| Box::pin(async move { + let session = 1; + + let candidate_receipt = CandidateReceipt::default(); + let candidate_hash = candidate_receipt.hash(); + + test_state.activate_leaf_at_session( + &mut virtual_overseer, + session, + 1, + ).await; + + let valid_vote = test_state.issue_statement_with_index( + 0, + candidate_hash, + session, + true, + ).await; + + let invalid_vote = test_state.issue_statement_with_index( + 1, + candidate_hash, + session, + false, + ).await; + + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ImportStatements { + candidate_hash, + candidate_receipt: candidate_receipt.clone(), + session, + statements: vec![ + (valid_vote, ValidatorIndex(0)), + (invalid_vote, ValidatorIndex(1)), + ], + }, + }).await; + let _ = virtual_overseer.recv().await; + + { + let (tx, rx) = oneshot::channel(); + + let block_hash_a = Hash::repeat_byte(0x0a); + let block_hash_b = Hash::repeat_byte(0x0b); + + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::DetermineUndisputedChain { + base_number: 10, + block_descriptions: vec![ + (block_hash_a, session, vec![candidate_hash]), + ], + tx, + }, + }).await; + + assert!(rx.await.unwrap().is_none()); + + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::DetermineUndisputedChain { + base_number: 10, + block_descriptions: vec![ + (block_hash_a, session, vec![]), + (block_hash_b, session, vec![candidate_hash]), + ], + tx, + }, + }).await; + + assert_eq!(rx.await.unwrap(), Some((11, block_hash_a))); + } + + virtual_overseer.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + assert!(virtual_overseer.try_recv().await.is_none()); + })); +} + +#[test] +fn supermajority_valid_dispute_may_be_finalized() { + test_harness(|test_state, mut virtual_overseer| Box::pin(async move { + let session = 1; + + let candidate_receipt = CandidateReceipt::default(); + let candidate_hash = candidate_receipt.hash(); + + test_state.activate_leaf_at_session( + &mut virtual_overseer, + session, + 1, + ).await; + + let supermajority_threshold = polkadot_primitives::v1::supermajority_threshold( + test_state.validators.len() + ); + + let valid_vote = test_state.issue_statement_with_index( + 0, + candidate_hash, + session, + true, + ).await; + + let invalid_vote = test_state.issue_statement_with_index( + 1, + candidate_hash, + session, + false, + ).await; + + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ImportStatements { + candidate_hash, + candidate_receipt: candidate_receipt.clone(), + session, + statements: vec![ + (valid_vote, ValidatorIndex(0)), + (invalid_vote, ValidatorIndex(1)), + ], + }, + }).await; + + let _ = virtual_overseer.recv().await; + + let mut statements = Vec::new(); + for i in (0..supermajority_threshold - 1).map(|i| i + 2) { + let vote = test_state.issue_statement_with_index( + i, + candidate_hash, + session, + true, + ).await; + + statements.push((vote, ValidatorIndex(i as _))); + }; + + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ImportStatements { + candidate_hash, + candidate_receipt: candidate_receipt.clone(), + session, + statements, + }, + }).await; + + { + let (tx, rx) = oneshot::channel(); + + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::ActiveDisputes(tx), + }).await; + + assert!(rx.await.unwrap().is_empty()); + + let (tx, rx) = oneshot::channel(); + + let block_hash_a = Hash::repeat_byte(0x0a); + let block_hash_b = Hash::repeat_byte(0x0b); + + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::DetermineUndisputedChain { + base_number: 10, + block_descriptions: vec![ + (block_hash_a, session, vec![candidate_hash]), + ], + tx, + }, + }).await; + + assert_eq!(rx.await.unwrap(), Some((11, block_hash_a))); + + let (tx, rx) = oneshot::channel(); + virtual_overseer.send(FromOverseer::Communication { + msg: DisputeCoordinatorMessage::DetermineUndisputedChain { + base_number: 10, + block_descriptions: vec![ + (block_hash_a, session, vec![]), + (block_hash_b, session, vec![candidate_hash]), + ], + tx, + }, + }).await; + + assert_eq!(rx.await.unwrap(), Some((12, block_hash_b))); + } + + virtual_overseer.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + assert!(virtual_overseer.try_recv().await.is_none()); + })); +} diff --git a/node/core/dispute-participation/Cargo.toml b/node/core/dispute-participation/Cargo.toml new file mode 100644 index 000000000000..7de5c195b6d7 --- /dev/null +++ b/node/core/dispute-participation/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "polkadot-node-core-dispute-participation" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" + +[dependencies] +futures = "0.3.12" +thiserror = "1.0.23" +tracing = "0.1.26" + +polkadot-node-primitives = { path = "../../primitives" } +polkadot-node-subsystem = { path = "../../subsystem" } +polkadot-primitives = { path = "../../../primitives" } + +[dev-dependencies] +assert_matches = "1.5.0" +parity-scale-codec = "2.0.0" +polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers"} +sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/core/dispute-participation/src/lib.rs b/node/core/dispute-participation/src/lib.rs new file mode 100644 index 000000000000..19bc56a020a6 --- /dev/null +++ b/node/core/dispute-participation/src/lib.rs @@ -0,0 +1,372 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Implements the dispute participation subsystem. +//! +//! This subsystem is responsible for actually participating in disputes: when +//! notified of a dispute, we recover the candidate data, validate the +//! candidate, and cast our vote in the dispute. + +use futures::channel::oneshot; +use futures::prelude::*; + +use polkadot_node_primitives::ValidationResult; +use polkadot_node_subsystem::{ + errors::{RecoveryError, RuntimeApiError}, + messages::{ + AllMessages, AvailabilityRecoveryMessage, AvailabilityStoreMessage, + CandidateValidationMessage, DisputeCoordinatorMessage, DisputeParticipationMessage, + RuntimeApiMessage, RuntimeApiRequest, + }, + ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, Subsystem, + SubsystemContext, SubsystemError, +}; +use polkadot_primitives::v1::{BlockNumber, CandidateHash, CandidateReceipt, Hash, SessionIndex}; + +#[cfg(test)] +mod tests; + +const LOG_TARGET: &str = "parachain::dispute-participation"; + +struct State { + recent_block: Option<(BlockNumber, Hash)>, +} + +/// An implementation of the dispute participation subsystem. +pub struct DisputeParticipationSubsystem; + +impl DisputeParticipationSubsystem { + /// Create a new instance of the subsystem. + pub fn new() -> Self { + DisputeParticipationSubsystem + } +} + +impl Subsystem for DisputeParticipationSubsystem +where + Context: SubsystemContext, +{ + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = run(ctx).map(|_| Ok(())).boxed(); + + SpawnedSubsystem { + name: "dispute-participation-subsystem", + future, + } + } +} + +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] +pub enum Error { + #[error(transparent)] + RuntimeApi(#[from] RuntimeApiError), + + #[error(transparent)] + Subsystem(#[from] SubsystemError), + + #[error(transparent)] + Oneshot(#[from] oneshot::Canceled), + + #[error(transparent)] + Participation(#[from] ParticipationError), +} + +#[derive(Debug, thiserror::Error)] +pub enum ParticipationError { + #[error("Missing recent block state to participate in dispute")] + MissingRecentBlockState, + #[error("Failed to recover available data for candidate {0}")] + MissingAvailableData(CandidateHash), + #[error("Failed to recover validation code for candidate {0}")] + MissingValidationCode(CandidateHash), +} + +impl Error { + fn trace(&self) { + match self { + // don't spam the log with spurious errors + Self::RuntimeApi(_) | Self::Oneshot(_) => { + tracing::debug!(target: LOG_TARGET, err = ?self) + } + // it's worth reporting otherwise + _ => tracing::warn!(target: LOG_TARGET, err = ?self), + } + } +} + +async fn run(mut ctx: Context) +where + Context: SubsystemContext, +{ + let mut state = State { recent_block: None }; + + loop { + match ctx.recv().await { + Err(_) => return, + Ok(FromOverseer::Signal(OverseerSignal::Conclude)) => { + tracing::info!(target: LOG_TARGET, "Received `Conclude` signal, exiting"); + return; + } + Ok(FromOverseer::Signal(OverseerSignal::BlockFinalized(_, _))) => {} + Ok(FromOverseer::Signal(OverseerSignal::ActiveLeaves(update))) => { + update_state(&mut state, update); + } + Ok(FromOverseer::Communication { msg }) => { + if let Err(err) = handle_incoming(&mut ctx, &mut state, msg).await { + err.trace(); + if let Error::Subsystem(SubsystemError::Context(_)) = err { + return; + } + } + } + } + } +} + +fn update_state(state: &mut State, update: ActiveLeavesUpdate) { + for active in update.activated { + if state.recent_block.map_or(true, |s| active.number > s.0) { + state.recent_block = Some((active.number, active.hash)); + } + } +} + +async fn handle_incoming( + ctx: &mut impl SubsystemContext, + state: &mut State, + message: DisputeParticipationMessage, +) -> Result<(), Error> { + match message { + DisputeParticipationMessage::Participate { + candidate_hash, + candidate_receipt, + session, + n_validators, + } => { + if let Some((_, block_hash)) = state.recent_block { + participate( + ctx, + block_hash, + candidate_hash, + candidate_receipt, + session, + n_validators, + ) + .await + } else { + return Err(ParticipationError::MissingRecentBlockState.into()); + } + } + } +} + +async fn participate( + ctx: &mut impl SubsystemContext, + block_hash: Hash, + candidate_hash: CandidateHash, + candidate_receipt: CandidateReceipt, + session: SessionIndex, + n_validators: u32, +) -> Result<(), Error> { + let (recover_available_data_tx, recover_available_data_rx) = oneshot::channel(); + let (code_tx, code_rx) = oneshot::channel(); + let (store_available_data_tx, store_available_data_rx) = oneshot::channel(); + let (validation_tx, validation_rx) = oneshot::channel(); + + // in order to validate a candidate we need to start by recovering the + // available data + ctx.send_message( + AvailabilityRecoveryMessage::RecoverAvailableData( + candidate_receipt.clone(), + session, + None, + recover_available_data_tx, + ) + .into(), + ) + .await; + + let available_data = match recover_available_data_rx.await? { + Ok(data) => data, + Err(RecoveryError::Invalid) => { + // the available data was recovered but it is invalid, therefore we'll + // vote negatively for the candidate dispute + cast_invalid_vote(ctx, candidate_hash, candidate_receipt, session).await; + return Ok(()); + } + Err(RecoveryError::Unavailable) => { + return Err(ParticipationError::MissingAvailableData(candidate_hash).into()); + } + }; + + // we also need to fetch the validation code which we can reference by its + // hash as taken from the candidate descriptor + ctx.send_message( + RuntimeApiMessage::Request( + block_hash, + RuntimeApiRequest::ValidationCodeByHash( + candidate_receipt.descriptor.validation_code_hash, + code_tx, + ), + ) + .into(), + ) + .await; + + let validation_code = match code_rx.await?? { + Some(code) => code, + None => { + tracing::warn!( + target: LOG_TARGET, + "Validation code unavailable for code hash {:?} in the state of block {:?}", + candidate_receipt.descriptor.validation_code_hash, + block_hash, + ); + + return Err(ParticipationError::MissingValidationCode(candidate_hash).into()); + } + }; + + // we dispatch a request to store the available data for the candidate. we + // want to maximize data availability for other potential checkers involved + // in the dispute + ctx.send_message( + AvailabilityStoreMessage::StoreAvailableData( + candidate_hash, + None, + n_validators, + available_data.clone(), + store_available_data_tx, + ) + .into(), + ) + .await; + + match store_available_data_rx.await? { + Err(_) => { + tracing::warn!( + target: LOG_TARGET, + "Failed to store available data for candidate {:?}", + candidate_hash, + ); + } + Ok(()) => {} + } + + // we issue a request to validate the candidate with the provided exhaustive + // parameters + ctx.send_message( + CandidateValidationMessage::ValidateFromExhaustive( + available_data.validation_data, + validation_code, + candidate_receipt.descriptor.clone(), + available_data.pov, + validation_tx, + ) + .into(), + ) + .await; + + // we cast votes (either positive or negative) depending on the outcome of + // the validation and if valid, whether the commitments hash matches + match validation_rx.await? { + Err(err) => { + tracing::warn!( + target: LOG_TARGET, + "Candidate {:?} validation failed with: {:?}", + candidate_receipt.hash(), + err, + ); + + cast_invalid_vote(ctx, candidate_hash, candidate_receipt, session).await; + } + Ok(ValidationResult::Invalid(invalid)) => { + tracing::warn!( + target: LOG_TARGET, + "Candidate {:?} considered invalid: {:?}", + candidate_hash, + invalid, + ); + + cast_invalid_vote(ctx, candidate_hash, candidate_receipt, session).await; + } + Ok(ValidationResult::Valid(commitments, _)) => { + if commitments.hash() != candidate_receipt.commitments_hash { + tracing::warn!( + target: LOG_TARGET, + expected = ?candidate_receipt.commitments_hash, + got = ?commitments.hash(), + "Candidate is valid but commitments hash doesn't match", + ); + + cast_invalid_vote(ctx, candidate_hash, candidate_receipt, session).await; + } else { + cast_valid_vote(ctx, candidate_hash, candidate_receipt, session).await; + } + } + } + + Ok(()) +} + +async fn cast_valid_vote( + ctx: &mut impl SubsystemContext, + candidate_hash: CandidateHash, + candidate_receipt: CandidateReceipt, + session: SessionIndex, +) { + tracing::info!( + target: LOG_TARGET, + "Casting valid vote in dispute for candidate {:?}", + candidate_hash, + ); + + issue_local_statement(ctx, candidate_hash, candidate_receipt, session, true).await; +} + +async fn cast_invalid_vote( + ctx: &mut impl SubsystemContext, + candidate_hash: CandidateHash, + candidate_receipt: CandidateReceipt, + session: SessionIndex, +) { + tracing::info!( + target: LOG_TARGET, + "Casting invalid vote in dispute for candidate {:?}", + candidate_hash, + ); + + issue_local_statement(ctx, candidate_hash, candidate_receipt, session, false).await; +} + +async fn issue_local_statement( + ctx: &mut impl SubsystemContext, + candidate_hash: CandidateHash, + candidate_receipt: CandidateReceipt, + session: SessionIndex, + valid: bool, +) { + ctx.send_message(AllMessages::DisputeCoordinator( + DisputeCoordinatorMessage::IssueLocalStatement( + session, + candidate_hash, + candidate_receipt, + valid, + ), + )) + .await +} diff --git a/node/core/dispute-participation/src/tests.rs b/node/core/dispute-participation/src/tests.rs new file mode 100644 index 000000000000..2b086c43d179 --- /dev/null +++ b/node/core/dispute-participation/src/tests.rs @@ -0,0 +1,425 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use assert_matches::assert_matches; +use futures::future::{self, BoxFuture}; +use std::sync::Arc; + +use sp_core::testing::TaskExecutor; + +use super::*; +use parity_scale_codec::Encode; +use polkadot_node_primitives::{AvailableData, BlockData, InvalidCandidate, PoV}; +use polkadot_node_subsystem::{ + jaeger, messages::ValidationFailed, ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, +}; +use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle}; +use polkadot_primitives::v1::{BlakeTwo256, CandidateCommitments, HashT, Header, ValidationCode}; + +type VirtualOverseer = TestSubsystemContextHandle; + +fn test_harness(test: F) +where + F: FnOnce(VirtualOverseer) -> BoxFuture<'static, VirtualOverseer>, +{ + let (ctx, ctx_handle) = make_subsystem_context(TaskExecutor::new()); + + let subsystem = DisputeParticipationSubsystem::new(); + let spawned_subsystem = subsystem.start(ctx); + let test_future = test(ctx_handle); + + let (subsystem_result, _) = + futures::executor::block_on(future::join(spawned_subsystem.future, async move { + let mut ctx_handle = test_future.await; + ctx_handle + .send(FromOverseer::Signal(OverseerSignal::Conclude)) + .await; + + // no further request is received by the overseer which means that + // no further attempt to participate was made + assert!(ctx_handle.try_recv().await.is_none()); + })); + + subsystem_result.unwrap(); +} + +async fn activate_leaf(virtual_overseer: &mut VirtualOverseer, block_number: BlockNumber) { + let block_header = Header { + parent_hash: BlakeTwo256::hash(&block_number.encode()), + number: block_number, + digest: Default::default(), + state_root: Default::default(), + extrinsics_root: Default::default(), + }; + + let block_hash = block_header.hash(); + + virtual_overseer + .send(FromOverseer::Signal(OverseerSignal::ActiveLeaves( + ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: block_hash, + span: Arc::new(jaeger::Span::Disabled), + number: block_number, + status: LeafStatus::Fresh, + }), + ))) + .await; +} + +async fn participate(virtual_overseer: &mut VirtualOverseer) { + let commitments = CandidateCommitments::default(); + let candidate_receipt = { + let mut receipt = CandidateReceipt::default(); + receipt.commitments_hash = commitments.hash(); + receipt + }; + let candidate_hash = candidate_receipt.hash(); + let session = 1; + let n_validators = 10; + + virtual_overseer + .send(FromOverseer::Communication { + msg: DisputeParticipationMessage::Participate { + candidate_hash, + candidate_receipt: candidate_receipt.clone(), + session, + n_validators, + }, + }) + .await; +} + +async fn recover_available_data(virtual_overseer: &mut VirtualOverseer) { + let pov_block = PoV { + block_data: BlockData(Vec::new()), + }; + + let available_data = AvailableData { + pov: Arc::new(pov_block), + validation_data: Default::default(), + }; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityRecovery( + AvailabilityRecoveryMessage::RecoverAvailableData(_, _, _, tx) + ) => { + tx.send(Ok(available_data)).unwrap(); + }, + "overseer did not receive recover available data message", + ); +} + +async fn fetch_validation_code(virtual_overseer: &mut VirtualOverseer) { + let validation_code = ValidationCode(Vec::new()); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + _, + RuntimeApiRequest::ValidationCodeByHash( + _, + tx, + ) + )) => { + tx.send(Ok(Some(validation_code))).unwrap(); + }, + "overseer did not receive runtime api request for validation code", + ); +} + +async fn store_available_data(virtual_overseer: &mut VirtualOverseer, success: bool) { + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityStore(AvailabilityStoreMessage::StoreAvailableData( + _, + _, + _, + _, + tx, + )) => { + if success { + tx.send(Ok(())).unwrap(); + } else { + tx.send(Err(())).unwrap(); + } + }, + "overseer did not receive store available data request", + ); +} + +#[test] +fn cannot_participate_when_recent_block_state_is_missing() { + test_harness(|mut virtual_overseer| { + Box::pin(async move { + participate(&mut virtual_overseer).await; + + virtual_overseer + }) + }); + + test_harness(|mut virtual_overseer| { + Box::pin(async move { + activate_leaf(&mut virtual_overseer, 10).await; + participate(&mut virtual_overseer).await; + + // after activating at least one leaf the recent block + // state should be available which should lead to trying + // to participate by first trying to recover the available + // data + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityRecovery( + AvailabilityRecoveryMessage::RecoverAvailableData(..) + ), + "overseer did not receive recover available data message", + ); + + virtual_overseer + }) + }); +} + +#[test] +fn cannot_participate_if_cannot_recover_available_data() { + test_harness(|mut virtual_overseer| { + Box::pin(async move { + activate_leaf(&mut virtual_overseer, 10).await; + participate(&mut virtual_overseer).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityRecovery( + AvailabilityRecoveryMessage::RecoverAvailableData(_, _, _, tx) + ) => { + tx.send(Err(RecoveryError::Unavailable)).unwrap(); + }, + "overseer did not receive recover available data message", + ); + + virtual_overseer + }) + }); +} + +#[test] +fn cannot_participate_if_cannot_recover_validation_code() { + test_harness(|mut virtual_overseer| { + Box::pin(async move { + activate_leaf(&mut virtual_overseer, 10).await; + participate(&mut virtual_overseer).await; + recover_available_data(&mut virtual_overseer).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + _, + RuntimeApiRequest::ValidationCodeByHash( + _, + tx, + ) + )) => { + tx.send(Ok(None)).unwrap(); + }, + "overseer did not receive runtime api request for validation code", + ); + + virtual_overseer + }) + }); +} + +#[test] +fn cast_invalid_vote_if_available_data_is_invalid() { + test_harness(|mut virtual_overseer| { + Box::pin(async move { + activate_leaf(&mut virtual_overseer, 10).await; + participate(&mut virtual_overseer).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::AvailabilityRecovery( + AvailabilityRecoveryMessage::RecoverAvailableData(_, _, _, tx) + ) => { + tx.send(Err(RecoveryError::Invalid)).unwrap(); + }, + "overseer did not receive recover available data message", + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::DisputeCoordinator(DisputeCoordinatorMessage::IssueLocalStatement( + _, + _, + _, + false, + )), + "overseer did not receive issue local statement message", + ); + + virtual_overseer + }) + }); +} + +#[test] +fn cast_invalid_vote_if_validation_fails_or_is_invalid() { + test_harness(|mut virtual_overseer| { + Box::pin(async move { + activate_leaf(&mut virtual_overseer, 10).await; + participate(&mut virtual_overseer).await; + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + store_available_data(&mut virtual_overseer, true).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, tx) + ) => { + tx.send(Ok(ValidationResult::Invalid(InvalidCandidate::Timeout))).unwrap(); + }, + "overseer did not receive candidate validation message", + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::DisputeCoordinator(DisputeCoordinatorMessage::IssueLocalStatement( + _, + _, + _, + false, + )), + "overseer did not receive issue local statement message", + ); + + virtual_overseer + }) + }); +} + +#[test] +fn cast_invalid_vote_if_validation_passes_but_commitments_dont_match() { + test_harness(|mut virtual_overseer| { + Box::pin(async move { + activate_leaf(&mut virtual_overseer, 10).await; + participate(&mut virtual_overseer).await; + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + store_available_data(&mut virtual_overseer, true).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, tx) + ) => { + let mut commitments = CandidateCommitments::default(); + // this should lead to a commitments hash mismatch + commitments.processed_downward_messages = 42; + + tx.send(Ok(ValidationResult::Valid(commitments, Default::default()))).unwrap(); + }, + "overseer did not receive candidate validation message", + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::DisputeCoordinator(DisputeCoordinatorMessage::IssueLocalStatement( + _, + _, + _, + false, + )), + "overseer did not receive issue local statement message", + ); + + virtual_overseer + }) + }); +} + +#[test] +fn cast_valid_vote_if_validation_passes() { + test_harness(|mut virtual_overseer| { + Box::pin(async move { + activate_leaf(&mut virtual_overseer, 10).await; + participate(&mut virtual_overseer).await; + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + store_available_data(&mut virtual_overseer, true).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, tx) + ) => { + tx.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))).unwrap(); + }, + "overseer did not receive candidate validation message", + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::DisputeCoordinator(DisputeCoordinatorMessage::IssueLocalStatement( + _, + _, + _, + true, + )), + "overseer did not receive issue local statement message", + ); + + virtual_overseer + }) + }); +} + +#[test] +fn failure_to_store_available_data_does_not_preclude_participation() { + test_harness(|mut virtual_overseer| { + Box::pin(async move { + activate_leaf(&mut virtual_overseer, 10).await; + participate(&mut virtual_overseer).await; + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + // the store available data request should fail + store_available_data(&mut virtual_overseer, false).await; + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromExhaustive(_, _, _, _, tx) + ) => { + tx.send(Err(ValidationFailed("fail".to_string()))).unwrap(); + }, + "overseer did not receive candidate validation message", + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::DisputeCoordinator(DisputeCoordinatorMessage::IssueLocalStatement( + _, + _, + _, + false, + )), + "overseer did not receive issue local statement message", + ); + + virtual_overseer + }) + }); +} diff --git a/node/core/parachains-inherent/Cargo.toml b/node/core/parachains-inherent/Cargo.toml index a4156008dbbf..6dd3d3bbdab6 100644 --- a/node/core/parachains-inherent/Cargo.toml +++ b/node/core/parachains-inherent/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Parity Technologies "] edition = "2018" [dependencies] -futures = "0.3.12" +futures = "0.3.15" futures-timer = "3.0.2" tracing = "0.1.26" thiserror = "1.0.23" diff --git a/node/core/provisioner/Cargo.toml b/node/core/provisioner/Cargo.toml index 492f37c51d69..fb6259f6878e 100644 --- a/node/core/provisioner/Cargo.toml +++ b/node/core/provisioner/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] bitvec = { version = "0.20.1", default-features = false, features = ["alloc"] } -futures = "0.3.12" +futures = "0.3.15" tracing = "0.1.26" thiserror = "1.0.23" polkadot-primitives = { path = "../../../primitives" } diff --git a/node/core/provisioner/src/lib.rs b/node/core/provisioner/src/lib.rs index 36ef5ffdd7c8..7a730aa8cfdf 100644 --- a/node/core/provisioner/src/lib.rs +++ b/node/core/provisioner/src/lib.rs @@ -43,6 +43,9 @@ use std::{pin::Pin, collections::BTreeMap, sync::Arc}; use thiserror::Error; use futures_timer::Delay; +#[cfg(test)] +mod tests; + /// How long to wait before proposing. const PRE_PROPOSE_TIMEOUT: std::time::Duration = core::time::Duration::from_millis(2000); @@ -140,7 +143,6 @@ impl JobTrait for ProvisioningJob { /// Run a job for the parent block indicated // // this function is in charge of creating and executing the job's main loop - #[tracing::instrument(skip(span, _run_args, metrics, receiver, sender), fields(subsystem = LOG_TARGET))] fn run( relay_parent: Hash, span: Arc, @@ -181,7 +183,7 @@ impl ProvisioningJob { async fn run_loop( mut self, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, span: PerLeafSpan, ) -> Result<(), Error> { use ProvisionerMessage::{ @@ -223,7 +225,7 @@ impl ProvisioningJob { async fn send_inherent_data( &mut self, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, return_senders: Vec>, ) { if let Err(err) = send_inherent_data( @@ -242,7 +244,6 @@ impl ProvisioningJob { } } - #[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))] fn note_provisionable_data(&mut self, span: &jaeger::Span, provisionable_data: ProvisionableData) { match provisionable_data { ProvisionableData::Bitfield(_, signed_bitfield) => { @@ -277,13 +278,12 @@ type CoreAvailability = BitVec; /// When we're choosing bitfields to include, the rule should be simple: /// maximize availability. So basically, include all bitfields. And then /// choose a coherent set of candidates along with that. -#[tracing::instrument(level = "trace", skip(return_senders, from_job), fields(subsystem = LOG_TARGET))] async fn send_inherent_data( relay_parent: Hash, bitfields: &[SignedAvailabilityBitfield], candidates: &[CandidateReceipt], return_senders: Vec>, - from_job: &mut impl SubsystemSender, + from_job: &mut impl SubsystemSender, ) -> Result<(), Error> { let availability_cores = request_availability_cores(relay_parent, from_job) .await @@ -321,7 +321,6 @@ async fn send_inherent_data( /// /// Note: This does not enforce any sorting precondition on the output; the ordering there will be unrelated /// to the sorting of the input. -#[tracing::instrument(level = "trace", fields(subsystem = LOG_TARGET))] fn select_availability_bitfields( cores: &[CoreState], bitfields: &[SignedAvailabilityBitfield], @@ -353,13 +352,12 @@ fn select_availability_bitfields( } /// Determine which cores are free, and then to the degree possible, pick a candidate appropriate to each free core. -#[tracing::instrument(level = "trace", skip(sender), fields(subsystem = LOG_TARGET))] async fn select_candidates( availability_cores: &[CoreState], bitfields: &[SignedAvailabilityBitfield], candidates: &[CandidateReceipt], relay_parent: Hash, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) -> Result, Error> { let block_number = get_block_number_under_construction(relay_parent, sender).await?; @@ -475,10 +473,9 @@ async fn select_candidates( /// Produces a block number 1 higher than that of the relay parent /// in the event of an invalid `relay_parent`, returns `Ok(0)` -#[tracing::instrument(level = "trace", skip(sender), fields(subsystem = LOG_TARGET))] async fn get_block_number_under_construction( relay_parent: Hash, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) -> Result { let (tx, rx) = oneshot::channel(); sender @@ -501,7 +498,6 @@ async fn get_block_number_under_construction( /// - construct a transverse slice along `core_idx` /// - bitwise-or it with the availability slice /// - count the 1 bits, compare to the total length; true on 2/3+ -#[tracing::instrument(level = "trace", fields(subsystem = LOG_TARGET))] fn bitfields_indicate_availability( core_idx: usize, bitfields: &[SignedAvailabilityBitfield], @@ -606,6 +602,3 @@ impl metrics::Metrics for Metrics { /// The provisioning subsystem. pub type ProvisioningSubsystem = JobSubsystem; - -#[cfg(test)] -mod tests; diff --git a/node/core/pvf/Cargo.toml b/node/core/pvf/Cargo.toml index e65aaca7cd57..13d0ce7e1dbb 100644 --- a/node/core/pvf/Cargo.toml +++ b/node/core/pvf/Cargo.toml @@ -13,12 +13,12 @@ always-assert = "0.1" async-std = { version = "1.8.0", features = ["attributes"] } async-process = "1.0.1" assert_matches = "1.4.0" -futures = "0.3.12" +futures = "0.3.15" futures-timer = "3.0.2" libc = "0.2.81" slotmap = "1.0" tracing = "0.1.26" -pin-project = "1.0.4" +pin-project = "1.0.7" rand = "0.8.3" parity-scale-codec = { version = "2.0.0", default-features = false, features = ["derive"] } polkadot-parachain = { path = "../../../parachain" } diff --git a/node/core/pvf/src/artifacts.rs b/node/core/pvf/src/artifacts.rs index 5a1df94f288d..2b739fe1230e 100644 --- a/node/core/pvf/src/artifacts.rs +++ b/node/core/pvf/src/artifacts.rs @@ -18,7 +18,7 @@ use always_assert::always; use async_std::{ path::{Path, PathBuf}, }; -use polkadot_core_primitives::Hash; +use polkadot_parachain::primitives::ValidationCodeHash; use std::{ collections::HashMap, time::{Duration, SystemTime}, @@ -56,14 +56,14 @@ impl Artifact { /// multiple engine implementations the artifact ID should include the engine type as well. #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct ArtifactId { - code_hash: Hash, + code_hash: ValidationCodeHash, } impl ArtifactId { const PREFIX: &'static str = "wasmtime_"; /// Creates a new artifact ID with the given hash. - pub fn new(code_hash: Hash) -> Self { + pub fn new(code_hash: ValidationCodeHash) -> Self { Self { code_hash } } @@ -71,9 +71,10 @@ impl ArtifactId { #[cfg(test)] pub fn from_file_name(file_name: &str) -> Option { use std::str::FromStr as _; + use polkadot_core_primitives::Hash; let file_name = file_name.strip_prefix(Self::PREFIX)?; - let code_hash = Hash::from_str(file_name).ok()?; + let code_hash = Hash::from_str(file_name).ok()?.into(); Some(Self { code_hash }) } @@ -212,7 +213,7 @@ mod tests { #[test] fn path() { let path = Path::new("/test"); - let hash = H256::from_str("1234567890123456789012345678901234567890123456789012345678901234").unwrap(); + let hash = H256::from_str("1234567890123456789012345678901234567890123456789012345678901234").unwrap().into(); assert_eq!( ArtifactId::new(hash).path(path).to_str(), diff --git a/node/core/pvf/src/executor_intf.rs b/node/core/pvf/src/executor_intf.rs index 55ea41b7696b..81fc4fb86494 100644 --- a/node/core/pvf/src/executor_intf.rs +++ b/node/core/pvf/src/executor_intf.rs @@ -29,7 +29,7 @@ use sp_wasm_interface::HostFunctions as _; const CONFIG: Config = Config { // TODO: Make sure we don't use more than 1GB: https://github.com/paritytech/polkadot/issues/699 - heap_pages: 1024, + heap_pages: 2048, allow_missing_func_imports: true, cache_path: None, semantics: Semantics { @@ -118,11 +118,11 @@ impl sp_externalities::Externalities for ValidationExternalities { panic!("kill_child_storage: unsupported feature for parachain validation") } - fn clear_prefix(&mut self, _: &[u8]) { + fn clear_prefix(&mut self, _: &[u8], _: Option) -> (bool, u32) { panic!("clear_prefix: unsupported feature for parachain validation") } - fn clear_child_prefix(&mut self, _: &ChildInfo, _: &[u8]) { + fn clear_child_prefix(&mut self, _: &ChildInfo, _: &[u8], _: Option) -> (bool, u32) { panic!("clear_child_prefix: unsupported feature for parachain validation") } diff --git a/node/core/pvf/src/pvf.rs b/node/core/pvf/src/pvf.rs index 7c0a70ac90bc..00c0777a5489 100644 --- a/node/core/pvf/src/pvf.rs +++ b/node/core/pvf/src/pvf.rs @@ -15,7 +15,7 @@ // along with Polkadot. If not, see . use crate::artifacts::ArtifactId; -use polkadot_core_primitives::Hash; +use polkadot_parachain::primitives::ValidationCodeHash; use sp_core::blake2_256; use std::{fmt, sync::Arc}; @@ -25,7 +25,7 @@ use std::{fmt, sync::Arc}; #[derive(Clone)] pub struct Pvf { pub(crate) code: Arc>, - pub(crate) code_hash: Hash, + pub(crate) code_hash: ValidationCodeHash, } impl fmt::Debug for Pvf { diff --git a/node/core/runtime-api/Cargo.toml b/node/core/runtime-api/Cargo.toml index 4b59d12364cc..27d2c5a20a3b 100644 --- a/node/core/runtime-api/Cargo.toml +++ b/node/core/runtime-api/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Parity Technologies "] edition = "2018" [dependencies] -futures = "0.3.12" +futures = "0.3.15" tracing = "0.1.26" memory-lru = "0.1.0" parity-util-mem = { version = "0.9.0", default-features = false } @@ -17,11 +17,10 @@ sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = polkadot-primitives = { path = "../../../primitives" } polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } -polkadot-overseer = { path = "../../overseer" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } [dev-dependencies] sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } -futures = { version = "0.3.12", features = ["thread-pool"] } +futures = { version = "0.3.15", features = ["thread-pool"] } polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } polkadot-node-primitives = { path = "../../primitives" } diff --git a/node/core/runtime-api/src/cache.rs b/node/core/runtime-api/src/cache.rs index 1bc6f682ff67..97d392863dd9 100644 --- a/node/core/runtime-api/src/cache.rs +++ b/node/core/runtime-api/src/cache.rs @@ -14,20 +14,18 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use polkadot_primitives::v1::{ - BlockNumber, CandidateCommitments, CommittedCandidateReceipt, CandidateEvent, - CoreState, GroupRotationInfo, InboundDownwardMessage, InboundHrmpMessage, Hash, - PersistedValidationData, Id as ParaId, OccupiedCoreAssumption, - SessionIndex, SessionInfo, ValidationCode, ValidatorId, ValidatorIndex, - AuthorityDiscoveryId, -}; -use sp_consensus_babe::Epoch; -use parity_util_mem::{MallocSizeOf, MallocSizeOfExt}; - +use std::collections::btree_map::BTreeMap; use memory_lru::{MemoryLruCache, ResidentSize}; +use parity_util_mem::{MallocSizeOf, MallocSizeOfExt}; +use sp_consensus_babe::Epoch; -use std::collections::btree_map::BTreeMap; +use polkadot_primitives::v1::{ + AuthorityDiscoveryId, BlockNumber, CandidateCommitments, CandidateEvent, + CommittedCandidateReceipt, CoreState, GroupRotationInfo, Hash, Id as ParaId, + InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, PersistedValidationData, + SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, +}; const AUTHORITIES_CACHE_SIZE: usize = 128 * 1024; const VALIDATORS_CACHE_SIZE: usize = 64 * 1024; @@ -37,7 +35,6 @@ const PERSISTED_VALIDATION_DATA_CACHE_SIZE: usize = 64 * 1024; const CHECK_VALIDATION_OUTPUTS_CACHE_SIZE: usize = 64 * 1024; const SESSION_INDEX_FOR_CHILD_CACHE_SIZE: usize = 64 * 1024; const VALIDATION_CODE_CACHE_SIZE: usize = 10 * 1024 * 1024; -const HISTORICAL_VALIDATION_CODE_CACHE_SIZE: usize = 10 * 1024 * 1024; const CANDIDATE_PENDING_AVAILABILITY_CACHE_SIZE: usize = 64 * 1024; const CANDIDATE_EVENTS_CACHE_SIZE: usize = 64 * 1024; const SESSION_INFO_CACHE_SIZE: usize = 64 * 1024; @@ -80,7 +77,7 @@ pub(crate) struct RequestResultCache { check_validation_outputs: MemoryLruCache<(Hash, ParaId, CandidateCommitments), ResidentSizeOf>, session_index_for_child: MemoryLruCache>, validation_code: MemoryLruCache<(Hash, ParaId, OccupiedCoreAssumption), ResidentSizeOf>>, - historical_validation_code: MemoryLruCache<(Hash, ParaId, BlockNumber), ResidentSizeOf>>, + validation_code_by_hash: MemoryLruCache<(Hash, ValidationCodeHash), ResidentSizeOf>>, candidate_pending_availability: MemoryLruCache<(Hash, ParaId), ResidentSizeOf>>, candidate_events: MemoryLruCache>>, session_info: MemoryLruCache<(Hash, SessionIndex), ResidentSizeOf>>, @@ -100,7 +97,7 @@ impl Default for RequestResultCache { check_validation_outputs: MemoryLruCache::new(CHECK_VALIDATION_OUTPUTS_CACHE_SIZE), session_index_for_child: MemoryLruCache::new(SESSION_INDEX_FOR_CHILD_CACHE_SIZE), validation_code: MemoryLruCache::new(VALIDATION_CODE_CACHE_SIZE), - historical_validation_code: MemoryLruCache::new(HISTORICAL_VALIDATION_CODE_CACHE_SIZE), + validation_code_by_hash: MemoryLruCache::new(VALIDATION_CODE_CACHE_SIZE), candidate_pending_availability: MemoryLruCache::new(CANDIDATE_PENDING_AVAILABILITY_CACHE_SIZE), candidate_events: MemoryLruCache::new(CANDIDATE_EVENTS_CACHE_SIZE), session_info: MemoryLruCache::new(SESSION_INFO_CACHE_SIZE), @@ -176,12 +173,12 @@ impl RequestResultCache { self.validation_code.insert(key, ResidentSizeOf(value)); } - pub(crate) fn historical_validation_code(&mut self, key: (Hash, ParaId, BlockNumber)) -> Option<&Option> { - self.historical_validation_code.get(&key).map(|v| &v.0) + pub(crate) fn validation_code_by_hash(&mut self, key: (Hash, ValidationCodeHash)) -> Option<&Option> { + self.validation_code_by_hash.get(&key).map(|v| &v.0) } - pub(crate) fn cache_historical_validation_code(&mut self, key: (Hash, ParaId, BlockNumber), value: Option) { - self.historical_validation_code.insert(key, ResidentSizeOf(value)); + pub(crate) fn cache_validation_code_by_hash(&mut self, key: (Hash, ValidationCodeHash), value: Option) { + self.validation_code_by_hash.insert(key, ResidentSizeOf(value)); } pub(crate) fn candidate_pending_availability(&mut self, key: (Hash, ParaId)) -> Option<&Option> { @@ -242,7 +239,7 @@ pub(crate) enum RequestResult { CheckValidationOutputs(Hash, ParaId, CandidateCommitments, bool), SessionIndexForChild(Hash, SessionIndex), ValidationCode(Hash, ParaId, OccupiedCoreAssumption, Option), - HistoricalValidationCode(Hash, ParaId, BlockNumber, Option), + ValidationCodeByHash(Hash, ValidationCodeHash, Option), CandidatePendingAvailability(Hash, ParaId, Option), CandidateEvents(Hash, Vec), SessionInfo(Hash, SessionIndex, Option), diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index daae21fb73c4..839cbfa3938c 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -22,13 +22,14 @@ #![deny(unused_crate_dependencies)] #![warn(missing_docs)] -use polkadot_overseer::{SubsystemError, SubsystemResult, gen::{ - Subsystem, SpawnedSubsystem, OverseerError, SubsystemContext, - FromOverseer, -}}; -use polkadot_subsystem::{OverseerSignal, errors::RuntimeApiError, messages::{ +use polkadot_subsystem::{ + Subsystem, SpawnedSubsystem, SubsystemResult, SubsystemContext, + FromOverseer, OverseerSignal, + messages::{ RuntimeApiMessage, RuntimeApiRequest as Request, - }}; + }, + errors::RuntimeApiError, +}; use polkadot_node_subsystem_util::metrics::{self, prometheus}; use polkadot_primitives::v1::{Block, BlockId, Hash, ParachainHost}; @@ -43,6 +44,9 @@ use cache::{RequestResult, RequestResultCache}; mod cache; +#[cfg(test)] +mod tests; + const LOG_TARGET: &str = "parachain::runtime-api"; /// The number of maximum runtime api requests can be executed in parallel. Further requests will be buffered. @@ -81,12 +85,12 @@ impl RuntimeApiSubsystem { } } -impl Subsystem for RuntimeApiSubsystem where +impl Subsystem for RuntimeApiSubsystem where Client: ProvideRuntimeApi + Send + 'static + Sync, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Context: SubsystemContext, + Context: SubsystemContext { - fn start(self, ctx: Context) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { SpawnedSubsystem { future: run(ctx, self).boxed(), name: "runtime-api-subsystem", @@ -118,8 +122,8 @@ impl RuntimeApiSubsystem where self.requests_cache.cache_session_index_for_child(relay_parent, session_index), ValidationCode(relay_parent, para_id, assumption, code) => self.requests_cache.cache_validation_code((relay_parent, para_id, assumption), code), - HistoricalValidationCode(relay_parent, para_id, n, code) => - self.requests_cache.cache_historical_validation_code((relay_parent, para_id, n), code), + ValidationCodeByHash(relay_parent, validation_code_hash, code) => + self.requests_cache.cache_validation_code_by_hash((relay_parent, validation_code_hash), code), CandidatePendingAvailability(relay_parent, para_id, candidate) => self.requests_cache.cache_candidate_pending_availability((relay_parent, para_id), candidate), CandidateEvents(relay_parent, events) => @@ -182,9 +186,9 @@ impl RuntimeApiSubsystem where Request::ValidationCode(para, assumption, sender) => query!(validation_code(para, assumption), sender) .map(|sender| Request::ValidationCode(para, assumption, sender)), - Request::HistoricalValidationCode(para, at, sender) => - query!(historical_validation_code(para, at), sender) - .map(|sender| Request::HistoricalValidationCode(para, at, sender)), + Request::ValidationCodeByHash(validation_code_hash, sender) => + query!(validation_code_by_hash(validation_code_hash), sender) + .map(|sender| Request::ValidationCodeByHash(validation_code_hash, sender)), Request::CandidatePendingAvailability(para, sender) => query!(candidate_pending_availability(para), sender) .map(|sender| Request::CandidatePendingAvailability(para, sender)), @@ -261,14 +265,12 @@ impl RuntimeApiSubsystem where } } -#[tracing::instrument(skip(ctx, subsystem), fields(subsystem = LOG_TARGET))] -async fn run( - mut ctx: Context, +async fn run( + mut ctx: impl SubsystemContext, mut subsystem: RuntimeApiSubsystem, ) -> SubsystemResult<()> where Client: ProvideRuntimeApi + Send + Sync + 'static, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Context: SubsystemContext { loop { select! { @@ -287,7 +289,6 @@ async fn run( } } -#[tracing::instrument(level = "trace", skip(client, metrics), fields(subsystem = LOG_TARGET))] fn make_runtime_api_request( client: Arc, metrics: Metrics, @@ -343,8 +344,8 @@ where Request::SessionIndexForChild(sender) => query!(SessionIndexForChild, session_index_for_child(), sender), Request::ValidationCode(para, assumption, sender) => query!(ValidationCode, validation_code(para, assumption), sender), - Request::HistoricalValidationCode(para, at, sender) => - query!(HistoricalValidationCode, historical_validation_code(para, at), sender), + Request::ValidationCodeByHash(validation_code_hash, sender) => + query!(ValidationCodeByHash, validation_code_by_hash(validation_code_hash), sender), Request::CandidatePendingAvailability(para, sender) => query!(CandidatePendingAvailability, candidate_pending_availability(para), sender), Request::CandidateEvents(sender) => query!(CandidateEvents, candidate_events(), sender), @@ -413,820 +414,3 @@ impl metrics::Metrics for Metrics { Ok(Metrics(Some(metrics))) } } - -#[cfg(test)] -mod tests { - use super::*; - - use polkadot_primitives::v1::{ - ValidatorId, ValidatorIndex, GroupRotationInfo, CoreState, PersistedValidationData, - Id as ParaId, OccupiedCoreAssumption, SessionIndex, ValidationCode, - CommittedCandidateReceipt, CandidateEvent, InboundDownwardMessage, - BlockNumber, InboundHrmpMessage, SessionInfo, AuthorityDiscoveryId, - }; - use polkadot_node_subsystem_test_helpers as test_helpers; - use sp_core::testing::TaskExecutor; - use std::{collections::{HashMap, BTreeMap}, sync::{Arc, Mutex}}; - use futures::channel::oneshot; - use polkadot_node_primitives::{ - BabeEpoch, BabeEpochConfiguration, BabeAllowedSlots, - }; - - #[derive(Default, Clone)] - struct MockRuntimeApi { - authorities: Vec, - validators: Vec, - validator_groups: Vec>, - availability_cores: Vec, - availability_cores_wait: Arc>, - validation_data: HashMap, - session_index_for_child: SessionIndex, - session_info: HashMap, - validation_code: HashMap, - historical_validation_code: HashMap>, - validation_outputs_results: HashMap, - candidate_pending_availability: HashMap, - candidate_events: Vec, - dmq_contents: HashMap>, - hrmp_channels: HashMap>>, - babe_epoch: Option, - } - - impl ProvideRuntimeApi for MockRuntimeApi { - type Api = Self; - - fn runtime_api<'a>(&'a self) -> sp_api::ApiRef<'a, Self::Api> { - self.clone().into() - } - } - - sp_api::mock_impl_runtime_apis! { - impl ParachainHost for MockRuntimeApi { - fn validators(&self) -> Vec { - self.validators.clone() - } - - fn validator_groups(&self) -> (Vec>, GroupRotationInfo) { - ( - self.validator_groups.clone(), - GroupRotationInfo { - session_start_block: 1, - group_rotation_frequency: 100, - now: 10, - }, - ) - } - - fn availability_cores(&self) -> Vec { - let _ = self.availability_cores_wait.lock().unwrap(); - self.availability_cores.clone() - } - - fn persisted_validation_data( - &self, - para: ParaId, - _assumption: OccupiedCoreAssumption, - ) -> Option { - self.validation_data.get(¶).cloned() - } - - fn check_validation_outputs( - &self, - para_id: ParaId, - _commitments: polkadot_primitives::v1::CandidateCommitments, - ) -> bool { - self.validation_outputs_results - .get(¶_id) - .cloned() - .expect( - "`check_validation_outputs` called but the expected result hasn't been supplied" - ) - } - - fn session_index_for_child(&self) -> SessionIndex { - self.session_index_for_child.clone() - } - - fn session_info(&self, index: SessionIndex) -> Option { - self.session_info.get(&index).cloned() - } - - fn validation_code( - &self, - para: ParaId, - _assumption: OccupiedCoreAssumption, - ) -> Option { - self.validation_code.get(¶).map(|c| c.clone()) - } - - fn historical_validation_code( - &self, - para: ParaId, - at: BlockNumber, - ) -> Option { - self.historical_validation_code.get(¶).and_then(|h_code| { - h_code.iter() - .take_while(|(changed_at, _)| changed_at <= &at) - .last() - .map(|(_, code)| code.clone()) - }) - } - - fn candidate_pending_availability( - &self, - para: ParaId, - ) -> Option { - self.candidate_pending_availability.get(¶).map(|c| c.clone()) - } - - fn candidate_events(&self) -> Vec { - self.candidate_events.clone() - } - - fn dmq_contents( - &self, - recipient: ParaId, - ) -> Vec { - self.dmq_contents.get(&recipient).map(|q| q.clone()).unwrap_or_default() - } - - fn inbound_hrmp_channels_contents( - &self, - recipient: ParaId - ) -> BTreeMap> { - self.hrmp_channels.get(&recipient).map(|q| q.clone()).unwrap_or_default() - } - - fn validation_code_by_hash( - &self, - _hash: Hash, - ) -> Option { - unreachable!("not used in tests"); - } - } - - impl BabeApi for MockRuntimeApi { - fn configuration(&self) -> sp_consensus_babe::BabeGenesisConfiguration { - unimplemented!() - } - - fn current_epoch_start(&self) -> sp_consensus_babe::Slot { - self.babe_epoch.as_ref().unwrap().start_slot - } - - fn current_epoch(&self) -> BabeEpoch { - self.babe_epoch.as_ref().unwrap().clone() - } - - fn next_epoch(&self) -> BabeEpoch { - unimplemented!() - } - - fn generate_key_ownership_proof( - _slot: sp_consensus_babe::Slot, - _authority_id: sp_consensus_babe::AuthorityId, - ) -> Option { - None - } - - fn submit_report_equivocation_unsigned_extrinsic( - _equivocation_proof: sp_consensus_babe::EquivocationProof, - _key_owner_proof: sp_consensus_babe::OpaqueKeyOwnershipProof, - ) -> Option<()> { - None - } - } - - impl AuthorityDiscoveryApi for MockRuntimeApi { - fn authorities(&self) -> Vec { - self.authorities.clone() - } - } - } - - #[test] - fn requests_authorities() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - let runtime_api = Arc::new(MockRuntimeApi::default()); - let relay_parent = [1; 32].into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request(relay_parent, Request::Authorities(tx)) - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), runtime_api.authorities); - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_validators() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - let runtime_api = Arc::new(MockRuntimeApi::default()); - let relay_parent = [1; 32].into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request(relay_parent, Request::Validators(tx)) - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), runtime_api.validators); - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_validator_groups() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - let runtime_api = Arc::new(MockRuntimeApi::default()); - let relay_parent = [1; 32].into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request(relay_parent, Request::ValidatorGroups(tx)) - }).await; - - assert_eq!(rx.await.unwrap().unwrap().0, runtime_api.validator_groups); - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_availability_cores() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - let runtime_api = Arc::new(MockRuntimeApi::default()); - let relay_parent = [1; 32].into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request(relay_parent, Request::AvailabilityCores(tx)) - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), runtime_api.availability_cores); - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_persisted_validation_data() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - let relay_parent = [1; 32].into(); - let para_a = 5.into(); - let para_b = 6.into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let mut runtime_api = MockRuntimeApi::default(); - runtime_api.validation_data.insert(para_a, Default::default()); - let runtime_api = Arc::new(runtime_api); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::PersistedValidationData(para_a, OccupiedCoreAssumption::Included, tx) - ), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), Some(Default::default())); - - let (tx, rx) = oneshot::channel(); - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::PersistedValidationData(para_b, OccupiedCoreAssumption::Included, tx) - ), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), None); - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_check_validation_outputs() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - let mut runtime_api = MockRuntimeApi::default(); - let relay_parent = [1; 32].into(); - let para_a = 5.into(); - let para_b = 6.into(); - let commitments = polkadot_primitives::v1::CandidateCommitments::default(); - let spawner = sp_core::testing::TaskExecutor::new(); - - runtime_api.validation_outputs_results.insert(para_a, false); - runtime_api.validation_outputs_results.insert(para_b, true); - - let runtime_api = Arc::new(runtime_api); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::CheckValidationOutputs( - para_a, - commitments.clone(), - tx, - ), - ) - }).await; - assert_eq!( - rx.await.unwrap().unwrap(), - runtime_api.validation_outputs_results[¶_a], - ); - - let (tx, rx) = oneshot::channel(); - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::CheckValidationOutputs( - para_b, - commitments, - tx, - ), - ) - }).await; - assert_eq!( - rx.await.unwrap().unwrap(), - runtime_api.validation_outputs_results[¶_b], - ); - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_session_index_for_child() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - let runtime_api = Arc::new(MockRuntimeApi::default()); - let relay_parent = [1; 32].into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request(relay_parent, Request::SessionIndexForChild(tx)) - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), runtime_api.session_index_for_child); - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_session_info() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - let mut runtime_api = MockRuntimeApi::default(); - let session_index = 1; - runtime_api.session_info.insert(session_index, Default::default()); - let runtime_api = Arc::new(runtime_api); - let spawner = sp_core::testing::TaskExecutor::new(); - - let relay_parent = [1; 32].into(); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request(relay_parent, Request::SessionInfo(session_index, tx)) - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), Some(Default::default())); - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_validation_code() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - - let relay_parent = [1; 32].into(); - let para_a = 5.into(); - let para_b = 6.into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let mut runtime_api = MockRuntimeApi::default(); - runtime_api.validation_code.insert(para_a, Default::default()); - let runtime_api = Arc::new(runtime_api); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::ValidationCode(para_a, OccupiedCoreAssumption::Included, tx) - ), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), Some(Default::default())); - - let (tx, rx) = oneshot::channel(); - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::ValidationCode(para_b, OccupiedCoreAssumption::Included, tx) - ), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), None); - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_candidate_pending_availability() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - let relay_parent = [1; 32].into(); - let para_a = 5.into(); - let para_b = 6.into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let mut runtime_api = MockRuntimeApi::default(); - runtime_api.candidate_pending_availability.insert(para_a, Default::default()); - let runtime_api = Arc::new(runtime_api); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::CandidatePendingAvailability(para_a, tx), - ) - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), Some(Default::default())); - - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::CandidatePendingAvailability(para_b, tx), - ) - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), None); - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_candidate_events() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - let runtime_api = Arc::new(MockRuntimeApi::default()); - let relay_parent = [1; 32].into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request(relay_parent, Request::CandidateEvents(tx)) - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), runtime_api.candidate_events); - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_dmq_contents() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - - let relay_parent = [1; 32].into(); - let para_a = 5.into(); - let para_b = 6.into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let runtime_api = Arc::new({ - let mut runtime_api = MockRuntimeApi::default(); - - runtime_api.dmq_contents.insert(para_a, vec![]); - runtime_api.dmq_contents.insert( - para_b, - vec![InboundDownwardMessage { - sent_at: 228, - msg: b"Novus Ordo Seclorum".to_vec(), - }], - ); - - runtime_api - }); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - ctx_handle - .send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request(relay_parent, Request::DmqContents(para_a, tx)), - }) - .await; - assert_eq!(rx.await.unwrap().unwrap(), vec![]); - - let (tx, rx) = oneshot::channel(); - ctx_handle - .send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request(relay_parent, Request::DmqContents(para_b, tx)), - }) - .await; - assert_eq!( - rx.await.unwrap().unwrap(), - vec![InboundDownwardMessage { - sent_at: 228, - msg: b"Novus Ordo Seclorum".to_vec(), - }] - ); - - ctx_handle - .send(FromOverseer::Signal(OverseerSignal::Conclude)) - .await; - }; - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_inbound_hrmp_channels_contents() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - - let relay_parent = [1; 32].into(); - let para_a = 99.into(); - let para_b = 66.into(); - let para_c = 33.into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let para_b_inbound_channels = [ - (para_a, vec![]), - ( - para_c, - vec![InboundHrmpMessage { - sent_at: 1, - data: "𝙀=𝙈𝘾²".as_bytes().to_owned(), - }], - ), - ] - .iter() - .cloned() - .collect::>(); - - let runtime_api = Arc::new({ - let mut runtime_api = MockRuntimeApi::default(); - - runtime_api.hrmp_channels.insert(para_a, BTreeMap::new()); - runtime_api - .hrmp_channels - .insert(para_b, para_b_inbound_channels.clone()); - - runtime_api - }); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - ctx_handle - .send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::InboundHrmpChannelsContents(para_a, tx), - ), - }) - .await; - assert_eq!(rx.await.unwrap().unwrap(), BTreeMap::new()); - - let (tx, rx) = oneshot::channel(); - ctx_handle - .send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::InboundHrmpChannelsContents(para_b, tx), - ), - }) - .await; - assert_eq!(rx.await.unwrap().unwrap(), para_b_inbound_channels,); - - ctx_handle - .send(FromOverseer::Signal(OverseerSignal::Conclude)) - .await; - }; - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn requests_historical_code() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - - let para_a = 5.into(); - let para_b = 6.into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let runtime_api = Arc::new({ - let mut runtime_api = MockRuntimeApi::default(); - - runtime_api.historical_validation_code.insert( - para_a, - vec![(1, vec![1, 2, 3].into()), (10, vec![4, 5, 6].into())], - ); - - runtime_api.historical_validation_code.insert( - para_b, - vec![(5, vec![7, 8, 9].into())], - ); - - runtime_api - }); - let relay_parent = [1; 32].into(); - - let subsystem = RuntimeApiSubsystem::new(runtime_api, Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - { - let (tx, rx) = oneshot::channel(); - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::HistoricalValidationCode(para_a, 5, tx), - ) - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), Some(ValidationCode::from(vec![1, 2, 3]))); - } - - { - let (tx, rx) = oneshot::channel(); - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::HistoricalValidationCode(para_a, 10, tx), - ) - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), Some(ValidationCode::from(vec![4, 5, 6]))); - } - - { - let (tx, rx) = oneshot::channel(); - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request( - relay_parent, - Request::HistoricalValidationCode(para_b, 1, tx), - ) - }).await; - - assert!(rx.await.unwrap().unwrap().is_none()); - } - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn multiple_requests_in_parallel_are_working() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - let runtime_api = Arc::new(MockRuntimeApi::default()); - let relay_parent = [1; 32].into(); - let spawner = sp_core::testing::TaskExecutor::new(); - let mutex = runtime_api.availability_cores_wait.clone(); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - // Make all requests block until we release this mutex. - let lock = mutex.lock().unwrap(); - - let mut receivers = Vec::new(); - - for _ in 0..MAX_PARALLEL_REQUESTS * 10 { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request(relay_parent, Request::AvailabilityCores(tx)) - }).await; - - receivers.push(rx); - } - - let join = future::join_all(receivers); - - drop(lock); - - join.await - .into_iter() - .for_each(|r| assert_eq!(r.unwrap().unwrap(), runtime_api.availability_cores)); - - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } - - #[test] - fn request_babe_epoch() { - let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); - let mut runtime_api = MockRuntimeApi::default(); - let epoch = BabeEpoch { - epoch_index: 100, - start_slot: sp_consensus_babe::Slot::from(1000), - duration: 10, - authorities: Vec::new(), - randomness: [1u8; 32], - config: BabeEpochConfiguration { - c: (1, 4), - allowed_slots: BabeAllowedSlots::PrimarySlots, - }, - }; - runtime_api.babe_epoch = Some(epoch.clone()); - let runtime_api = Arc::new(runtime_api); - let relay_parent = [1; 32].into(); - let spawner = sp_core::testing::TaskExecutor::new(); - - let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); - let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = async move { - let (tx, rx) = oneshot::channel(); - - ctx_handle.send(FromOverseer::Communication { - msg: RuntimeApiMessage::Request(relay_parent, Request::CurrentBabeEpoch(tx)) - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), epoch); - ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }; - - futures::executor::block_on(future::join(subsystem_task, test_task)); - } -} diff --git a/node/core/runtime-api/src/tests.rs b/node/core/runtime-api/src/tests.rs new file mode 100644 index 000000000000..787dc8884b0b --- /dev/null +++ b/node/core/runtime-api/src/tests.rs @@ -0,0 +1,790 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; + +use polkadot_primitives::v1::{ + ValidatorId, ValidatorIndex, GroupRotationInfo, CoreState, PersistedValidationData, + Id as ParaId, OccupiedCoreAssumption, SessionIndex, ValidationCode, + CommittedCandidateReceipt, CandidateEvent, InboundDownwardMessage, + InboundHrmpMessage, SessionInfo, AuthorityDiscoveryId, ValidationCodeHash, +}; +use polkadot_node_subsystem_test_helpers as test_helpers; +use sp_core::testing::TaskExecutor; +use std::{collections::{HashMap, BTreeMap}, sync::{Arc, Mutex}}; +use futures::channel::oneshot; +use polkadot_node_primitives::{ + BabeEpoch, BabeEpochConfiguration, BabeAllowedSlots, +}; + +#[derive(Default, Clone)] +struct MockRuntimeApi { + authorities: Vec, + validators: Vec, + validator_groups: Vec>, + availability_cores: Vec, + availability_cores_wait: Arc>, + validation_data: HashMap, + session_index_for_child: SessionIndex, + session_info: HashMap, + validation_code: HashMap, + validation_code_by_hash: HashMap, + validation_outputs_results: HashMap, + candidate_pending_availability: HashMap, + candidate_events: Vec, + dmq_contents: HashMap>, + hrmp_channels: HashMap>>, + babe_epoch: Option, +} + +impl ProvideRuntimeApi for MockRuntimeApi { + type Api = Self; + + fn runtime_api<'a>(&'a self) -> sp_api::ApiRef<'a, Self::Api> { + self.clone().into() + } +} + +sp_api::mock_impl_runtime_apis! { + impl ParachainHost for MockRuntimeApi { + fn validators(&self) -> Vec { + self.validators.clone() + } + + fn validator_groups(&self) -> (Vec>, GroupRotationInfo) { + ( + self.validator_groups.clone(), + GroupRotationInfo { + session_start_block: 1, + group_rotation_frequency: 100, + now: 10, + }, + ) + } + + fn availability_cores(&self) -> Vec { + let _ = self.availability_cores_wait.lock().unwrap(); + self.availability_cores.clone() + } + + fn persisted_validation_data( + &self, + para: ParaId, + _assumption: OccupiedCoreAssumption, + ) -> Option { + self.validation_data.get(¶).cloned() + } + + fn check_validation_outputs( + &self, + para_id: ParaId, + _commitments: polkadot_primitives::v1::CandidateCommitments, + ) -> bool { + self.validation_outputs_results + .get(¶_id) + .cloned() + .expect( + "`check_validation_outputs` called but the expected result hasn't been supplied" + ) + } + + fn session_index_for_child(&self) -> SessionIndex { + self.session_index_for_child.clone() + } + + fn session_info(&self, index: SessionIndex) -> Option { + self.session_info.get(&index).cloned() + } + + fn validation_code( + &self, + para: ParaId, + _assumption: OccupiedCoreAssumption, + ) -> Option { + self.validation_code.get(¶).map(|c| c.clone()) + } + + fn candidate_pending_availability( + &self, + para: ParaId, + ) -> Option { + self.candidate_pending_availability.get(¶).map(|c| c.clone()) + } + + fn candidate_events(&self) -> Vec { + self.candidate_events.clone() + } + + fn dmq_contents( + &self, + recipient: ParaId, + ) -> Vec { + self.dmq_contents.get(&recipient).map(|q| q.clone()).unwrap_or_default() + } + + fn inbound_hrmp_channels_contents( + &self, + recipient: ParaId + ) -> BTreeMap> { + self.hrmp_channels.get(&recipient).map(|q| q.clone()).unwrap_or_default() + } + + fn validation_code_by_hash( + &self, + hash: ValidationCodeHash, + ) -> Option { + self.validation_code_by_hash.get(&hash).map(|c| c.clone()) + } + } + + impl BabeApi for MockRuntimeApi { + fn configuration(&self) -> sp_consensus_babe::BabeGenesisConfiguration { + unimplemented!() + } + + fn current_epoch_start(&self) -> sp_consensus_babe::Slot { + self.babe_epoch.as_ref().unwrap().start_slot + } + + fn current_epoch(&self) -> BabeEpoch { + self.babe_epoch.as_ref().unwrap().clone() + } + + fn next_epoch(&self) -> BabeEpoch { + unimplemented!() + } + + fn generate_key_ownership_proof( + _slot: sp_consensus_babe::Slot, + _authority_id: sp_consensus_babe::AuthorityId, + ) -> Option { + None + } + + fn submit_report_equivocation_unsigned_extrinsic( + _equivocation_proof: sp_consensus_babe::EquivocationProof, + _key_owner_proof: sp_consensus_babe::OpaqueKeyOwnershipProof, + ) -> Option<()> { + None + } + } + + impl AuthorityDiscoveryApi for MockRuntimeApi { + fn authorities(&self) -> Vec { + self.authorities.clone() + } + } +} + +#[test] +fn requests_authorities() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let runtime_api = Arc::new(MockRuntimeApi::default()); + let relay_parent = [1; 32].into(); + let spawner = sp_core::testing::TaskExecutor::new(); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request(relay_parent, Request::Authorities(tx)) + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), runtime_api.authorities); + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_validators() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let runtime_api = Arc::new(MockRuntimeApi::default()); + let relay_parent = [1; 32].into(); + let spawner = sp_core::testing::TaskExecutor::new(); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request(relay_parent, Request::Validators(tx)) + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), runtime_api.validators); + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_validator_groups() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let runtime_api = Arc::new(MockRuntimeApi::default()); + let relay_parent = [1; 32].into(); + let spawner = sp_core::testing::TaskExecutor::new(); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request(relay_parent, Request::ValidatorGroups(tx)) + }).await; + + assert_eq!(rx.await.unwrap().unwrap().0, runtime_api.validator_groups); + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_availability_cores() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let runtime_api = Arc::new(MockRuntimeApi::default()); + let relay_parent = [1; 32].into(); + let spawner = sp_core::testing::TaskExecutor::new(); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request(relay_parent, Request::AvailabilityCores(tx)) + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), runtime_api.availability_cores); + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_persisted_validation_data() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let relay_parent = [1; 32].into(); + let para_a = 5.into(); + let para_b = 6.into(); + let spawner = sp_core::testing::TaskExecutor::new(); + + let mut runtime_api = MockRuntimeApi::default(); + runtime_api.validation_data.insert(para_a, Default::default()); + let runtime_api = Arc::new(runtime_api); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request( + relay_parent, + Request::PersistedValidationData(para_a, OccupiedCoreAssumption::Included, tx) + ), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), Some(Default::default())); + + let (tx, rx) = oneshot::channel(); + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request( + relay_parent, + Request::PersistedValidationData(para_b, OccupiedCoreAssumption::Included, tx) + ), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), None); + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_check_validation_outputs() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let mut runtime_api = MockRuntimeApi::default(); + let relay_parent = [1; 32].into(); + let para_a = 5.into(); + let para_b = 6.into(); + let commitments = polkadot_primitives::v1::CandidateCommitments::default(); + let spawner = sp_core::testing::TaskExecutor::new(); + + runtime_api.validation_outputs_results.insert(para_a, false); + runtime_api.validation_outputs_results.insert(para_b, true); + + let runtime_api = Arc::new(runtime_api); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request( + relay_parent, + Request::CheckValidationOutputs( + para_a, + commitments.clone(), + tx, + ), + ) + }).await; + assert_eq!( + rx.await.unwrap().unwrap(), + runtime_api.validation_outputs_results[¶_a], + ); + + let (tx, rx) = oneshot::channel(); + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request( + relay_parent, + Request::CheckValidationOutputs( + para_b, + commitments, + tx, + ), + ) + }).await; + assert_eq!( + rx.await.unwrap().unwrap(), + runtime_api.validation_outputs_results[¶_b], + ); + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_session_index_for_child() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let runtime_api = Arc::new(MockRuntimeApi::default()); + let relay_parent = [1; 32].into(); + let spawner = sp_core::testing::TaskExecutor::new(); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request(relay_parent, Request::SessionIndexForChild(tx)) + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), runtime_api.session_index_for_child); + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_session_info() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let mut runtime_api = MockRuntimeApi::default(); + let session_index = 1; + runtime_api.session_info.insert(session_index, Default::default()); + let runtime_api = Arc::new(runtime_api); + let spawner = sp_core::testing::TaskExecutor::new(); + + let relay_parent = [1; 32].into(); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request(relay_parent, Request::SessionInfo(session_index, tx)) + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), Some(Default::default())); + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_validation_code() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + + let relay_parent = [1; 32].into(); + let para_a = 5.into(); + let para_b = 6.into(); + let spawner = sp_core::testing::TaskExecutor::new(); + + let mut runtime_api = MockRuntimeApi::default(); + runtime_api.validation_code.insert(para_a, Default::default()); + let runtime_api = Arc::new(runtime_api); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request( + relay_parent, + Request::ValidationCode(para_a, OccupiedCoreAssumption::Included, tx) + ), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), Some(Default::default())); + + let (tx, rx) = oneshot::channel(); + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request( + relay_parent, + Request::ValidationCode(para_b, OccupiedCoreAssumption::Included, tx) + ), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), None); + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_candidate_pending_availability() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let relay_parent = [1; 32].into(); + let para_a = 5.into(); + let para_b = 6.into(); + let spawner = sp_core::testing::TaskExecutor::new(); + + let mut runtime_api = MockRuntimeApi::default(); + runtime_api.candidate_pending_availability.insert(para_a, Default::default()); + let runtime_api = Arc::new(runtime_api); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request( + relay_parent, + Request::CandidatePendingAvailability(para_a, tx), + ) + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), Some(Default::default())); + + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request( + relay_parent, + Request::CandidatePendingAvailability(para_b, tx), + ) + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), None); + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_candidate_events() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let runtime_api = Arc::new(MockRuntimeApi::default()); + let relay_parent = [1; 32].into(); + let spawner = sp_core::testing::TaskExecutor::new(); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request(relay_parent, Request::CandidateEvents(tx)) + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), runtime_api.candidate_events); + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_dmq_contents() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + + let relay_parent = [1; 32].into(); + let para_a = 5.into(); + let para_b = 6.into(); + let spawner = sp_core::testing::TaskExecutor::new(); + + let runtime_api = Arc::new({ + let mut runtime_api = MockRuntimeApi::default(); + + runtime_api.dmq_contents.insert(para_a, vec![]); + runtime_api.dmq_contents.insert( + para_b, + vec![InboundDownwardMessage { + sent_at: 228, + msg: b"Novus Ordo Seclorum".to_vec(), + }], + ); + + runtime_api + }); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + ctx_handle + .send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request(relay_parent, Request::DmqContents(para_a, tx)), + }) + .await; + assert_eq!(rx.await.unwrap().unwrap(), vec![]); + + let (tx, rx) = oneshot::channel(); + ctx_handle + .send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request(relay_parent, Request::DmqContents(para_b, tx)), + }) + .await; + assert_eq!( + rx.await.unwrap().unwrap(), + vec![InboundDownwardMessage { + sent_at: 228, + msg: b"Novus Ordo Seclorum".to_vec(), + }] + ); + + ctx_handle + .send(FromOverseer::Signal(OverseerSignal::Conclude)) + .await; + }; + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_inbound_hrmp_channels_contents() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + + let relay_parent = [1; 32].into(); + let para_a = 99.into(); + let para_b = 66.into(); + let para_c = 33.into(); + let spawner = sp_core::testing::TaskExecutor::new(); + + let para_b_inbound_channels = [ + (para_a, vec![]), + ( + para_c, + vec![InboundHrmpMessage { + sent_at: 1, + data: "𝙀=𝙈𝘾²".as_bytes().to_owned(), + }], + ), + ] + .iter() + .cloned() + .collect::>(); + + let runtime_api = Arc::new({ + let mut runtime_api = MockRuntimeApi::default(); + + runtime_api.hrmp_channels.insert(para_a, BTreeMap::new()); + runtime_api + .hrmp_channels + .insert(para_b, para_b_inbound_channels.clone()); + + runtime_api + }); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + ctx_handle + .send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request( + relay_parent, + Request::InboundHrmpChannelsContents(para_a, tx), + ), + }) + .await; + assert_eq!(rx.await.unwrap().unwrap(), BTreeMap::new()); + + let (tx, rx) = oneshot::channel(); + ctx_handle + .send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request( + relay_parent, + Request::InboundHrmpChannelsContents(para_b, tx), + ), + }) + .await; + assert_eq!(rx.await.unwrap().unwrap(), para_b_inbound_channels,); + + ctx_handle + .send(FromOverseer::Signal(OverseerSignal::Conclude)) + .await; + }; + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn requests_validation_code_by_hash() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let spawner = sp_core::testing::TaskExecutor::new(); + + let (runtime_api, validation_code) = { + let mut runtime_api = MockRuntimeApi::default(); + let mut validation_code = Vec::new(); + + for n in 0..5 { + let code = ValidationCode::from(vec![n; 32]); + runtime_api.validation_code_by_hash.insert( + code.hash(), + code.clone(), + ); + validation_code.push(code); + } + + (runtime_api, validation_code) + }; + + let subsystem = RuntimeApiSubsystem::new(Arc::new(runtime_api), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + + let relay_parent = [1; 32].into(); + let test_task = async move { + for code in validation_code { + let (tx, rx) = oneshot::channel(); + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request( + relay_parent, + Request::ValidationCodeByHash(code.hash(), tx), + ) + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), Some(code)); + } + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn multiple_requests_in_parallel_are_working() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let runtime_api = Arc::new(MockRuntimeApi::default()); + let relay_parent = [1; 32].into(); + let spawner = sp_core::testing::TaskExecutor::new(); + let mutex = runtime_api.availability_cores_wait.clone(); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + // Make all requests block until we release this mutex. + let lock = mutex.lock().unwrap(); + + let mut receivers = Vec::new(); + + for _ in 0..MAX_PARALLEL_REQUESTS * 10 { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request(relay_parent, Request::AvailabilityCores(tx)) + }).await; + + receivers.push(rx); + } + + let join = future::join_all(receivers); + + drop(lock); + + join.await + .into_iter() + .for_each(|r| assert_eq!(r.unwrap().unwrap(), runtime_api.availability_cores)); + + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} + +#[test] +fn request_babe_epoch() { + let (ctx, mut ctx_handle) = test_helpers::make_subsystem_context(TaskExecutor::new()); + let mut runtime_api = MockRuntimeApi::default(); + let epoch = BabeEpoch { + epoch_index: 100, + start_slot: sp_consensus_babe::Slot::from(1000), + duration: 10, + authorities: Vec::new(), + randomness: [1u8; 32], + config: BabeEpochConfiguration { + c: (1, 4), + allowed_slots: BabeAllowedSlots::PrimarySlots, + }, + }; + runtime_api.babe_epoch = Some(epoch.clone()); + let runtime_api = Arc::new(runtime_api); + let relay_parent = [1; 32].into(); + let spawner = sp_core::testing::TaskExecutor::new(); + + let subsystem = RuntimeApiSubsystem::new(runtime_api.clone(), Metrics(None), spawner); + let subsystem_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = async move { + let (tx, rx) = oneshot::channel(); + + ctx_handle.send(FromOverseer::Communication { + msg: RuntimeApiMessage::Request(relay_parent, Request::CurrentBabeEpoch(tx)) + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), epoch); + ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }; + + futures::executor::block_on(future::join(subsystem_task, test_task)); +} diff --git a/node/primitives/Cargo.toml b/node/primitives/Cargo.toml index e8b01dea00f9..7cefeab9ba03 100644 --- a/node/primitives/Cargo.toml +++ b/node/primitives/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" description = "Primitives types for the Node-side" [dependencies] -futures = "0.3.12" +futures = "0.3.15" polkadot-primitives = { path = "../../primitives" } polkadot-statement-table = { path = "../../statement-table" } parity-scale-codec = { version = "2.0.0", default-features = false, features = ["derive"] } @@ -15,6 +15,7 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-consensus-vrf = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-maybe-compressed-blob = { git = "https://github.com/paritytech/substrate", branch = "master" } polkadot-parachain = { path = "../../parachain", default-features = false } schnorrkel = "0.9.1" diff --git a/node/primitives/src/approval.rs b/node/primitives/src/approval.rs index 8303478aa53c..743c37f32759 100644 --- a/node/primitives/src/approval.rs +++ b/node/primitives/src/approval.rs @@ -98,10 +98,6 @@ pub struct IndirectAssignmentCert { pub cert: AssignmentCert, } -/// A vote of approval on a candidate. -#[derive(Debug, Clone, Encode, Decode)] -pub struct ApprovalVote(pub CandidateHash); - /// A signed approval vote which references the candidate indirectly via the block. /// /// In practice, we have a look-up from block hash and candidate index to candidate hash, diff --git a/node/primitives/src/lib.rs b/node/primitives/src/lib.rs index 7646ca1ff204..490367db4cca 100644 --- a/node/primitives/src/lib.rs +++ b/node/primitives/src/lib.rs @@ -22,30 +22,40 @@ #![deny(missing_docs)] +use std::convert::TryInto; use std::pin::Pin; use serde::{Serialize, Deserialize}; use futures::Future; use parity_scale_codec::{Decode, Encode}; +use sp_keystore::{CryptoStore, SyncCryptoStorePtr, Error as KeystoreError}; +use sp_application_crypto::AppKey; pub use sp_core::traits::SpawnNamed; pub use sp_consensus_babe::{ Epoch as BabeEpoch, BabeEpochConfiguration, AllowedSlots as BabeAllowedSlots, }; -use polkadot_primitives::v1::{BlakeTwo256, CandidateCommitments, CandidateHash, CollatorPair, CommittedCandidateReceipt, CompactStatement, EncodeAs, Hash, HashT, HeadData, Id as ParaId, OutboundHrmpMessage, PersistedValidationData, Signed, UncheckedSigned, UpwardMessage, ValidationCode, ValidatorIndex}; +use polkadot_primitives::v1::{ + BlakeTwo256, CandidateCommitments, CandidateHash, CollatorPair, CommittedCandidateReceipt, + CompactStatement, EncodeAs, Hash, HashT, HeadData, Id as ParaId, OutboundHrmpMessage, + PersistedValidationData, Signed, UncheckedSigned, UpwardMessage, ValidationCode, + ValidatorIndex, ValidatorSignature, ValidDisputeStatementKind, InvalidDisputeStatementKind, + CandidateReceipt, ValidatorId, SessionIndex, DisputeStatement, MAX_CODE_SIZE, MAX_POV_SIZE, +}; + pub use polkadot_parachain::primitives::BlockData; pub mod approval; /// The bomb limit for decompressing code blobs. -pub const VALIDATION_CODE_BOMB_LIMIT: usize = 16 * 1024 * 1024; - -/// Maximum PoV size we support right now. -pub const MAX_POV_SIZE: u32 = 20 * 1024 * 1024; +pub const VALIDATION_CODE_BOMB_LIMIT: usize = (MAX_CODE_SIZE * 4u32) as usize; /// The bomb limit for decompressing PoV blobs. -pub const POV_BOMB_LIMIT: usize = MAX_POV_SIZE as usize; +pub const POV_BOMB_LIMIT: usize = (MAX_POV_SIZE * 4u32) as usize; + +/// The cumulative weight of a block in a fork-choice rule. +pub type BlockWeight = u32; /// A statement, where the candidate receipt is included in the `Seconded` variant. /// @@ -273,3 +283,125 @@ pub fn maybe_compress_pov(pov: PoV) -> PoV { let pov = PoV { block_data: BlockData(raw) }; pov } + +/// Tracked votes on candidates, for the purposes of dispute resolution. +#[derive(Debug, Clone)] +pub struct CandidateVotes { + /// The receipt of the candidate itself. + pub candidate_receipt: CandidateReceipt, + /// Votes of validity, sorted by validator index. + pub valid: Vec<(ValidDisputeStatementKind, ValidatorIndex, ValidatorSignature)>, + /// Votes of invalidity, sorted by validator index. + pub invalid: Vec<(InvalidDisputeStatementKind, ValidatorIndex, ValidatorSignature)>, +} + +impl CandidateVotes { + /// Get the set of all validators who have votes in the set, ascending. + pub fn voted_indices(&self) -> Vec { + let mut v: Vec<_> = self.valid.iter().map(|x| x.1).chain( + self.invalid.iter().map(|x| x.1) + ).collect(); + + v.sort(); + v.dedup(); + + v + } +} + + +/// A checked dispute statement from an associated validator. +#[derive(Debug, Clone)] +pub struct SignedDisputeStatement { + dispute_statement: DisputeStatement, + candidate_hash: CandidateHash, + validator_public: ValidatorId, + validator_signature: ValidatorSignature, + session_index: SessionIndex, +} + +impl SignedDisputeStatement { + /// Create a new `SignedDisputeStatement`, which is only possible by checking the signature. + pub fn new_checked( + dispute_statement: DisputeStatement, + candidate_hash: CandidateHash, + session_index: SessionIndex, + validator_public: ValidatorId, + validator_signature: ValidatorSignature, + ) -> Result { + dispute_statement.check_signature( + &validator_public, + candidate_hash, + session_index, + &validator_signature, + ).map(|_| SignedDisputeStatement { + dispute_statement, + candidate_hash, + validator_public, + validator_signature, + session_index, + }) + } + + /// Sign this statement with the given keystore and key. Pass `valid = true` to + /// indicate validity of the candidate, and `valid = false` to indicate invalidity. + pub async fn sign_explicit( + keystore: &SyncCryptoStorePtr, + valid: bool, + candidate_hash: CandidateHash, + session_index: SessionIndex, + validator_public: ValidatorId, + ) -> Result, KeystoreError> { + let dispute_statement = if valid { + DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) + } else { + DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) + }; + + let data = dispute_statement.payload_data(candidate_hash, session_index); + let signature = CryptoStore::sign_with( + &**keystore, + ValidatorId::ID, + &validator_public.clone().into(), + &data, + ).await?; + + let signature = match signature { + Some(sig) => sig.try_into().map_err(|_| KeystoreError::KeyNotSupported(ValidatorId::ID))?, + None => return Ok(None), + }; + + Ok(Some(Self { + dispute_statement, + candidate_hash, + validator_public, + validator_signature: signature, + session_index, + })) + } + + /// Access the underlying dispute statement + pub fn statement(&self) -> &DisputeStatement { + &self.dispute_statement + } + + /// Access the underlying candidate hash. + pub fn candidate_hash(&self) -> &CandidateHash { + &self.candidate_hash + } + + /// Access the underlying validator public key. + pub fn validator_public(&self) -> &ValidatorId { + &self.validator_public + } + + /// Access the underlying validator signature. + pub fn validator_signature(&self) -> &ValidatorSignature { + &self.validator_signature + } + + /// Access the underlying session index. + pub fn session_index(&self) -> SessionIndex { + self.session_index + } +} diff --git a/primitives/Cargo.toml b/primitives/Cargo.toml index e26e638fa9f6..677c8b31649c 100644 --- a/primitives/Cargo.toml +++ b/primitives/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "polkadot-primitives" -version = "0.9.4" +version = "0.9.7" authors = ["Parity Technologies "] edition = "2018" diff --git a/primitives/src/v0.rs b/primitives/src/v0.rs index ab7c3fe18b8d..917bda2aa72a 100644 --- a/primitives/src/v0.rs +++ b/primitives/src/v0.rs @@ -672,6 +672,14 @@ pub enum CompactStatement { Valid(CandidateHash), } +impl CompactStatement { + /// Yields the payload used for validator signatures on this kind + /// of statement. + pub fn signing_payload(&self, context: &SigningContext) -> Vec { + (self, context).encode() + } +} + // Inner helper for codec on `CompactStatement`. #[derive(Encode, Decode)] enum CompactStatementInner { diff --git a/primitives/src/v1/mod.rs b/primitives/src/v1/mod.rs index 8950f689fa7a..2f9fdfe5f084 100644 --- a/primitives/src/v1/mod.rs +++ b/primitives/src/v1/mod.rs @@ -38,7 +38,7 @@ pub use polkadot_core_primitives::v1::{ // Export some polkadot-parachain primitives pub use polkadot_parachain::primitives::{ - Id, LOWEST_USER_ID, LOWEST_PUBLIC_ID, HrmpChannelId, UpwardMessage, HeadData, ValidationCode, + Id, LOWEST_USER_ID, LOWEST_PUBLIC_ID, HrmpChannelId, UpwardMessage, HeadData, ValidationCode, ValidationCodeHash, }; // Export some basic parachain primitives from v0. @@ -190,8 +190,21 @@ pub const ASSIGNMENT_KEY_TYPE_ID: KeyTypeId = KeyTypeId(*b"asgn"); /// Maximum compressed code size we support right now. /// At the moment we have runtime upgrade on chain, which restricts scalability severely. If we want /// to have bigger values, we should fix that first. +/// +/// Used for: +/// * initial genesis for the Parachains configuration +/// * checking updates to this stored runtime configuration do not exceed this limit +/// * when detecting a code decompression bomb in the client pub const MAX_CODE_SIZE: u32 = 3 * 1024 * 1024; +/// Maximum PoV size we support right now. +/// +/// Used for: +/// * initial genesis for the Parachains configuration +/// * checking updates to this stored runtime configuration do not exceed this limit +/// * when detecting a PoV decompression bomb in the client +pub const MAX_POV_SIZE: u32 = 5 * 1024 * 1024; + // The public key of a keypair used by a validator for determining assignments /// to approve included parachain candidates. mod assignment_app { @@ -228,7 +241,7 @@ pub fn collator_signature_payload>( para_id: &Id, persisted_validation_data_hash: &Hash, pov_hash: &Hash, - validation_code_hash: &Hash, + validation_code_hash: &ValidationCodeHash, ) -> [u8; 132] { // 32-byte hash length is protected in a test below. let mut payload = [0u8; 132]; @@ -247,10 +260,10 @@ fn check_collator_signature>( para_id: &Id, persisted_validation_data_hash: &Hash, pov_hash: &Hash, - validation_code_hash: &Hash, + validation_code_hash: &ValidationCodeHash, collator: &CollatorId, signature: &CollatorSignature, -) -> Result<(),()> { +) -> Result<(), ()> { let payload = collator_signature_payload( relay_parent, para_id, @@ -290,7 +303,7 @@ pub struct CandidateDescriptor { /// Hash of the para header that is being generated by this candidate. pub para_head: Hash, /// The blake2-256 hash of the validation code bytes. - pub validation_code_hash: Hash, + pub validation_code_hash: ValidationCodeHash, } impl> CandidateDescriptor { @@ -842,6 +855,22 @@ pub struct SessionInfo { pub needed_approvals: u32, } +/// A vote of approval on a candidate. +#[derive(Clone, RuntimeDebug)] +pub struct ApprovalVote(pub CandidateHash); + +impl ApprovalVote { + /// Yields the signing payload for this approval vote. + pub fn signing_payload( + &self, + session_index: SessionIndex, + ) -> Vec { + const MAGIC: [u8; 4] = *b"APPR"; + + (MAGIC, &self.0, session_index).encode() + } +} + sp_api::decl_runtime_apis! { /// The API for querying the state of parachains on-chain. pub trait ParachainHost { @@ -909,15 +938,6 @@ sp_api::decl_runtime_apis! { fn validation_code(para_id: Id, assumption: OccupiedCoreAssumption) -> Option; - /// Fetch the historical validation code used by a para for candidates executed in the - /// context of a given block height in the current chain. - /// - /// `context_height` may be no greater than the height of the block in whose - /// state the runtime API is executed. - #[skip_initialize_block] - fn historical_validation_code(para_id: Id, context_height: N) - -> Option; - /// Get the receipt of a candidate pending availability. This returns `Some` for any paras /// assigned to occupied cores in `availability_cores` and `None` otherwise. #[skip_initialize_block] @@ -940,7 +960,7 @@ sp_api::decl_runtime_apis! { /// Get the validation code from its hash. #[skip_initialize_block] - fn validation_code_by_hash(hash: Hash) -> Option; + fn validation_code_by_hash(hash: ValidationCodeHash) -> Option; } } @@ -1031,14 +1051,24 @@ pub const POLKADOT_ENGINE_ID: runtime_primitives::ConsensusEngineId = *b"POL1"; pub enum ConsensusLog { /// A parachain or parathread upgraded its code. #[codec(index = 1)] - ParaUpgradeCode(Id, Hash), + ParaUpgradeCode(Id, ValidationCodeHash), /// A parachain or parathread scheduled a code upgrade. #[codec(index = 2)] - ParaScheduleUpgradeCode(Id, Hash, BlockNumber), + ParaScheduleUpgradeCode(Id, ValidationCodeHash, BlockNumber), /// Governance requests to auto-approve every candidate included up to the given block /// number in the current chain, inclusive. #[codec(index = 3)] ForceApprove(BlockNumber), + /// A signal to revert the block number in the same chain as the + /// header this digest is part of and all of its descendents. + /// + /// It is a no-op for a block to contain a revert digest targeting + /// its own number or a higher number. + /// + /// In practice, these are issued when on-chain logic has detected an + /// invalid parachain block within its own chain, due to a dispute. + #[codec(index = 4)] + Revert(BlockNumber) } impl ConsensusLog { @@ -1073,6 +1103,60 @@ pub enum DisputeStatement { Invalid(InvalidDisputeStatementKind), } +impl DisputeStatement { + /// Get the payload data for this type of dispute statement. + pub fn payload_data(&self, candidate_hash: CandidateHash, session: SessionIndex) -> Vec { + match *self { + DisputeStatement::Valid(ValidDisputeStatementKind::Explicit) => { + ExplicitDisputeStatement { + valid: true, + candidate_hash, + session, + }.signing_payload() + }, + DisputeStatement::Valid(ValidDisputeStatementKind::BackingSeconded(inclusion_parent)) => { + CompactStatement::Seconded(candidate_hash).signing_payload(&SigningContext { + session_index: session, + parent_hash: inclusion_parent, + }) + }, + DisputeStatement::Valid(ValidDisputeStatementKind::BackingValid(inclusion_parent)) => { + CompactStatement::Valid(candidate_hash).signing_payload(&SigningContext { + session_index: session, + parent_hash: inclusion_parent, + }) + }, + DisputeStatement::Valid(ValidDisputeStatementKind::ApprovalChecking) => { + ApprovalVote(candidate_hash).signing_payload(session) + }, + DisputeStatement::Invalid(InvalidDisputeStatementKind::Explicit) => { + ExplicitDisputeStatement { + valid: false, + candidate_hash, + session, + }.signing_payload() + }, + } + } + + /// Check the signature on a dispute statement. + pub fn check_signature( + &self, + validator_public: &ValidatorId, + candidate_hash: CandidateHash, + session: SessionIndex, + validator_signature: &ValidatorSignature, + ) -> Result<(), ()> { + let payload = self.payload_data(candidate_hash, session); + + if validator_signature.verify(&payload[..] , &validator_public) { + Ok(()) + } else { + Err(()) + } + } +} + /// Different kinds of statements of validity on a candidate. #[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug)] pub enum ValidDisputeStatementKind { @@ -1081,10 +1165,10 @@ pub enum ValidDisputeStatementKind { Explicit, /// A seconded statement on a candidate from the backing phase. #[codec(index = 1)] - BackingSeconded, + BackingSeconded(Hash), /// A valid statement on a candidate from the backing phase. #[codec(index = 2)] - BackingValid, + BackingValid(Hash), /// An approval vote from the approval checking phase. #[codec(index = 3)] ApprovalChecking, @@ -1099,7 +1183,7 @@ pub enum InvalidDisputeStatementKind { } /// An explicit statement on a candidate issued as part of a dispute. -#[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug)] +#[derive(Clone, PartialEq, RuntimeDebug)] pub struct ExplicitDisputeStatement { /// Whether the candidate is valid pub valid: bool, @@ -1109,6 +1193,15 @@ pub struct ExplicitDisputeStatement { pub session: SessionIndex, } +impl ExplicitDisputeStatement { + /// Produce the payload used for signing this type of statement. + pub fn signing_payload(&self) -> Vec { + const MAGIC: [u8; 4] = *b"DISP"; + + (MAGIC, self.valid, self.candidate_hash, self.session).encode() + } +} + /// A set of statements about a specific candidate. #[derive(Encode, Decode, Clone, PartialEq, RuntimeDebug)] pub struct DisputeStatementSet { @@ -1149,6 +1242,19 @@ pub struct InherentData { pub parent_header: HDR, } +/// The maximum number of validators `f` which may safely be faulty. +/// +/// The total number of validators is `n = 3f + e` where `e in { 1, 2, 3 }`. +pub fn byzantine_threshold(n: usize) -> usize { + n.saturating_sub(1) / 3 +} + +/// The supermajority threshold of validators which represents a subset +/// guaranteed to have at least f+1 honest validators. +pub fn supermajority_threshold(n: usize) -> usize { + n - byzantine_threshold(n) +} + #[cfg(test)] mod tests { use super::*; @@ -1196,7 +1302,31 @@ mod tests { &5u32.into(), &Hash::repeat_byte(2), &Hash::repeat_byte(3), - &Hash::repeat_byte(4), + &Hash::repeat_byte(4).into(), ); } + + #[test] + fn test_byzantine_threshold() { + assert_eq!(byzantine_threshold(0), 0); + assert_eq!(byzantine_threshold(1), 0); + assert_eq!(byzantine_threshold(2), 0); + assert_eq!(byzantine_threshold(3), 0); + assert_eq!(byzantine_threshold(4), 1); + assert_eq!(byzantine_threshold(5), 1); + assert_eq!(byzantine_threshold(6), 1); + assert_eq!(byzantine_threshold(7), 2); + } + + #[test] + fn test_supermajority_threshold() { + assert_eq!(supermajority_threshold(0), 0); + assert_eq!(supermajority_threshold(1), 1); + assert_eq!(supermajority_threshold(2), 2); + assert_eq!(supermajority_threshold(3), 3); + assert_eq!(supermajority_threshold(4), 3); + assert_eq!(supermajority_threshold(5), 4); + assert_eq!(supermajority_threshold(6), 5); + assert_eq!(supermajority_threshold(7), 5); + } } From 6049ccbf376886a40f7e424c2999ece0ccb1d9b0 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 29 Jun 2021 16:10:01 +0200 Subject: [PATCH 073/161] split out metrics --- Cargo.lock | 672 +++++++----------- node/metrics/Cargo.toml | 19 + node/metrics/src/lib.rs | 123 ++++ node/subsystem-util/Cargo.toml | 3 +- .../src/determine_new_blocks.rs | 17 +- node/subsystem-util/src/lib.rs | 102 +-- .../src/rolling_session_window.rs | 2 +- node/subsystem-util/src/tests.rs | 3 +- 8 files changed, 409 insertions(+), 532 deletions(-) create mode 100644 node/metrics/Cargo.toml create mode 100644 node/metrics/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index a3bb19aa5664..6871396a086c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -481,7 +481,7 @@ dependencies = [ [[package]] name = "beefy-gadget" version = "0.1.0" -source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" +source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#29244671a1db0ee9c30b04f9a56f6bf2489522a5" dependencies = [ "beefy-primitives", "futures 0.3.15", @@ -509,7 +509,7 @@ dependencies = [ [[package]] name = "beefy-gadget-rpc" version = "0.1.0" -source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" +source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#29244671a1db0ee9c30b04f9a56f6bf2489522a5" dependencies = [ "beefy-gadget", "beefy-primitives", @@ -530,7 +530,7 @@ dependencies = [ [[package]] name = "beefy-primitives" version = "0.1.0" -source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" +source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#29244671a1db0ee9c30b04f9a56f6bf2489522a5" dependencies = [ "parity-scale-codec", "sp-api", @@ -540,22 +540,6 @@ dependencies = [ "sp-std", ] -[[package]] -name = "behavior-tests" -version = "0.1.0" -dependencies = [ - "env_logger 0.8.4", - "futures 0.3.15", - "log", - "polkadot-service", - "sc-chain-spec", - "sc-service", - "sp-core", - "test-runner", - "thiserror", - "tokio 0.2.21", -] - [[package]] name = "bincode" version = "1.3.1" @@ -739,17 +723,6 @@ dependencies = [ "sp-std", ] -[[package]] -name = "bp-kusama" -version = "0.1.0" -dependencies = [ - "bp-messages", - "bp-polkadot-core", - "bp-runtime", - "sp-api", - "sp-std", -] - [[package]] name = "bp-messages" version = "0.1.0" @@ -764,17 +737,6 @@ dependencies = [ "sp-std", ] -[[package]] -name = "bp-polkadot" -version = "0.1.0" -dependencies = [ - "bp-messages", - "bp-polkadot-core", - "bp-runtime", - "sp-api", - "sp-std", -] - [[package]] name = "bp-polkadot-core" version = "0.1.0" @@ -838,21 +800,6 @@ dependencies = [ "sp-std", ] -[[package]] -name = "bp-westend" -version = "0.1.0" -dependencies = [ - "bp-header-chain", - "bp-messages", - "bp-polkadot-core", - "bp-runtime", - "parity-scale-codec", - "sp-api", - "sp-runtime", - "sp-std", - "sp-version", -] - [[package]] name = "bp-wococo" version = "0.1.0" @@ -1211,9 +1158,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.1.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" +checksum = "ed00c67cb5d0a7d64a44f6ad2668db7e7530311dd53ea79bcd4fb022c64911c8" dependencies = [ "libc", ] @@ -1932,7 +1879,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" dependencies = [ "byteorder", - "rand 0.8.3", + "rand 0.8.4", "rustc-hex", "static_assertions", ] @@ -1965,7 +1912,7 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "fork-tree" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "parity-scale-codec", ] @@ -1983,14 +1930,14 @@ dependencies = [ [[package]] name = "frame-benchmarking" version = "3.1.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "frame-system", "linregress", "log", "parity-scale-codec", - "paste 1.0.4", + "paste 1.0.5", "sp-api", "sp-io", "sp-runtime", @@ -2002,7 +1949,7 @@ dependencies = [ [[package]] name = "frame-benchmarking-cli" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "Inflector", "chrono", @@ -2025,7 +1972,7 @@ dependencies = [ [[package]] name = "frame-election-provider-support" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "frame-system", @@ -2038,7 +1985,7 @@ dependencies = [ [[package]] name = "frame-executive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "frame-system", @@ -2053,7 +2000,7 @@ dependencies = [ [[package]] name = "frame-metadata" version = "13.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "parity-scale-codec", "serde", @@ -2064,7 +2011,7 @@ dependencies = [ [[package]] name = "frame-support" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "bitflags", "frame-metadata", @@ -2074,7 +2021,7 @@ dependencies = [ "max-encoded-len", "once_cell", "parity-scale-codec", - "paste 1.0.4", + "paste 1.0.5", "serde", "smallvec 1.6.1", "sp-arithmetic", @@ -2091,7 +2038,7 @@ dependencies = [ [[package]] name = "frame-support-procedural" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "Inflector", "frame-support-procedural-tools", @@ -2103,7 +2050,7 @@ dependencies = [ [[package]] name = "frame-support-procedural-tools" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate 1.0.0", @@ -2115,7 +2062,7 @@ dependencies = [ [[package]] name = "frame-support-procedural-tools-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "proc-macro2", "quote", @@ -2125,7 +2072,7 @@ dependencies = [ [[package]] name = "frame-support-test" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-metadata", "frame-support", @@ -2145,7 +2092,7 @@ dependencies = [ [[package]] name = "frame-system" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "impl-trait-for-tuples", @@ -2162,7 +2109,7 @@ dependencies = [ [[package]] name = "frame-system-benchmarking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -2176,7 +2123,7 @@ dependencies = [ [[package]] name = "frame-system-rpc-runtime-api" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "parity-scale-codec", "sp-api", @@ -2185,7 +2132,7 @@ dependencies = [ [[package]] name = "frame-try-runtime" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "parity-scale-codec", @@ -2303,22 +2250,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "futures-diagnose" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdcef58a173af8148b182684c9f2d5250875adbcaff7b5794073894f9d8634a9" -dependencies = [ - "futures 0.1.29", - "futures 0.3.15", - "lazy_static", - "log", - "parking_lot 0.9.0", - "pin-project 0.4.23", - "serde", - "serde_json", -] - [[package]] name = "futures-executor" version = "0.3.15" @@ -2590,7 +2521,7 @@ dependencies = [ "log", "slab", "tokio 0.2.21", - "tokio-util 0.3.1", + "tokio-util", ] [[package]] @@ -3301,7 +3232,7 @@ dependencies = [ "thiserror", "tokio 0.2.21", "tokio-rustls 0.15.0", - "tokio-util 0.3.1", + "tokio-util", "url 2.2.0", ] @@ -3497,9 +3428,9 @@ checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" [[package]] name = "libc" -version = "0.2.97" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" +checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7" [[package]] name = "libloading" @@ -3726,7 +3657,7 @@ dependencies = [ "libp2p-core", "libp2p-swarm", "log", - "rand 0.8.3", + "rand 0.8.4", "smallvec 1.6.1", "socket2 0.4.0", "void", @@ -4133,7 +4064,7 @@ dependencies = [ [[package]] name = "max-encoded-len" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "impl-trait-for-tuples", "max-encoded-len-derive", @@ -4144,7 +4075,7 @@ dependencies = [ [[package]] name = "max-encoded-len-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -4647,7 +4578,7 @@ checksum = "13370dae44474229701bb69b90b4f4dca6404cb0357a2d50d635f1171dc3aa7b" [[package]] name = "pallet-authority-discovery" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "frame-system", @@ -4662,7 +4593,7 @@ dependencies = [ [[package]] name = "pallet-authorship" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "frame-system", @@ -4676,7 +4607,7 @@ dependencies = [ [[package]] name = "pallet-babe" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -4699,7 +4630,7 @@ dependencies = [ [[package]] name = "pallet-balances" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -4714,7 +4645,7 @@ dependencies = [ [[package]] name = "pallet-beefy" version = "0.1.0" -source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#a58179d4f59f4c31c7a09d5b0ce14df8450743a3" +source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#29244671a1db0ee9c30b04f9a56f6bf2489522a5" dependencies = [ "beefy-primitives", "frame-support", @@ -4729,7 +4660,7 @@ dependencies = [ [[package]] name = "pallet-bounties" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -4765,7 +4696,7 @@ dependencies = [ [[package]] name = "pallet-collective" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -4781,7 +4712,7 @@ dependencies = [ [[package]] name = "pallet-democracy" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -4796,7 +4727,7 @@ dependencies = [ [[package]] name = "pallet-election-provider-multi-phase" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -4817,7 +4748,7 @@ dependencies = [ [[package]] name = "pallet-elections-phragmen" version = "4.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -4834,7 +4765,7 @@ dependencies = [ [[package]] name = "pallet-gilt" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -4848,7 +4779,7 @@ dependencies = [ [[package]] name = "pallet-grandpa" version = "3.1.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -4870,7 +4801,7 @@ dependencies = [ [[package]] name = "pallet-identity" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "enumflags2", "frame-benchmarking", @@ -4885,7 +4816,7 @@ dependencies = [ [[package]] name = "pallet-im-online" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -4904,7 +4835,7 @@ dependencies = [ [[package]] name = "pallet-indices" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -4920,7 +4851,7 @@ dependencies = [ [[package]] name = "pallet-membership" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -4935,7 +4866,7 @@ dependencies = [ [[package]] name = "pallet-mmr" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "ckb-merkle-mountain-range", "frame-benchmarking", @@ -4952,7 +4883,7 @@ dependencies = [ [[package]] name = "pallet-mmr-primitives" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "frame-system", @@ -4968,7 +4899,7 @@ dependencies = [ [[package]] name = "pallet-mmr-rpc" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -4986,7 +4917,7 @@ dependencies = [ [[package]] name = "pallet-multisig" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -5001,7 +4932,7 @@ dependencies = [ [[package]] name = "pallet-nicks" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "frame-system", @@ -5014,7 +4945,7 @@ dependencies = [ [[package]] name = "pallet-offences" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "frame-system", @@ -5030,7 +4961,7 @@ dependencies = [ [[package]] name = "pallet-offences-benchmarking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -5052,7 +4983,7 @@ dependencies = [ [[package]] name = "pallet-proxy" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -5068,7 +4999,7 @@ dependencies = [ [[package]] name = "pallet-recovery" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "enumflags2", "frame-support", @@ -5082,7 +5013,7 @@ dependencies = [ [[package]] name = "pallet-scheduler" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -5097,7 +5028,7 @@ dependencies = [ [[package]] name = "pallet-session" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "frame-system", @@ -5117,7 +5048,7 @@ dependencies = [ [[package]] name = "pallet-session-benchmarking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -5133,7 +5064,7 @@ dependencies = [ [[package]] name = "pallet-society" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "frame-system", @@ -5146,7 +5077,7 @@ dependencies = [ [[package]] name = "pallet-staking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -5156,7 +5087,7 @@ dependencies = [ "pallet-authorship", "pallet-session", "parity-scale-codec", - "paste 1.0.4", + "paste 1.0.5", "rand_chacha 0.2.2", "serde", "sp-application-crypto", @@ -5170,7 +5101,7 @@ dependencies = [ [[package]] name = "pallet-staking-reward-curve" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -5181,7 +5112,7 @@ dependencies = [ [[package]] name = "pallet-staking-reward-fn" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "log", "sp-arithmetic", @@ -5190,7 +5121,7 @@ dependencies = [ [[package]] name = "pallet-sudo" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "frame-system", @@ -5203,7 +5134,7 @@ dependencies = [ [[package]] name = "pallet-timestamp" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -5221,7 +5152,7 @@ dependencies = [ [[package]] name = "pallet-tips" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -5236,7 +5167,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-support", "frame-system", @@ -5252,7 +5183,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -5269,7 +5200,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc-runtime-api" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "pallet-transaction-payment", "parity-scale-codec", @@ -5280,7 +5211,7 @@ dependencies = [ [[package]] name = "pallet-treasury" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -5296,7 +5227,7 @@ dependencies = [ [[package]] name = "pallet-utility" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-benchmarking", "frame-support", @@ -5311,7 +5242,7 @@ dependencies = [ [[package]] name = "pallet-vesting" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "enumflags2", "frame-benchmarking", @@ -5350,7 +5281,7 @@ dependencies = [ "log", "memmap2", "parking_lot 0.11.1", - "rand 0.8.3", + "rand 0.8.4", ] [[package]] @@ -5578,9 +5509,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d65c4d95931acda4498f675e332fcbdc9a06705cd07086c510e9b6009cd1c1" +checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" [[package]] name = "paste-impl" @@ -5828,7 +5759,7 @@ dependencies = [ "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", "polkadot-primitives", - "rand 0.8.3", + "rand 0.8.4", "sc-keystore", "sc-network", "smallvec 1.6.1", @@ -5859,7 +5790,7 @@ dependencies = [ "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", "polkadot-primitives", - "rand 0.8.3", + "rand 0.8.4", "sc-network", "smallvec 1.6.1", "sp-application-crypto", @@ -5937,7 +5868,6 @@ dependencies = [ "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", - "polkadot-overseer", "polkadot-primitives", "sp-core", "sp-keyring", @@ -5982,7 +5912,7 @@ dependencies = [ "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", "polkadot-primitives", - "rand 0.8.3", + "rand 0.8.4", "rand_chacha 0.3.1", "sc-keystore", "sp-application-crypto", @@ -6007,7 +5937,6 @@ dependencies = [ "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", - "polkadot-overseer", "polkadot-primitives", "sc-authority-discovery", "sc-network", @@ -6290,7 +6219,7 @@ dependencies = [ "pin-project 1.0.7", "polkadot-core-primitives", "polkadot-parachain", - "rand 0.8.3", + "rand 0.8.4", "sc-executor", "sc-executor-common", "sc-executor-wasmtime", @@ -6382,62 +6311,52 @@ dependencies = [ name = "polkadot-node-subsystem" version = "0.1.0" dependencies = [ - "polkadot-node-jaeger", - "polkadot-node-subsystem-types", - "polkadot-overseer", - "polkadot-overseer-gen", -] - -[[package]] -name = "polkadot-node-subsystem-test-helpers" -version = "0.1.0" -dependencies = [ + "assert_matches", + "async-std", "async-trait", + "derive_more", "futures 0.3.15", "futures-timer 3.0.2", + "lazy_static", + "log", + "mick-jaeger", "parity-scale-codec", "parking_lot 0.11.1", "pin-project 1.0.7", + "polkadot-node-jaeger", + "polkadot-node-network-protocol", "polkadot-node-primitives", - "polkadot-node-subsystem", - "polkadot-node-subsystem-util", - "polkadot-overseer", + "polkadot-node-subsystem-test-helpers", "polkadot-primitives", + "polkadot-procmacro-subsystem-dispatch-gen", "polkadot-statement-table", "sc-network", "smallvec 1.6.1", "sp-core", + "substrate-prometheus-endpoint", + "thiserror", "tracing", ] [[package]] -name = "polkadot-node-subsystem-types" +name = "polkadot-node-subsystem-test-helpers" version = "0.1.0" dependencies = [ - "assert_matches", - "async-std", "async-trait", - "derive_more", "futures 0.3.15", "futures-timer 3.0.2", - "lazy_static", - "log", - "mick-jaeger", "parity-scale-codec", "parking_lot 0.11.1", "pin-project 1.0.7", - "polkadot-node-jaeger", - "polkadot-node-network-protocol", "polkadot-node-primitives", - "polkadot-node-subsystem-test-helpers", - "polkadot-overseer-gen", + "polkadot-node-subsystem", + "polkadot-node-subsystem-util", + "polkadot-overseer", "polkadot-primitives", "polkadot-statement-table", "sc-network", "smallvec 1.6.1", "sp-core", - "substrate-prometheus-endpoint", - "thiserror", "tracing", ] @@ -6460,11 +6379,10 @@ dependencies = [ "polkadot-node-jaeger", "polkadot-node-network-protocol", "polkadot-node-primitives", + "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", - "polkadot-node-subsystem-types", - "polkadot-overseer-gen", "polkadot-primitives", - "rand 0.8.3", + "rand 0.8.4", "sc-network", "sp-application-crypto", "sp-core", @@ -6487,48 +6405,16 @@ dependencies = [ "lru", "polkadot-node-network-protocol", "polkadot-node-primitives", - "polkadot-node-subsystem-types", + "polkadot-node-subsystem", "polkadot-node-subsystem-util", - "polkadot-overseer-gen", "polkadot-primitives", + "polkadot-procmacro-overseer-subsystems-gen", "sc-client-api", "sp-api", "sp-core", "tracing", ] -[[package]] -name = "polkadot-overseer-gen" -version = "0.1.0" -dependencies = [ - "async-trait", - "futures 0.3.15", - "futures-timer 3.0.2", - "futures-util", - "metered-channel", - "pin-project 1.0.7", - "polkadot-node-network-protocol", - "polkadot-node-subsystem", - "polkadot-overseer-gen-proc-macro", - "sp-core", - "thiserror", - "tokio-util 0.6.7", - "tracing", - "trybuild", -] - -[[package]] -name = "polkadot-overseer-gen-proc-macro" -version = "0.1.0" -dependencies = [ - "assert_matches", - "proc-macro-crate 1.0.0", - "proc-macro2", - "quote", - "syn", - "trybuild", -] - [[package]] name = "polkadot-parachain" version = "0.9.7" @@ -6574,6 +6460,28 @@ dependencies = [ "thiserror", ] +[[package]] +name = "polkadot-procmacro-overseer-subsystems-gen" +version = "0.1.0" +dependencies = [ + "assert_matches", + "proc-macro2", + "quote", + "syn", + "trybuild", +] + +[[package]] +name = "polkadot-procmacro-subsystem-dispatch-gen" +version = "0.1.0" +dependencies = [ + "assert_matches", + "proc-macro2", + "quote", + "syn", + "trybuild", +] + [[package]] name = "polkadot-rpc" version = "0.9.7" @@ -6768,7 +6676,7 @@ dependencies = [ "pallet-vesting", "parity-scale-codec", "polkadot-primitives", - "rand 0.8.3", + "rand 0.8.4", "rand_chacha 0.3.1", "rustc-hex", "sc-keystore", @@ -7045,7 +6953,7 @@ dependencies = [ "polkadot-runtime-parachains", "polkadot-service", "polkadot-test-runtime", - "rand 0.8.3", + "rand 0.8.4", "sc-authority-discovery", "sc-chain-spec", "sc-cli", @@ -7235,9 +7143,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.27" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" dependencies = [ "unicode-xid", ] @@ -7404,9 +7312,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -7667,7 +7575,7 @@ dependencies = [ [[package]] name = "remote-externalities" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "env_logger 0.8.4", "hex", @@ -7904,7 +7812,7 @@ dependencies = [ "openssl-probe", "rustls 0.19.1", "schannel", - "security-framework 2.3.1", + "security-framework 2.2.0", ] [[package]] @@ -7961,7 +7869,7 @@ dependencies = [ [[package]] name = "sc-authority-discovery" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "derive_more", @@ -7990,7 +7898,7 @@ dependencies = [ [[package]] name = "sc-basic-authorship" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "futures 0.3.15", "futures-timer 3.0.2", @@ -8013,7 +7921,7 @@ dependencies = [ [[package]] name = "sc-block-builder" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "parity-scale-codec", "sc-client-api", @@ -8029,7 +7937,7 @@ dependencies = [ [[package]] name = "sc-chain-spec" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -8050,7 +7958,7 @@ dependencies = [ [[package]] name = "sc-chain-spec-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -8061,7 +7969,7 @@ dependencies = [ [[package]] name = "sc-cli" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "chrono", "fdlimit", @@ -8099,7 +8007,7 @@ dependencies = [ [[package]] name = "sc-client-api" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "derive_more", "fnv", @@ -8133,7 +8041,7 @@ dependencies = [ [[package]] name = "sc-client-db" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "blake2-rfc", "hash-db", @@ -8163,8 +8071,9 @@ dependencies = [ [[package]] name = "sc-consensus" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ + "async-trait", "parking_lot 0.11.1", "sc-client-api", "sp-blockchain", @@ -8175,7 +8084,7 @@ dependencies = [ [[package]] name = "sc-consensus-babe" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "derive_more", @@ -8221,7 +8130,7 @@ dependencies = [ [[package]] name = "sc-consensus-babe-rpc" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "derive_more", "futures 0.3.15", @@ -8245,7 +8154,7 @@ dependencies = [ [[package]] name = "sc-consensus-epochs" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "fork-tree", "parity-scale-codec", @@ -8255,45 +8164,10 @@ dependencies = [ "sp-runtime", ] -[[package]] -name = "sc-consensus-manual-seal" -version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" -dependencies = [ - "assert_matches", - "async-trait", - "derive_more", - "futures 0.3.15", - "jsonrpc-core", - "jsonrpc-core-client", - "jsonrpc-derive", - "log", - "parity-scale-codec", - "parking_lot 0.11.1", - "sc-client-api", - "sc-consensus-babe", - "sc-consensus-epochs", - "sc-transaction-pool", - "serde", - "sp-api", - "sp-blockchain", - "sp-consensus", - "sp-consensus-babe", - "sp-consensus-slots", - "sp-core", - "sp-inherents", - "sp-keyring", - "sp-keystore", - "sp-runtime", - "sp-timestamp", - "sp-transaction-pool", - "substrate-prometheus-endpoint", -] - [[package]] name = "sc-consensus-slots" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "futures 0.3.15", @@ -8321,7 +8195,7 @@ dependencies = [ [[package]] name = "sc-consensus-uncles" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "sc-client-api", "sp-authorship", @@ -8332,7 +8206,7 @@ dependencies = [ [[package]] name = "sc-executor" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "derive_more", "lazy_static", @@ -8361,7 +8235,7 @@ dependencies = [ [[package]] name = "sc-executor-common" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "derive_more", "parity-scale-codec", @@ -8378,7 +8252,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmi" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "log", "parity-scale-codec", @@ -8393,7 +8267,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmtime" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "cfg-if 1.0.0", "libc", @@ -8412,7 +8286,7 @@ dependencies = [ [[package]] name = "sc-finality-grandpa" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "derive_more", @@ -8453,7 +8327,7 @@ dependencies = [ [[package]] name = "sc-finality-grandpa-rpc" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "derive_more", "finality-grandpa", @@ -8477,7 +8351,7 @@ dependencies = [ [[package]] name = "sc-finality-grandpa-warp-sync" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "derive_more", "futures 0.3.15", @@ -8498,7 +8372,7 @@ dependencies = [ [[package]] name = "sc-informant" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "ansi_term 0.12.1", "futures 0.3.15", @@ -8516,7 +8390,7 @@ dependencies = [ [[package]] name = "sc-keystore" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "derive_more", @@ -8536,7 +8410,7 @@ dependencies = [ [[package]] name = "sc-light" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "hash-db", "lazy_static", @@ -8555,7 +8429,7 @@ dependencies = [ [[package]] name = "sc-network" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-std", "async-trait", @@ -8608,7 +8482,7 @@ dependencies = [ [[package]] name = "sc-network-gossip" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "futures 0.3.15", "futures-timer 3.0.2", @@ -8625,7 +8499,7 @@ dependencies = [ [[package]] name = "sc-offchain" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "bytes 0.5.6", "fnv", @@ -8653,7 +8527,7 @@ dependencies = [ [[package]] name = "sc-peerset" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "futures 0.3.15", "libp2p", @@ -8666,7 +8540,7 @@ dependencies = [ [[package]] name = "sc-proposer-metrics" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "log", "substrate-prometheus-endpoint", @@ -8675,7 +8549,7 @@ dependencies = [ [[package]] name = "sc-rpc" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "futures 0.3.15", "hash-db", @@ -8710,7 +8584,7 @@ dependencies = [ [[package]] name = "sc-rpc-api" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "derive_more", "futures 0.3.15", @@ -8735,7 +8609,7 @@ dependencies = [ [[package]] name = "sc-rpc-server" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "futures 0.1.29", "jsonrpc-core", @@ -8753,7 +8627,7 @@ dependencies = [ [[package]] name = "sc-service" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "directories", @@ -8819,7 +8693,7 @@ dependencies = [ [[package]] name = "sc-state-db" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "log", "parity-scale-codec", @@ -8834,7 +8708,7 @@ dependencies = [ [[package]] name = "sc-sync-state-rpc" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -8854,7 +8728,7 @@ dependencies = [ [[package]] name = "sc-telemetry" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "chrono", "futures 0.3.15", @@ -8874,7 +8748,7 @@ dependencies = [ [[package]] name = "sc-tracing" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "ansi_term 0.12.1", "atty", @@ -8911,7 +8785,7 @@ dependencies = [ [[package]] name = "sc-tracing-proc-macro" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -8922,7 +8796,7 @@ dependencies = [ [[package]] name = "sc-transaction-graph" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "derive_more", "futures 0.3.15", @@ -8944,10 +8818,9 @@ dependencies = [ [[package]] name = "sc-transaction-pool" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "futures 0.3.15", - "futures-diagnose", "intervalier", "log", "parity-scale-codec", @@ -8990,7 +8863,6 @@ dependencies = [ "merlin", "rand 0.7.3", "rand_core 0.5.1", - "serde", "sha2 0.8.2", "subtle 2.2.3", "zeroize", @@ -9062,15 +8934,15 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.3.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" dependencies = [ "bitflags", "core-foundation 0.9.1", "core-foundation-sys 0.8.2", "libc", - "security-framework-sys 2.3.0", + "security-framework-sys 2.2.0", ] [[package]] @@ -9085,9 +8957,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" +checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339" dependencies = [ "core-foundation-sys 0.8.2", "libc", @@ -9322,7 +9194,7 @@ version = "0.9.7" dependencies = [ "enumn", "parity-scale-codec", - "paste 1.0.4", + "paste 1.0.5", "sp-runtime", "sp-std", ] @@ -9418,14 +9290,14 @@ dependencies = [ "futures 0.3.15", "httparse", "log", - "rand 0.8.3", + "rand 0.8.4", "sha-1 0.9.6", ] [[package]] name = "sp-allocator" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "log", "sp-core", @@ -9437,7 +9309,7 @@ dependencies = [ [[package]] name = "sp-api" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "hash-db", "log", @@ -9454,7 +9326,7 @@ dependencies = [ [[package]] name = "sp-api-proc-macro" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "blake2-rfc", "proc-macro-crate 1.0.0", @@ -9466,7 +9338,7 @@ dependencies = [ [[package]] name = "sp-application-crypto" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "max-encoded-len", "parity-scale-codec", @@ -9479,7 +9351,7 @@ dependencies = [ [[package]] name = "sp-arithmetic" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "integer-sqrt", "num-traits", @@ -9493,7 +9365,7 @@ dependencies = [ [[package]] name = "sp-authority-discovery" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "parity-scale-codec", "sp-api", @@ -9505,7 +9377,7 @@ dependencies = [ [[package]] name = "sp-authorship" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "parity-scale-codec", @@ -9517,7 +9389,7 @@ dependencies = [ [[package]] name = "sp-block-builder" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "parity-scale-codec", "sp-api", @@ -9529,7 +9401,7 @@ dependencies = [ [[package]] name = "sp-blockchain" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "futures 0.3.15", "log", @@ -9547,7 +9419,7 @@ dependencies = [ [[package]] name = "sp-chain-spec" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "serde", "serde_json", @@ -9556,7 +9428,7 @@ dependencies = [ [[package]] name = "sp-consensus" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "futures 0.3.15", @@ -9583,7 +9455,7 @@ dependencies = [ [[package]] name = "sp-consensus-babe" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "merlin", @@ -9605,7 +9477,7 @@ dependencies = [ [[package]] name = "sp-consensus-slots" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "parity-scale-codec", "sp-arithmetic", @@ -9615,7 +9487,7 @@ dependencies = [ [[package]] name = "sp-consensus-vrf" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "parity-scale-codec", "schnorrkel", @@ -9627,7 +9499,7 @@ dependencies = [ [[package]] name = "sp-core" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "base58", "blake2-rfc", @@ -9672,7 +9544,7 @@ dependencies = [ [[package]] name = "sp-database" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "kvdb", "parking_lot 0.11.1", @@ -9681,7 +9553,7 @@ dependencies = [ [[package]] name = "sp-debug-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "proc-macro2", "quote", @@ -9691,7 +9563,7 @@ dependencies = [ [[package]] name = "sp-externalities" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "environmental", "parity-scale-codec", @@ -9702,7 +9574,7 @@ dependencies = [ [[package]] name = "sp-finality-grandpa" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "finality-grandpa", "log", @@ -9719,7 +9591,7 @@ dependencies = [ [[package]] name = "sp-inherents" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "impl-trait-for-tuples", @@ -9733,7 +9605,7 @@ dependencies = [ [[package]] name = "sp-io" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "futures 0.3.15", "hash-db", @@ -9758,7 +9630,7 @@ dependencies = [ [[package]] name = "sp-keyring" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "lazy_static", "sp-core", @@ -9769,7 +9641,7 @@ dependencies = [ [[package]] name = "sp-keystore" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "derive_more", @@ -9786,7 +9658,7 @@ dependencies = [ [[package]] name = "sp-maybe-compressed-blob" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "ruzstd", "zstd", @@ -9795,7 +9667,7 @@ dependencies = [ [[package]] name = "sp-npos-elections" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "parity-scale-codec", "serde", @@ -9808,7 +9680,7 @@ dependencies = [ [[package]] name = "sp-npos-elections-compact" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -9819,7 +9691,7 @@ dependencies = [ [[package]] name = "sp-offchain" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "sp-api", "sp-core", @@ -9829,7 +9701,7 @@ dependencies = [ [[package]] name = "sp-panic-handler" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "backtrace", ] @@ -9837,7 +9709,7 @@ dependencies = [ [[package]] name = "sp-rpc" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "rustc-hash", "serde", @@ -9848,7 +9720,7 @@ dependencies = [ [[package]] name = "sp-runtime" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "either", "hash256-std-hasher", @@ -9857,7 +9729,7 @@ dependencies = [ "max-encoded-len", "parity-scale-codec", "parity-util-mem", - "paste 1.0.4", + "paste 1.0.5", "rand 0.7.3", "serde", "sp-application-crypto", @@ -9870,7 +9742,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -9887,7 +9759,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "Inflector", "proc-macro-crate 1.0.0", @@ -9899,7 +9771,7 @@ dependencies = [ [[package]] name = "sp-serializer" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "serde", "serde_json", @@ -9908,7 +9780,7 @@ dependencies = [ [[package]] name = "sp-session" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "parity-scale-codec", "sp-api", @@ -9921,7 +9793,7 @@ dependencies = [ [[package]] name = "sp-staking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -9931,7 +9803,7 @@ dependencies = [ [[package]] name = "sp-state-machine" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "hash-db", "log", @@ -9954,12 +9826,12 @@ dependencies = [ [[package]] name = "sp-std" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" [[package]] name = "sp-storage" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "impl-serde", "parity-scale-codec", @@ -9972,7 +9844,7 @@ dependencies = [ [[package]] name = "sp-tasks" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "log", "sp-core", @@ -9985,7 +9857,7 @@ dependencies = [ [[package]] name = "sp-timestamp" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "futures-timer 3.0.2", @@ -10002,7 +9874,7 @@ dependencies = [ [[package]] name = "sp-tracing" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "erased-serde", "log", @@ -10020,7 +9892,7 @@ dependencies = [ [[package]] name = "sp-transaction-pool" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "derive_more", "futures 0.3.15", @@ -10036,7 +9908,7 @@ dependencies = [ [[package]] name = "sp-transaction-storage-proof" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "log", @@ -10051,7 +9923,7 @@ dependencies = [ [[package]] name = "sp-trie" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "hash-db", "memory-db", @@ -10065,7 +9937,7 @@ dependencies = [ [[package]] name = "sp-utils" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "futures 0.3.15", "futures-core", @@ -10077,7 +9949,7 @@ dependencies = [ [[package]] name = "sp-version" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "impl-serde", "parity-scale-codec", @@ -10090,7 +9962,7 @@ dependencies = [ [[package]] name = "sp-version-proc-macro" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "parity-scale-codec", "proc-macro-crate 1.0.0", @@ -10102,7 +9974,7 @@ dependencies = [ [[package]] name = "sp-wasm-interface" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -10247,7 +10119,7 @@ dependencies = [ [[package]] name = "substrate-browser-utils" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "chrono", "console_error_panic_hook", @@ -10273,7 +10145,7 @@ dependencies = [ [[package]] name = "substrate-build-script-utils" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "platforms", ] @@ -10281,7 +10153,7 @@ dependencies = [ [[package]] name = "substrate-frame-rpc-system" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-system-rpc-runtime-api", "futures 0.3.15", @@ -10304,7 +10176,7 @@ dependencies = [ [[package]] name = "substrate-prometheus-endpoint" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-std", "derive_more", @@ -10318,7 +10190,7 @@ dependencies = [ [[package]] name = "substrate-test-client" version = "2.0.1" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "async-trait", "futures 0.1.29", @@ -10347,7 +10219,7 @@ dependencies = [ [[package]] name = "substrate-test-utils" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "futures 0.3.15", "substrate-test-utils-derive", @@ -10357,7 +10229,7 @@ dependencies = [ [[package]] name = "substrate-test-utils-derive" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "proc-macro-crate 1.0.0", "quote", @@ -10441,7 +10313,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ "cfg-if 1.0.0", "libc", - "rand 0.8.3", + "rand 0.8.4", "redox_syscall 0.2.4", "remove_dir_all", "winapi 0.3.9", @@ -10515,48 +10387,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "test-runner" -version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" -dependencies = [ - "env_logger 0.7.1", - "frame-system", - "futures 0.3.15", - "jsonrpc-core", - "log", - "sc-basic-authorship", - "sc-cli", - "sc-client-api", - "sc-consensus-manual-seal", - "sc-executor", - "sc-informant", - "sc-network", - "sc-rpc", - "sc-rpc-server", - "sc-service", - "sc-transaction-graph", - "sc-transaction-pool", - "sp-api", - "sp-block-builder", - "sp-blockchain", - "sp-consensus", - "sp-core", - "sp-externalities", - "sp-inherents", - "sp-io", - "sp-keyring", - "sp-keystore", - "sp-offchain", - "sp-runtime", - "sp-runtime-interface", - "sp-session", - "sp-state-machine", - "sp-transaction-pool", - "sp-wasm-interface", - "tokio 0.2.21", -] - [[package]] name = "textwrap" version = "0.11.0" @@ -10713,7 +10543,6 @@ dependencies = [ "libc", "memchr", "mio", - "mio-named-pipes", "mio-uds", "num_cpus", "pin-project-lite 0.1.7", @@ -10974,21 +10803,6 @@ dependencies = [ "tokio 0.2.21", ] -[[package]] -name = "tokio-util" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" -dependencies = [ - "bytes 1.0.1", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.2.4", - "slab", - "tokio 1.6.1", -] - [[package]] name = "toml" version = "0.5.6" @@ -11070,9 +10884,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.19" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab69019741fca4d98be3c62d2b75254528b5432233fd8a4d2739fec20278de48" +checksum = "aa5553bf0883ba7c9cbe493b085c29926bd41b66afc31ff72cf17ff4fb60dcd5" dependencies = [ "ansi_term 0.12.1", "chrono", @@ -11098,9 +10912,9 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-db" -version = "0.22.3" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec051edf7f0fc9499a2cb0947652cab2148b9d7f61cee7605e312e9f970dacaf" +checksum = "cd81fe0c8bc2b528a51c9d2c31dae4483367a26a723a3c9a4a8120311d7774e3" dependencies = [ "hash-db", "hashbrown", @@ -11135,7 +10949,7 @@ dependencies = [ "ipnet", "lazy_static", "log", - "rand 0.8.3", + "rand 0.8.4", "smallvec 1.6.1", "thiserror", "tinyvec 1.1.1", @@ -11170,7 +10984,7 @@ checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" [[package]] name = "try-runtime-cli" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#0a2472d8364bc103a0a13c8e2dcb7f6ed3e44342" +source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" dependencies = [ "frame-try-runtime", "log", @@ -11593,7 +11407,7 @@ dependencies = [ "lazy_static", "libc", "log", - "paste 1.0.4", + "paste 1.0.5", "psm", "region", "rustc-demangle", @@ -11776,7 +11590,7 @@ dependencies = [ "mach", "memoffset 0.6.1", "more-asserts", - "rand 0.8.3", + "rand 0.8.4", "region", "thiserror", "wasmtime-environ", @@ -12086,7 +11900,7 @@ dependencies = [ "log", "nohash-hasher", "parking_lot 0.11.1", - "rand 0.8.3", + "rand 0.8.4", "static_assertions", ] diff --git a/node/metrics/Cargo.toml b/node/metrics/Cargo.toml new file mode 100644 index 000000000000..6fb4ce12f6ed --- /dev/null +++ b/node/metrics/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "polkadot-node-metrics" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" +description = "Subsystem traits and message definitions" + +[dependencies] +async-trait = "0.1.42" +futures = "0.3.15" +futures-timer = "3.0.2" + +metered-channel = { path = "../metered-channel"} + +sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } +substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/metrics/src/lib.rs b/node/metrics/src/lib.rs new file mode 100644 index 000000000000..806970b9ab70 --- /dev/null +++ b/node/metrics/src/lib.rs @@ -0,0 +1,123 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Utility module for subsystems +//! +//! Many subsystems have common interests such as canceling a bunch of spawned jobs, +//! or determining what their validator ID is. These common interests are factored into +//! this module. +//! +//! This crate also reexports Prometheus metric types which are expected to be implemented by subsystems. + +#![warn(missing_docs)] + +use futures::prelude::*; +use futures_timer::Delay; +use std::{ + pin::Pin, + task::{Poll, Context}, + time::Duration, +}; + +pub use metered_channel as metered; + +/// This module reexports Prometheus types and defines the [`Metrics`] trait. +pub mod metrics { + /// Reexport Substrate Prometheus types. + pub use substrate_prometheus_endpoint as prometheus; + + + /// Subsystem- or job-specific Prometheus metrics. + /// + /// Usually implemented as a wrapper for `Option` + /// to ensure `Default` bounds or as a dummy type (). + /// Prometheus metrics internally hold an `Arc` reference, so cloning them is fine. + pub trait Metrics: Default + Clone { + /// Try to register metrics in the Prometheus registry. + fn try_register(registry: &prometheus::Registry) -> Result; + + /// Convenience method to register metrics in the optional Promethius registry. + /// + /// If no registry is provided, returns `Default::default()`. Otherwise, returns the same + /// thing that `try_register` does. + fn register(registry: Option<&prometheus::Registry>) -> Result { + match registry { + None => Ok(Self::default()), + Some(registry) => Self::try_register(registry), + } + } + } + + // dummy impl + impl Metrics for () { + fn try_register(_registry: &prometheus::Registry) -> Result<(), prometheus::PrometheusError> { + Ok(()) + } + } +} + +#[derive(Copy, Clone)] +enum MetronomeState { + Snooze, + SetAlarm, +} + +/// Create a stream of ticks with a defined cycle duration. +pub struct Metronome { + delay: Delay, + period: Duration, + state: MetronomeState, +} + +impl Metronome +{ + /// Create a new metronome source with a defined cycle duration. + pub fn new(cycle: Duration) -> Self { + let period = cycle.into(); + Self { + period, + delay: Delay::new(period), + state: MetronomeState::Snooze, + } + } +} + +impl futures::Stream for Metronome +{ + type Item = (); + fn poll_next( + mut self: Pin<&mut Self>, + cx: &mut Context<'_> + ) -> Poll> { + loop { + match self.state { + MetronomeState::SetAlarm => { + let val = self.period.clone(); + self.delay.reset(val); + self.state = MetronomeState::Snooze; + } + MetronomeState::Snooze => { + if !Pin::new(&mut self.delay).poll(cx).is_ready() { + break + } + self.state = MetronomeState::SetAlarm; + return Poll::Ready(Some(())); + } + } + } + Poll::Pending + } +} diff --git a/node/subsystem-util/Cargo.toml b/node/subsystem-util/Cargo.toml index 7e919c68bff9..0d8b583e611a 100644 --- a/node/subsystem-util/Cargo.toml +++ b/node/subsystem-util/Cargo.toml @@ -21,9 +21,10 @@ lru = "0.6.5" polkadot-node-primitives = { path = "../primitives" } polkadot-node-subsystem = { package = "polkadot-node-subsystem-types", path = "../subsystem-types" } polkadot-node-jaeger = { path = "../jaeger" } +polkadot-node-metrics = { path = "../metrics" } polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-primitives = { path = "../../primitives" } -polkadot-overseer-gen = { path = "../overseer/overseer-gen" } +polkadot-overseer = { path = "../overseer" } metered-channel = { path = "../metered-channel"} sc-network = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/subsystem-util/src/determine_new_blocks.rs b/node/subsystem-util/src/determine_new_blocks.rs index adfc614beef9..1566181bea04 100644 --- a/node/subsystem-util/src/determine_new_blocks.rs +++ b/node/subsystem-util/src/determine_new_blocks.rs @@ -17,8 +17,9 @@ //! A utility for fetching all unknown blocks based on a new chain-head hash. use polkadot_node_subsystem::{ - messages::ChainApiMessage, SubsystemSender, + messages::ChainApiMessage, }; +use polkadot_overseer_gen::SubsystemSender; use polkadot_primitives::v1::{Hash, Header, BlockNumber}; use futures::prelude::*; use futures::channel::oneshot; @@ -34,13 +35,15 @@ use futures::channel::oneshot; /// then the returned list will be empty. /// /// This may be somewhat expensive when first recovering from major sync. -pub async fn determine_new_blocks( - ctx: &mut impl SubsystemSender, +pub async fn determine_new_blocks( + sender: &mut Sender, is_known: impl Fn(&Hash) -> Result, head: Hash, header: &Header, lower_bound_number: BlockNumber, -) -> Result, E> { +) -> Result, E> where + Sender: SubsystemSender, +{ const ANCESTRY_STEP: usize = 4; let min_block_needed = lower_bound_number + 1; @@ -87,7 +90,7 @@ pub async fn determine_new_blocks( let batch_hashes = if ancestry_step == 1 { vec![last_header.parent_hash] } else { - ctx.send_message(ChainApiMessage::Ancestors { + sender.send_message(ChainApiMessage::Ancestors { hash: *last_hash, k: ancestry_step, response_channel: tx, @@ -106,7 +109,7 @@ pub async fn determine_new_blocks( .unzip::<_, _, Vec<_>, Vec<_>>(); for (hash, sender) in batch_hashes.iter().cloned().zip(batch_senders) { - ctx.send_message(ChainApiMessage::BlockHeader(hash, sender).into()).await; + sender.send_message(ChainApiMessage::BlockHeader(hash, sender).into()).await; } let mut requests = futures::stream::FuturesOrdered::new(); @@ -156,7 +159,7 @@ mod tests { use super::*; use std::collections::{HashSet, HashMap}; use sp_core::testing::TaskExecutor; - use polkadot_node_subsystem::{messages::AllMessages, SubsystemContext}; + use polkadot_overseer::{AllMessages, SubsystemContext}; use polkadot_node_subsystem_test_helpers::make_subsystem_context; use assert_matches::assert_matches; diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 6480c9eb3cd9..1649bd09519c 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -34,15 +34,20 @@ use polkadot_node_subsystem::{ }, }; -pub use polkadot_overseer_gen::{ +pub use polkadot_overseer::{ FromOverseer, - SpawnedSubsystem, Subsystem, SubsystemContext, - OverseerError, SubsystemSender, TimeoutExt, - Timeout, + gen::SpawnedSubsystem, + gen::OverseerError, + gen::Timeout, +}; + +pub use subsystem_node_metrics::{ + Metronome, + metrics, }; use polkadot_node_jaeger as jaeger; @@ -381,41 +386,6 @@ impl JobHandle { } } -/// This module reexports Prometheus types and defines the [`Metrics`] trait. -pub mod metrics { - /// Reexport Substrate Prometheus types. - pub use substrate_prometheus_endpoint as prometheus; - - - /// Subsystem- or job-specific Prometheus metrics. - /// - /// Usually implemented as a wrapper for `Option` - /// to ensure `Default` bounds or as a dummy type (). - /// Prometheus metrics internally hold an `Arc` reference, so cloning them is fine. - pub trait Metrics: Default + Clone { - /// Try to register metrics in the Prometheus registry. - fn try_register(registry: &prometheus::Registry) -> Result; - - /// Convenience method to register metrics in the optional Promethius registry. - /// - /// If no registry is provided, returns `Default::default()`. Otherwise, returns the same - /// thing that `try_register` does. - fn register(registry: Option<&prometheus::Registry>) -> Result { - match registry { - None => Ok(Self::default()), - Some(registry) => Self::try_register(registry), - } - } - } - - // dummy impl - impl Metrics for () { - fn try_register(_registry: &prometheus::Registry) -> Result<(), prometheus::PrometheusError> { - Ok(()) - } - } -} - /// Commands from a job to the broader subsystem. pub enum FromJobCommand { /// Spawn a child task on the executor. @@ -816,59 +786,5 @@ where } } - -#[derive(Copy, Clone)] -enum MetronomeState { - Snooze, - SetAlarm, -} - -/// Create a stream of ticks with a defined cycle duration. -pub struct Metronome { - delay: Delay, - period: Duration, - state: MetronomeState, -} - -impl Metronome -{ - /// Create a new metronome source with a defined cycle duration. - pub fn new(cycle: Duration) -> Self { - let period = cycle.into(); - Self { - period, - delay: Delay::new(period), - state: MetronomeState::Snooze, - } - } -} - -impl futures::Stream for Metronome -{ - type Item = (); - fn poll_next( - mut self: Pin<&mut Self>, - cx: &mut Context<'_> - ) -> Poll> { - loop { - match self.state { - MetronomeState::SetAlarm => { - let val = self.period.clone(); - self.delay.reset(val); - self.state = MetronomeState::Snooze; - } - MetronomeState::Snooze => { - if !Pin::new(&mut self.delay).poll(cx).is_ready() { - break - } - self.state = MetronomeState::SetAlarm; - return Poll::Ready(Some(())); - } - } - } - Poll::Pending - } -} - #[cfg(test)] mod tests; diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index 1b857ee7893c..28484bcb1a79 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -20,8 +20,8 @@ //! care about the state of particular blocks. use polkadot_primitives::v1::{Hash, Header, SessionInfo, SessionIndex}; +use polkadot_overseer::SubsystemContext; use polkadot_node_subsystem::{ - SubsystemContext, messages::{RuntimeApiMessage, RuntimeApiRequest}, errors::RuntimeApiError, }; diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs index 10eb7436716d..ccca60eed373 100644 --- a/node/subsystem-util/src/tests.rs +++ b/node/subsystem-util/src/tests.rs @@ -18,8 +18,9 @@ use super::*; use executor::block_on; use thiserror::Error; use polkadot_node_jaeger as jaeger; +use polkadot_overseer::AllMessages; use polkadot_node_subsystem::{ - messages::{AllMessages, CollatorProtocolMessage}, ActiveLeavesUpdate, FromOverseer, OverseerSignal, + messages::{CollatorProtocolMessage}, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, ActivatedLeaf, LeafStatus, }; use assert_matches::assert_matches; From d3e75968f54366c8c520b2a1a5ba2980113e1cdb Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 29 Jun 2021 17:38:04 +0200 Subject: [PATCH 074/161] split subsystem-util up --- Cargo.lock | 593 ++++++++++++------ Cargo.toml | 1 + node/core/parachains-inherent/src/lib.rs | 9 +- node/jaeger/src/spans.rs | 1 - node/malus/src/variant-a.rs | 4 +- node/overseer/Cargo.toml | 2 +- node/overseer/overseer-gen/Cargo.toml | 2 +- .../proc-macro/src/impl_overseer.rs | 2 + node/overseer/src/lib.rs | 15 +- node/overseer/src/metrics.rs | 2 +- node/overseer/src/subsystems.rs | 16 +- node/overseer/src/tests.rs | 4 - node/service/src/grandpa_support.rs | 6 +- node/service/src/lib.rs | 4 +- node/service/src/overseer.rs | 6 +- node/service/src/relay_chain_selection.rs | 8 +- node/subsystem-types/Cargo.toml | 2 +- node/subsystem-types/src/messages.rs | 6 +- .../src/determine_new_blocks.rs | 5 +- node/subsystem-util/src/lib.rs | 4 +- node/subsystem-util/src/runtime/mod.rs | 2 +- node/subsystem/Cargo.toml | 6 + node/subsystem/src/lib.rs | 131 +--- node/test/service/src/lib.rs | 4 +- 24 files changed, 459 insertions(+), 376 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6871396a086c..6701615b751d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -540,6 +540,22 @@ dependencies = [ "sp-std", ] +[[package]] +name = "behavior-tests" +version = "0.1.0" +dependencies = [ + "env_logger 0.8.4", + "futures 0.3.15", + "log", + "polkadot-service", + "sc-chain-spec", + "sc-service", + "sp-core", + "test-runner", + "thiserror", + "tokio 0.2.21", +] + [[package]] name = "bincode" version = "1.3.1" @@ -723,6 +739,17 @@ dependencies = [ "sp-std", ] +[[package]] +name = "bp-kusama" +version = "0.1.0" +dependencies = [ + "bp-messages", + "bp-polkadot-core", + "bp-runtime", + "sp-api", + "sp-std", +] + [[package]] name = "bp-messages" version = "0.1.0" @@ -737,6 +764,17 @@ dependencies = [ "sp-std", ] +[[package]] +name = "bp-polkadot" +version = "0.1.0" +dependencies = [ + "bp-messages", + "bp-polkadot-core", + "bp-runtime", + "sp-api", + "sp-std", +] + [[package]] name = "bp-polkadot-core" version = "0.1.0" @@ -800,6 +838,21 @@ dependencies = [ "sp-std", ] +[[package]] +name = "bp-westend" +version = "0.1.0" +dependencies = [ + "bp-header-chain", + "bp-messages", + "bp-polkadot-core", + "bp-runtime", + "parity-scale-codec", + "sp-api", + "sp-runtime", + "sp-std", + "sp-version", +] + [[package]] name = "bp-wococo" version = "0.1.0" @@ -1912,7 +1965,7 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "fork-tree" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "parity-scale-codec", ] @@ -1930,7 +1983,7 @@ dependencies = [ [[package]] name = "frame-benchmarking" version = "3.1.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "frame-system", @@ -1949,7 +2002,7 @@ dependencies = [ [[package]] name = "frame-benchmarking-cli" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "Inflector", "chrono", @@ -1972,7 +2025,7 @@ dependencies = [ [[package]] name = "frame-election-provider-support" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "frame-system", @@ -1985,7 +2038,7 @@ dependencies = [ [[package]] name = "frame-executive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "frame-system", @@ -2000,7 +2053,7 @@ dependencies = [ [[package]] name = "frame-metadata" version = "13.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "parity-scale-codec", "serde", @@ -2011,7 +2064,7 @@ dependencies = [ [[package]] name = "frame-support" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "bitflags", "frame-metadata", @@ -2038,7 +2091,7 @@ dependencies = [ [[package]] name = "frame-support-procedural" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "Inflector", "frame-support-procedural-tools", @@ -2050,7 +2103,7 @@ dependencies = [ [[package]] name = "frame-support-procedural-tools" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate 1.0.0", @@ -2062,7 +2115,7 @@ dependencies = [ [[package]] name = "frame-support-procedural-tools-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "proc-macro2", "quote", @@ -2072,7 +2125,7 @@ dependencies = [ [[package]] name = "frame-support-test" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-metadata", "frame-support", @@ -2092,7 +2145,7 @@ dependencies = [ [[package]] name = "frame-system" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "impl-trait-for-tuples", @@ -2109,7 +2162,7 @@ dependencies = [ [[package]] name = "frame-system-benchmarking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -2123,7 +2176,7 @@ dependencies = [ [[package]] name = "frame-system-rpc-runtime-api" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "parity-scale-codec", "sp-api", @@ -2132,7 +2185,7 @@ dependencies = [ [[package]] name = "frame-try-runtime" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "parity-scale-codec", @@ -2521,7 +2574,7 @@ dependencies = [ "log", "slab", "tokio 0.2.21", - "tokio-util", + "tokio-util 0.3.1", ] [[package]] @@ -3232,7 +3285,7 @@ dependencies = [ "thiserror", "tokio 0.2.21", "tokio-rustls 0.15.0", - "tokio-util", + "tokio-util 0.3.1", "url 2.2.0", ] @@ -4064,7 +4117,7 @@ dependencies = [ [[package]] name = "max-encoded-len" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "impl-trait-for-tuples", "max-encoded-len-derive", @@ -4075,7 +4128,7 @@ dependencies = [ [[package]] name = "max-encoded-len-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -4578,7 +4631,7 @@ checksum = "13370dae44474229701bb69b90b4f4dca6404cb0357a2d50d635f1171dc3aa7b" [[package]] name = "pallet-authority-discovery" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "frame-system", @@ -4593,7 +4646,7 @@ dependencies = [ [[package]] name = "pallet-authorship" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "frame-system", @@ -4607,7 +4660,7 @@ dependencies = [ [[package]] name = "pallet-babe" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4630,7 +4683,7 @@ dependencies = [ [[package]] name = "pallet-balances" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4660,7 +4713,7 @@ dependencies = [ [[package]] name = "pallet-bounties" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4696,7 +4749,7 @@ dependencies = [ [[package]] name = "pallet-collective" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4712,7 +4765,7 @@ dependencies = [ [[package]] name = "pallet-democracy" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4727,7 +4780,7 @@ dependencies = [ [[package]] name = "pallet-election-provider-multi-phase" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -4748,7 +4801,7 @@ dependencies = [ [[package]] name = "pallet-elections-phragmen" version = "4.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4765,7 +4818,7 @@ dependencies = [ [[package]] name = "pallet-gilt" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4779,7 +4832,7 @@ dependencies = [ [[package]] name = "pallet-grandpa" version = "3.1.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4801,7 +4854,7 @@ dependencies = [ [[package]] name = "pallet-identity" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "enumflags2", "frame-benchmarking", @@ -4816,7 +4869,7 @@ dependencies = [ [[package]] name = "pallet-im-online" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4835,7 +4888,7 @@ dependencies = [ [[package]] name = "pallet-indices" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4851,7 +4904,7 @@ dependencies = [ [[package]] name = "pallet-membership" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4866,7 +4919,7 @@ dependencies = [ [[package]] name = "pallet-mmr" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "ckb-merkle-mountain-range", "frame-benchmarking", @@ -4883,7 +4936,7 @@ dependencies = [ [[package]] name = "pallet-mmr-primitives" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "frame-system", @@ -4899,7 +4952,7 @@ dependencies = [ [[package]] name = "pallet-mmr-rpc" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -4917,7 +4970,7 @@ dependencies = [ [[package]] name = "pallet-multisig" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4932,7 +4985,7 @@ dependencies = [ [[package]] name = "pallet-nicks" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "frame-system", @@ -4945,7 +4998,7 @@ dependencies = [ [[package]] name = "pallet-offences" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "frame-system", @@ -4961,7 +5014,7 @@ dependencies = [ [[package]] name = "pallet-offences-benchmarking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -4983,7 +5036,7 @@ dependencies = [ [[package]] name = "pallet-proxy" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -4999,7 +5052,7 @@ dependencies = [ [[package]] name = "pallet-recovery" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "enumflags2", "frame-support", @@ -5013,7 +5066,7 @@ dependencies = [ [[package]] name = "pallet-scheduler" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -5028,7 +5081,7 @@ dependencies = [ [[package]] name = "pallet-session" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "frame-system", @@ -5048,7 +5101,7 @@ dependencies = [ [[package]] name = "pallet-session-benchmarking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -5064,7 +5117,7 @@ dependencies = [ [[package]] name = "pallet-society" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "frame-system", @@ -5077,7 +5130,7 @@ dependencies = [ [[package]] name = "pallet-staking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -5101,7 +5154,7 @@ dependencies = [ [[package]] name = "pallet-staking-reward-curve" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -5112,7 +5165,7 @@ dependencies = [ [[package]] name = "pallet-staking-reward-fn" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "log", "sp-arithmetic", @@ -5121,7 +5174,7 @@ dependencies = [ [[package]] name = "pallet-sudo" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "frame-system", @@ -5134,7 +5187,7 @@ dependencies = [ [[package]] name = "pallet-timestamp" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -5152,7 +5205,7 @@ dependencies = [ [[package]] name = "pallet-tips" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -5167,7 +5220,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-support", "frame-system", @@ -5183,7 +5236,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -5200,7 +5253,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc-runtime-api" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "pallet-transaction-payment", "parity-scale-codec", @@ -5211,7 +5264,7 @@ dependencies = [ [[package]] name = "pallet-treasury" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -5227,7 +5280,7 @@ dependencies = [ [[package]] name = "pallet-utility" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-benchmarking", "frame-support", @@ -5242,7 +5295,7 @@ dependencies = [ [[package]] name = "pallet-vesting" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "enumflags2", "frame-benchmarking", @@ -5868,6 +5921,7 @@ dependencies = [ "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", + "polkadot-overseer", "polkadot-primitives", "sp-core", "sp-keyring", @@ -5937,6 +5991,7 @@ dependencies = [ "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", + "polkadot-overseer", "polkadot-primitives", "sc-authority-discovery", "sc-network", @@ -6271,6 +6326,21 @@ dependencies = [ "thiserror", ] +[[package]] +name = "polkadot-node-metrics" +version = "0.1.0" +dependencies = [ + "async-trait", + "futures 0.3.15", + "futures-timer 3.0.2", + "metered-channel", + "sc-network", + "sp-application-crypto", + "sp-core", + "sp-keystore", + "substrate-prometheus-endpoint", +] + [[package]] name = "polkadot-node-network-protocol" version = "0.1.0" @@ -6311,52 +6381,68 @@ dependencies = [ name = "polkadot-node-subsystem" version = "0.1.0" dependencies = [ - "assert_matches", - "async-std", + "futures 0.3.15", + "polkadot-node-jaeger", + "polkadot-node-metrics", + "polkadot-node-primitives", + "polkadot-node-subsystem-types", + "polkadot-overseer", + "polkadot-overseer-gen", + "polkadot-primitives", + "smallvec 1.6.1", + "thiserror", +] + +[[package]] +name = "polkadot-node-subsystem-test-helpers" +version = "0.1.0" +dependencies = [ "async-trait", - "derive_more", "futures 0.3.15", "futures-timer 3.0.2", - "lazy_static", - "log", - "mick-jaeger", "parity-scale-codec", "parking_lot 0.11.1", "pin-project 1.0.7", - "polkadot-node-jaeger", - "polkadot-node-network-protocol", "polkadot-node-primitives", - "polkadot-node-subsystem-test-helpers", + "polkadot-node-subsystem", + "polkadot-node-subsystem-util", + "polkadot-overseer", "polkadot-primitives", - "polkadot-procmacro-subsystem-dispatch-gen", "polkadot-statement-table", "sc-network", "smallvec 1.6.1", "sp-core", - "substrate-prometheus-endpoint", - "thiserror", "tracing", ] [[package]] -name = "polkadot-node-subsystem-test-helpers" +name = "polkadot-node-subsystem-types" version = "0.1.0" dependencies = [ + "assert_matches", + "async-std", "async-trait", + "derive_more", "futures 0.3.15", "futures-timer 3.0.2", + "lazy_static", + "log", + "mick-jaeger", "parity-scale-codec", "parking_lot 0.11.1", "pin-project 1.0.7", + "polkadot-node-jaeger", + "polkadot-node-network-protocol", "polkadot-node-primitives", - "polkadot-node-subsystem", - "polkadot-node-subsystem-util", - "polkadot-overseer", + "polkadot-node-subsystem-test-helpers", + "polkadot-overseer-gen", "polkadot-primitives", "polkadot-statement-table", "sc-network", "smallvec 1.6.1", "sp-core", + "substrate-prometheus-endpoint", + "thiserror", "tracing", ] @@ -6377,10 +6463,12 @@ dependencies = [ "parking_lot 0.11.1", "pin-project 1.0.7", "polkadot-node-jaeger", + "polkadot-node-metrics", "polkadot-node-network-protocol", "polkadot-node-primitives", - "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", + "polkadot-node-subsystem-types", + "polkadot-overseer", "polkadot-primitives", "rand 0.8.4", "sc-network", @@ -6403,18 +6491,50 @@ dependencies = [ "futures-timer 3.0.2", "kv-log-macro", "lru", + "polkadot-node-metrics", "polkadot-node-network-protocol", "polkadot-node-primitives", - "polkadot-node-subsystem", - "polkadot-node-subsystem-util", + "polkadot-node-subsystem-types", + "polkadot-overseer-gen", "polkadot-primitives", - "polkadot-procmacro-overseer-subsystems-gen", "sc-client-api", "sp-api", "sp-core", "tracing", ] +[[package]] +name = "polkadot-overseer-gen" +version = "0.1.0" +dependencies = [ + "async-trait", + "futures 0.3.15", + "futures-timer 3.0.2", + "futures-util", + "metered-channel", + "pin-project 1.0.7", + "polkadot-node-network-protocol", + "polkadot-node-subsystem", + "polkadot-overseer-gen-proc-macro", + "sp-core", + "thiserror", + "tokio-util 0.6.7", + "tracing", + "trybuild", +] + +[[package]] +name = "polkadot-overseer-gen-proc-macro" +version = "0.1.0" +dependencies = [ + "assert_matches", + "proc-macro-crate 1.0.0", + "proc-macro2", + "quote", + "syn", + "trybuild", +] + [[package]] name = "polkadot-parachain" version = "0.9.7" @@ -6460,28 +6580,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "polkadot-procmacro-overseer-subsystems-gen" -version = "0.1.0" -dependencies = [ - "assert_matches", - "proc-macro2", - "quote", - "syn", - "trybuild", -] - -[[package]] -name = "polkadot-procmacro-subsystem-dispatch-gen" -version = "0.1.0" -dependencies = [ - "assert_matches", - "proc-macro2", - "quote", - "syn", - "trybuild", -] - [[package]] name = "polkadot-rpc" version = "0.9.7" @@ -7143,9 +7241,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" [[package]] name = "proc-macro2" -version = "1.0.24" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" dependencies = [ "unicode-xid", ] @@ -7575,7 +7673,7 @@ dependencies = [ [[package]] name = "remote-externalities" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "env_logger 0.8.4", "hex", @@ -7869,7 +7967,7 @@ dependencies = [ [[package]] name = "sc-authority-discovery" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "derive_more", @@ -7898,7 +7996,7 @@ dependencies = [ [[package]] name = "sc-basic-authorship" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "futures 0.3.15", "futures-timer 3.0.2", @@ -7921,7 +8019,7 @@ dependencies = [ [[package]] name = "sc-block-builder" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "parity-scale-codec", "sc-client-api", @@ -7937,7 +8035,7 @@ dependencies = [ [[package]] name = "sc-chain-spec" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -7958,7 +8056,7 @@ dependencies = [ [[package]] name = "sc-chain-spec-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -7969,7 +8067,7 @@ dependencies = [ [[package]] name = "sc-cli" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "chrono", "fdlimit", @@ -8007,7 +8105,7 @@ dependencies = [ [[package]] name = "sc-client-api" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "derive_more", "fnv", @@ -8041,7 +8139,7 @@ dependencies = [ [[package]] name = "sc-client-db" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "blake2-rfc", "hash-db", @@ -8071,7 +8169,7 @@ dependencies = [ [[package]] name = "sc-consensus" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "parking_lot 0.11.1", @@ -8084,7 +8182,7 @@ dependencies = [ [[package]] name = "sc-consensus-babe" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "derive_more", @@ -8130,7 +8228,7 @@ dependencies = [ [[package]] name = "sc-consensus-babe-rpc" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "derive_more", "futures 0.3.15", @@ -8154,7 +8252,7 @@ dependencies = [ [[package]] name = "sc-consensus-epochs" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "fork-tree", "parity-scale-codec", @@ -8164,10 +8262,45 @@ dependencies = [ "sp-runtime", ] +[[package]] +name = "sc-consensus-manual-seal" +version = "0.9.0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" +dependencies = [ + "assert_matches", + "async-trait", + "derive_more", + "futures 0.3.15", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "log", + "parity-scale-codec", + "parking_lot 0.11.1", + "sc-client-api", + "sc-consensus-babe", + "sc-consensus-epochs", + "sc-transaction-pool", + "serde", + "sp-api", + "sp-blockchain", + "sp-consensus", + "sp-consensus-babe", + "sp-consensus-slots", + "sp-core", + "sp-inherents", + "sp-keyring", + "sp-keystore", + "sp-runtime", + "sp-timestamp", + "sp-transaction-pool", + "substrate-prometheus-endpoint", +] + [[package]] name = "sc-consensus-slots" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "futures 0.3.15", @@ -8195,7 +8328,7 @@ dependencies = [ [[package]] name = "sc-consensus-uncles" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "sc-client-api", "sp-authorship", @@ -8206,7 +8339,7 @@ dependencies = [ [[package]] name = "sc-executor" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "derive_more", "lazy_static", @@ -8235,7 +8368,7 @@ dependencies = [ [[package]] name = "sc-executor-common" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "derive_more", "parity-scale-codec", @@ -8252,7 +8385,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmi" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "log", "parity-scale-codec", @@ -8267,7 +8400,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmtime" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "cfg-if 1.0.0", "libc", @@ -8286,7 +8419,7 @@ dependencies = [ [[package]] name = "sc-finality-grandpa" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "derive_more", @@ -8327,7 +8460,7 @@ dependencies = [ [[package]] name = "sc-finality-grandpa-rpc" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "derive_more", "finality-grandpa", @@ -8351,7 +8484,7 @@ dependencies = [ [[package]] name = "sc-finality-grandpa-warp-sync" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "derive_more", "futures 0.3.15", @@ -8372,7 +8505,7 @@ dependencies = [ [[package]] name = "sc-informant" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "ansi_term 0.12.1", "futures 0.3.15", @@ -8390,7 +8523,7 @@ dependencies = [ [[package]] name = "sc-keystore" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "derive_more", @@ -8410,7 +8543,7 @@ dependencies = [ [[package]] name = "sc-light" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "hash-db", "lazy_static", @@ -8429,7 +8562,7 @@ dependencies = [ [[package]] name = "sc-network" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-std", "async-trait", @@ -8482,7 +8615,7 @@ dependencies = [ [[package]] name = "sc-network-gossip" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "futures 0.3.15", "futures-timer 3.0.2", @@ -8499,7 +8632,7 @@ dependencies = [ [[package]] name = "sc-offchain" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "bytes 0.5.6", "fnv", @@ -8527,7 +8660,7 @@ dependencies = [ [[package]] name = "sc-peerset" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "futures 0.3.15", "libp2p", @@ -8540,7 +8673,7 @@ dependencies = [ [[package]] name = "sc-proposer-metrics" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "log", "substrate-prometheus-endpoint", @@ -8549,7 +8682,7 @@ dependencies = [ [[package]] name = "sc-rpc" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "futures 0.3.15", "hash-db", @@ -8584,7 +8717,7 @@ dependencies = [ [[package]] name = "sc-rpc-api" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "derive_more", "futures 0.3.15", @@ -8609,7 +8742,7 @@ dependencies = [ [[package]] name = "sc-rpc-server" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "futures 0.1.29", "jsonrpc-core", @@ -8627,7 +8760,7 @@ dependencies = [ [[package]] name = "sc-service" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "directories", @@ -8693,7 +8826,7 @@ dependencies = [ [[package]] name = "sc-state-db" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "log", "parity-scale-codec", @@ -8708,7 +8841,7 @@ dependencies = [ [[package]] name = "sc-sync-state-rpc" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "jsonrpc-core", "jsonrpc-core-client", @@ -8728,7 +8861,7 @@ dependencies = [ [[package]] name = "sc-telemetry" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "chrono", "futures 0.3.15", @@ -8748,7 +8881,7 @@ dependencies = [ [[package]] name = "sc-tracing" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "ansi_term 0.12.1", "atty", @@ -8785,7 +8918,7 @@ dependencies = [ [[package]] name = "sc-tracing-proc-macro" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -8796,7 +8929,7 @@ dependencies = [ [[package]] name = "sc-transaction-graph" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "derive_more", "futures 0.3.15", @@ -8818,7 +8951,7 @@ dependencies = [ [[package]] name = "sc-transaction-pool" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "futures 0.3.15", "intervalier", @@ -9297,7 +9430,7 @@ dependencies = [ [[package]] name = "sp-allocator" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "log", "sp-core", @@ -9309,7 +9442,7 @@ dependencies = [ [[package]] name = "sp-api" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "hash-db", "log", @@ -9326,7 +9459,7 @@ dependencies = [ [[package]] name = "sp-api-proc-macro" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "blake2-rfc", "proc-macro-crate 1.0.0", @@ -9338,7 +9471,7 @@ dependencies = [ [[package]] name = "sp-application-crypto" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "max-encoded-len", "parity-scale-codec", @@ -9351,7 +9484,7 @@ dependencies = [ [[package]] name = "sp-arithmetic" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "integer-sqrt", "num-traits", @@ -9365,7 +9498,7 @@ dependencies = [ [[package]] name = "sp-authority-discovery" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "parity-scale-codec", "sp-api", @@ -9377,7 +9510,7 @@ dependencies = [ [[package]] name = "sp-authorship" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "parity-scale-codec", @@ -9389,7 +9522,7 @@ dependencies = [ [[package]] name = "sp-block-builder" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "parity-scale-codec", "sp-api", @@ -9401,7 +9534,7 @@ dependencies = [ [[package]] name = "sp-blockchain" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "futures 0.3.15", "log", @@ -9419,7 +9552,7 @@ dependencies = [ [[package]] name = "sp-chain-spec" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "serde", "serde_json", @@ -9428,7 +9561,7 @@ dependencies = [ [[package]] name = "sp-consensus" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "futures 0.3.15", @@ -9455,7 +9588,7 @@ dependencies = [ [[package]] name = "sp-consensus-babe" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "merlin", @@ -9477,7 +9610,7 @@ dependencies = [ [[package]] name = "sp-consensus-slots" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "parity-scale-codec", "sp-arithmetic", @@ -9487,7 +9620,7 @@ dependencies = [ [[package]] name = "sp-consensus-vrf" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "parity-scale-codec", "schnorrkel", @@ -9499,7 +9632,7 @@ dependencies = [ [[package]] name = "sp-core" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "base58", "blake2-rfc", @@ -9544,7 +9677,7 @@ dependencies = [ [[package]] name = "sp-database" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "kvdb", "parking_lot 0.11.1", @@ -9553,7 +9686,7 @@ dependencies = [ [[package]] name = "sp-debug-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "proc-macro2", "quote", @@ -9563,7 +9696,7 @@ dependencies = [ [[package]] name = "sp-externalities" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "environmental", "parity-scale-codec", @@ -9574,7 +9707,7 @@ dependencies = [ [[package]] name = "sp-finality-grandpa" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "finality-grandpa", "log", @@ -9591,7 +9724,7 @@ dependencies = [ [[package]] name = "sp-inherents" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "impl-trait-for-tuples", @@ -9605,7 +9738,7 @@ dependencies = [ [[package]] name = "sp-io" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "futures 0.3.15", "hash-db", @@ -9630,7 +9763,7 @@ dependencies = [ [[package]] name = "sp-keyring" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "lazy_static", "sp-core", @@ -9641,7 +9774,7 @@ dependencies = [ [[package]] name = "sp-keystore" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "derive_more", @@ -9658,7 +9791,7 @@ dependencies = [ [[package]] name = "sp-maybe-compressed-blob" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "ruzstd", "zstd", @@ -9667,7 +9800,7 @@ dependencies = [ [[package]] name = "sp-npos-elections" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "parity-scale-codec", "serde", @@ -9680,7 +9813,7 @@ dependencies = [ [[package]] name = "sp-npos-elections-compact" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "proc-macro-crate 1.0.0", "proc-macro2", @@ -9691,7 +9824,7 @@ dependencies = [ [[package]] name = "sp-offchain" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "sp-api", "sp-core", @@ -9701,7 +9834,7 @@ dependencies = [ [[package]] name = "sp-panic-handler" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "backtrace", ] @@ -9709,7 +9842,7 @@ dependencies = [ [[package]] name = "sp-rpc" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "rustc-hash", "serde", @@ -9720,7 +9853,7 @@ dependencies = [ [[package]] name = "sp-runtime" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "either", "hash256-std-hasher", @@ -9742,7 +9875,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -9759,7 +9892,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "Inflector", "proc-macro-crate 1.0.0", @@ -9771,7 +9904,7 @@ dependencies = [ [[package]] name = "sp-serializer" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "serde", "serde_json", @@ -9780,7 +9913,7 @@ dependencies = [ [[package]] name = "sp-session" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "parity-scale-codec", "sp-api", @@ -9793,7 +9926,7 @@ dependencies = [ [[package]] name = "sp-staking" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "parity-scale-codec", "sp-runtime", @@ -9803,7 +9936,7 @@ dependencies = [ [[package]] name = "sp-state-machine" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "hash-db", "log", @@ -9826,12 +9959,12 @@ dependencies = [ [[package]] name = "sp-std" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" [[package]] name = "sp-storage" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "impl-serde", "parity-scale-codec", @@ -9844,7 +9977,7 @@ dependencies = [ [[package]] name = "sp-tasks" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "log", "sp-core", @@ -9857,7 +9990,7 @@ dependencies = [ [[package]] name = "sp-timestamp" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "futures-timer 3.0.2", @@ -9874,7 +10007,7 @@ dependencies = [ [[package]] name = "sp-tracing" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "erased-serde", "log", @@ -9892,7 +10025,7 @@ dependencies = [ [[package]] name = "sp-transaction-pool" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "derive_more", "futures 0.3.15", @@ -9908,7 +10041,7 @@ dependencies = [ [[package]] name = "sp-transaction-storage-proof" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "log", @@ -9923,7 +10056,7 @@ dependencies = [ [[package]] name = "sp-trie" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "hash-db", "memory-db", @@ -9937,7 +10070,7 @@ dependencies = [ [[package]] name = "sp-utils" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "futures 0.3.15", "futures-core", @@ -9949,7 +10082,7 @@ dependencies = [ [[package]] name = "sp-version" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "impl-serde", "parity-scale-codec", @@ -9962,7 +10095,7 @@ dependencies = [ [[package]] name = "sp-version-proc-macro" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "parity-scale-codec", "proc-macro-crate 1.0.0", @@ -9974,7 +10107,7 @@ dependencies = [ [[package]] name = "sp-wasm-interface" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "impl-trait-for-tuples", "parity-scale-codec", @@ -10119,7 +10252,7 @@ dependencies = [ [[package]] name = "substrate-browser-utils" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "chrono", "console_error_panic_hook", @@ -10145,7 +10278,7 @@ dependencies = [ [[package]] name = "substrate-build-script-utils" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "platforms", ] @@ -10153,7 +10286,7 @@ dependencies = [ [[package]] name = "substrate-frame-rpc-system" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-system-rpc-runtime-api", "futures 0.3.15", @@ -10176,7 +10309,7 @@ dependencies = [ [[package]] name = "substrate-prometheus-endpoint" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-std", "derive_more", @@ -10190,7 +10323,7 @@ dependencies = [ [[package]] name = "substrate-test-client" version = "2.0.1" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "async-trait", "futures 0.1.29", @@ -10219,7 +10352,7 @@ dependencies = [ [[package]] name = "substrate-test-utils" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "futures 0.3.15", "substrate-test-utils-derive", @@ -10229,7 +10362,7 @@ dependencies = [ [[package]] name = "substrate-test-utils-derive" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "proc-macro-crate 1.0.0", "quote", @@ -10239,7 +10372,7 @@ dependencies = [ [[package]] name = "substrate-wasm-builder" version = "4.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "ansi_term 0.12.1", "atty", @@ -10387,6 +10520,48 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "test-runner" +version = "0.9.0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" +dependencies = [ + "env_logger 0.7.1", + "frame-system", + "futures 0.3.15", + "jsonrpc-core", + "log", + "sc-basic-authorship", + "sc-cli", + "sc-client-api", + "sc-consensus-manual-seal", + "sc-executor", + "sc-informant", + "sc-network", + "sc-rpc", + "sc-rpc-server", + "sc-service", + "sc-transaction-graph", + "sc-transaction-pool", + "sp-api", + "sp-block-builder", + "sp-blockchain", + "sp-consensus", + "sp-core", + "sp-externalities", + "sp-inherents", + "sp-io", + "sp-keyring", + "sp-keystore", + "sp-offchain", + "sp-runtime", + "sp-runtime-interface", + "sp-session", + "sp-state-machine", + "sp-transaction-pool", + "sp-wasm-interface", + "tokio 0.2.21", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -10543,6 +10718,7 @@ dependencies = [ "libc", "memchr", "mio", + "mio-named-pipes", "mio-uds", "num_cpus", "pin-project-lite 0.1.7", @@ -10803,6 +10979,21 @@ dependencies = [ "tokio 0.2.21", ] +[[package]] +name = "tokio-util" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" +dependencies = [ + "bytes 1.0.1", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.2.4", + "slab", + "tokio 1.6.1", +] + [[package]] name = "toml" version = "0.5.6" @@ -10984,7 +11175,7 @@ checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" [[package]] name = "try-runtime-cli" version = "0.9.0" -source = "git+https://github.com/paritytech/substrate?branch=master#c6240ce90083f9e8db94b6c33f81297c11534ea0" +source = "git+https://github.com/paritytech/substrate?branch=master#3f7d2b7658cb87de61b75e3a782d17abd8a915d1" dependencies = [ "frame-try-runtime", "log", diff --git a/Cargo.toml b/Cargo.toml index 0e7789a9ad31..720b441924c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,6 +81,7 @@ members = [ "node/subsystem-test-helpers", "node/subsystem-util", "node/jaeger", + "node/metrics", "node/metered-channel", "node/test/client", "node/test/service", diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs index e8ffb573658c..83e1b7718ac6 100644 --- a/node/core/parachains-inherent/src/lib.rs +++ b/node/core/parachains-inherent/src/lib.rs @@ -25,10 +25,13 @@ #![deny(unused_crate_dependencies, unused_results)] use futures::{select, FutureExt}; +use polkadot_overseer::{ + AllMessages, + Handler, +}; use polkadot_node_subsystem::{ - messages::{AllMessages, ProvisionerMessage}, SubsystemError, + messages::ProvisionerMessage, SubsystemError, }; -use polkadot_overseer::OverseerHandler; use polkadot_primitives::v1::{ Block, Hash, InherentData as ParachainsInherentData, }; @@ -48,7 +51,7 @@ impl ParachainsInherentDataProvider { /// Create a new instance of the [`ParachainsInherentDataProvider`]. pub async fn create>( client: &C, - mut overseer: OverseerHandler, + mut overseer: Handler, parent: Hash, ) -> Result { let pid = async { diff --git a/node/jaeger/src/spans.rs b/node/jaeger/src/spans.rs index acbb8541f291..4f3114d39b59 100644 --- a/node/jaeger/src/spans.rs +++ b/node/jaeger/src/spans.rs @@ -143,7 +143,6 @@ impl std::ops::Deref for PerLeafSpan { #[repr(u8)] #[non_exhaustive] pub enum Stage { - CandidateSelection = 1, CandidateBacking = 2, StatementDistribution = 3, PoVDistribution = 4, diff --git a/node/malus/src/variant-a.rs b/node/malus/src/variant-a.rs index 1e9cb7928cb2..7406a3cb509b 100644 --- a/node/malus/src/variant-a.rs +++ b/node/malus/src/variant-a.rs @@ -27,7 +27,7 @@ use polkadot_cli::{ create_default_subsystems, service::{ AuthorityDiscoveryApi, AuxStore, BabeApi, Block, Error, HeaderBackend, Overseer, - OverseerGen, OverseerGenArgs, OverseerHandler, ParachainHost, ProvideRuntimeApi, + OverseerGen, OverseerGenArgs, Handler, ParachainHost, ProvideRuntimeApi, SpawnNamed, }, Cli, @@ -73,7 +73,7 @@ impl OverseerGen for BehaveMaleficient { fn generate<'a, Spawner, RuntimeClient>( &self, args: OverseerGenArgs<'a, Spawner, RuntimeClient>, - ) -> Result<(Overseer>, OverseerHandler), Error> + ) -> Result<(Overseer>, Handler), Error> where RuntimeClient: 'static + ProvideRuntimeApi + HeaderBackend + AuxStore, RuntimeClient::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml index c4e8da1fb4a4..d563200b7d9f 100644 --- a/node/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -13,7 +13,7 @@ futures-timer = "3.0.2" polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-node-primitives = { package = "polkadot-node-primitives", path = "../primitives" } polkadot-node-subsystem = { package = "polkadot-node-subsystem-types", path = "../subsystem-types" } -polkadot-node-subsystem-util = { path = "../subsystem-util" } +polkadot-node-metrics = { path = "../metrics" } polkadot-primitives = { path = "../../primitives" } polkadot-overseer-gen = { package = "polkadot-overseer-gen", path = "./overseer-gen" } tracing = "0.1.26" diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index 82339ceb63a3..23dc35c84684 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -23,4 +23,4 @@ futures-util = "0.3" [dev-dependencies] trybuild = "1.0.41" -polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } +polkadot-node-subsystem = { path = "../../subsystem" } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index ec0e56bd84fe..b6943ff2438f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -161,6 +161,8 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { #[subsystem(no_dispatch, CandidateBackingMessage)] candidate_backing: CandidateBacking, - #[subsystem(no_dispatch, CandidateSelectionMessage)] - candidate_selection: CandidateSelection, - #[subsystem(StatementDistributionMessage)] statement_distribution: StatementDistribution, @@ -501,7 +504,6 @@ where where CV: Subsystem, SubsystemError> + Send, CB: Subsystem, SubsystemError> + Send, - CS: Subsystem, SubsystemError> + Send, SD: Subsystem, SubsystemError> + Send, AD: Subsystem, SubsystemError> + Send, AR: Subsystem, SubsystemError> + Send, @@ -524,7 +526,6 @@ where let (mut overseer, handler) = Self::builder() .candidate_validation(all_subsystems.candidate_validation) .candidate_backing(all_subsystems.candidate_backing) - .candidate_selection(all_subsystems.candidate_selection) .statement_distribution(all_subsystems.statement_distribution) .availability_distribution(all_subsystems.availability_distribution) .availability_recovery(all_subsystems.availability_recovery) diff --git a/node/overseer/src/metrics.rs b/node/overseer/src/metrics.rs index 750d9d7e9800..3563f23fbd4b 100644 --- a/node/overseer/src/metrics.rs +++ b/node/overseer/src/metrics.rs @@ -17,7 +17,7 @@ //! Prometheus metrics related to the overseer and its channels. use super::*; -use polkadot_node_subsystem_util::metrics::{self, prometheus}; +use polkadot_node_metrics::metrics::{self, prometheus}; /// Overseer Prometheus metrics. #[derive(Clone)] diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index bced4a4837b0..b3808f7f021f 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -55,7 +55,7 @@ where /// subsystems are implemented and the rest can be mocked with the [`DummySubsystem`]. #[derive(Debug, Clone)] pub struct AllSubsystems< - CV = (), CB = (), CS = (), SD = (), AD = (), AR = (), BS = (), BD = (), P = (), + CV = (), CB = (), SD = (), AD = (), AR = (), BS = (), BD = (), P = (), RA = (), AS = (), NB = (), CA = (), CG = (), CP = (), ApD = (), ApV = (), GS = (), > { @@ -63,8 +63,6 @@ pub struct AllSubsystems< pub candidate_validation: CV, /// A candidate backing subsystem. pub candidate_backing: CB, - /// A candidate selection subsystem. - pub candidate_selection: CS, /// A statement distribution subsystem. pub statement_distribution: SD, /// An availability distribution subsystem. @@ -97,8 +95,8 @@ pub struct AllSubsystems< pub gossip_support: GS, } -impl - AllSubsystems +impl + AllSubsystems { /// Create a new instance of [`AllSubsystems`]. /// @@ -130,12 +128,10 @@ impl DummySubsystem, DummySubsystem, DummySubsystem, - DummySubsystem, > { AllSubsystems { candidate_validation: DummySubsystem, candidate_backing: DummySubsystem, - candidate_selection: DummySubsystem, statement_distribution: DummySubsystem, availability_distribution: DummySubsystem, availability_recovery: DummySubsystem, @@ -154,11 +150,10 @@ impl } } - pub fn as_ref(&self) -> AllSubsystems<&'_ CV, &'_ CB, &'_ CS, &'_ SD, &'_ AD, &'_ AR, &'_ BS, &'_ BD, &'_ P, &'_ RA, &'_ AS, &'_ NB, &'_ CA, &'_ CG, &'_ CP, &'_ ApD, &'_ ApV, &'_ GS> { + pub fn as_ref(&self) -> AllSubsystems<&'_ CV, &'_ CB, &'_ SD, &'_ AD, &'_ AR, &'_ BS, &'_ BD, &'_ P, &'_ RA, &'_ AS, &'_ NB, &'_ CA, &'_ CG, &'_ CP, &'_ ApD, &'_ ApV, &'_ GS> { AllSubsystems { candidate_validation: &self.candidate_validation, candidate_backing: &self.candidate_backing, - candidate_selection: &self.candidate_selection, statement_distribution: &self.statement_distribution, availability_distribution: &self.availability_distribution, availability_recovery: &self.availability_recovery, @@ -181,7 +176,6 @@ impl -> AllSubsystems< >::Output, >::Output, - >::Output, >::Output, >::Output, >::Output, @@ -201,7 +195,6 @@ impl where Mapper: MapSubsystem, Mapper: MapSubsystem, - Mapper: MapSubsystem, Mapper: MapSubsystem, Mapper: MapSubsystem, Mapper: MapSubsystem, @@ -221,7 +214,6 @@ impl AllSubsystems { candidate_validation: >::map_subsystem(&mapper, self.candidate_validation), candidate_backing: >::map_subsystem(&mapper, self.candidate_backing), - candidate_selection: >::map_subsystem(&mapper, self.candidate_selection), statement_distribution: >::map_subsystem(&mapper, self.statement_distribution), availability_distribution: >::map_subsystem(&mapper, self.availability_distribution), availability_recovery: >::map_subsystem(&mapper, self.availability_recovery), diff --git a/node/overseer/src/tests.rs b/node/overseer/src/tests.rs index 342ff69b4c6e..c33455ccb9f0 100644 --- a/node/overseer/src/tests.rs +++ b/node/overseer/src/tests.rs @@ -702,10 +702,6 @@ fn test_candidate_backing_msg() -> CandidateBackingMessage { CandidateBackingMessage::GetBackedCandidates(Default::default(), Vec::new(), sender) } -fn test_candidate_selection_msg() -> CandidateSelectionMessage { - CandidateSelectionMessage::default() -} - fn test_chain_api_msg() -> ChainApiMessage { let (sender, _) = oneshot::channel(); ChainApiMessage::FinalizedBlockNumber(sender) diff --git a/node/service/src/grandpa_support.rs b/node/service/src/grandpa_support.rs index bb45709ebbc7..5f4d6fe1cfa9 100644 --- a/node/service/src/grandpa_support.rs +++ b/node/service/src/grandpa_support.rs @@ -27,7 +27,7 @@ use { polkadot_primitives::v1::{Hash, Block as PolkadotBlock, Header as PolkadotHeader}, polkadot_subsystem::messages::ApprovalVotingMessage, prometheus_endpoint::{self, Registry}, - polkadot_overseer::OverseerHandler, + polkadot_overseer::Handler, futures::channel::oneshot, }; @@ -41,13 +41,13 @@ use { #[derive(Clone)] pub(crate) struct ApprovalCheckingVotingRule { checking_lag: Option>, - overseer: OverseerHandler, + overseer: Handler, } #[cfg(feature = "full-node")] impl ApprovalCheckingVotingRule { /// Create a new approval checking diagnostic voting rule. - pub fn new(overseer: OverseerHandler, registry: Option<&Registry>) + pub fn new(overseer: Handler, registry: Option<&Registry>) -> Result { Ok(ApprovalCheckingVotingRule { diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 0484c2b9a0f5..6d5a1f95f6bb 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -56,7 +56,7 @@ pub use { sp_authority_discovery::AuthorityDiscoveryApi, sc_client_api::AuxStore, polkadot_primitives::v1::ParachainHost, - polkadot_overseer::{Overseer, OverseerHandler}, + polkadot_overseer::{Overseer, Handler}, }; pub use sp_core::traits::SpawnNamed; @@ -427,7 +427,7 @@ fn new_partial( pub struct NewFull { pub task_manager: TaskManager, pub client: C, - pub overseer_handler: Option, + pub overseer_handler: Option, pub network: Arc::Hash>>, pub rpc_handlers: RpcHandlers, pub backend: Arc, diff --git a/node/service/src/overseer.rs b/node/service/src/overseer.rs index 968240074b8d..be9c9d5dab0e 100644 --- a/node/service/src/overseer.rs +++ b/node/service/src/overseer.rs @@ -28,7 +28,7 @@ use polkadot_network_bridge::RequestMultiplexer; use polkadot_node_core_av_store::Config as AvailabilityConfig; use polkadot_node_core_approval_voting::Config as ApprovalVotingConfig; use polkadot_node_core_candidate_validation::Config as CandidateValidationConfig; -use polkadot_overseer::{AllSubsystems, BlockInfo, Overseer, OverseerHandler}; +use polkadot_overseer::{AllSubsystems, BlockInfo, Overseer, Handler}; use polkadot_primitives::v1::ParachainHost; use sc_authority_discovery::Service as AuthorityDiscoveryService; use sp_api::ProvideRuntimeApi; @@ -237,7 +237,7 @@ where /// would do. pub trait OverseerGen { /// Overwrite the full generation of the overseer, including the subsystems. - fn generate<'a, Spawner, RuntimeClient>(&self, args: OverseerGenArgs<'a, Spawner, RuntimeClient>) -> Result<(Overseer>, OverseerHandler), Error> + fn generate<'a, Spawner, RuntimeClient>(&self, args: OverseerGenArgs<'a, Spawner, RuntimeClient>) -> Result<(Overseer>, Handler), Error> where RuntimeClient: 'static + ProvideRuntimeApi + HeaderBackend + AuxStore, RuntimeClient::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, @@ -256,7 +256,7 @@ pub struct RealOverseerGen; impl OverseerGen for RealOverseerGen { fn generate<'a, Spawner, RuntimeClient>(&self, args : OverseerGenArgs<'a, Spawner, RuntimeClient> - ) -> Result<(Overseer>, OverseerHandler), Error> + ) -> Result<(Overseer>, Handler), Error> where RuntimeClient: 'static + ProvideRuntimeApi + HeaderBackend + AuxStore, RuntimeClient::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, diff --git a/node/service/src/relay_chain_selection.rs b/node/service/src/relay_chain_selection.rs index 1ca1c5c5f29c..98e7186b352a 100644 --- a/node/service/src/relay_chain_selection.rs +++ b/node/service/src/relay_chain_selection.rs @@ -41,7 +41,7 @@ use { }, polkadot_subsystem::messages::{ApprovalVotingMessage, ChainSelectionMessage}, polkadot_node_subsystem_util::metrics::{self, prometheus}, - polkadot_overseer::OverseerHandler, + polkadot_overseer::Handler, futures::channel::oneshot, consensus_common::{Error as ConsensusError, SelectChain}, sp_blockchain::HeaderBackend, @@ -111,7 +111,7 @@ impl Metrics { /// A chain-selection implementation which provides safety for relay chains. pub struct SelectRelayChain { backend: Arc, - overseer: OverseerHandler, + overseer: Handler, // A fallback to use in case the overseer is disconnected. // // This is used on relay chains which have not yet enabled @@ -126,7 +126,7 @@ impl SelectRelayChain /// Create a new [`SelectRelayChain`] wrapping the given chain backend /// and a handle to the overseer. #[allow(unused)] - pub fn new(backend: Arc, overseer: OverseerHandler, metrics: Metrics) -> Self { + pub fn new(backend: Arc, overseer: Handler, metrics: Metrics) -> Self { SelectRelayChain { fallback: sc_consensus::LongestChain::new(backend.clone()), backend, @@ -172,7 +172,7 @@ impl SelectRelayChain { #[allow(unused)] pub fn connect_overseer_handler( &mut self, - other_handler: &OverseerHandler, + other_handler: &Handler, ) { other_handler.connect_other(&mut self.overseer); } diff --git a/node/subsystem-types/Cargo.toml b/node/subsystem-types/Cargo.toml index 339a42f38a3a..538a8ebab956 100644 --- a/node/subsystem-types/Cargo.toml +++ b/node/subsystem-types/Cargo.toml @@ -17,9 +17,9 @@ tracing = "0.1.26" parity-scale-codec = { version = "2.0.0", default-features = false, features = ["derive"] } parking_lot = "0.11.1" pin-project = "1.0.4" +polkadot-primitives = { path = "../../primitives" } polkadot-node-primitives = { path = "../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } -polkadot-primitives = { path = "../../primitives" } polkadot-statement-table = { path = "../../statement-table" } polkadot-node-jaeger = { path = "../jaeger" } polkadot-overseer-gen = { path = "../overseer/overseer-gen" } diff --git a/node/subsystem-types/src/messages.rs b/node/subsystem-types/src/messages.rs index bb9204fd1c97..22c43eb68cf5 100644 --- a/node/subsystem-types/src/messages.rs +++ b/node/subsystem-types/src/messages.rs @@ -22,7 +22,6 @@ //! //! Subsystems' APIs are defined separately from their implementation, leading to easier mocking. -use std::{collections::{BTreeMap, HashSet}, sync::Arc}; use futures::channel::{mpsc, oneshot}; use thiserror::Error; @@ -49,7 +48,10 @@ use polkadot_primitives::v1::{ ValidatorSignature, }; use polkadot_statement_table::v1::Misbehavior; -use std::{sync::Arc, collections::btree_map::BTreeMap}; +use std::{ + collections::{BTreeMap, HashSet}, + sync::Arc, +}; /// Network events as transmitted to other subsystems, wrapped in their message types. pub mod network_bridge_event; diff --git a/node/subsystem-util/src/determine_new_blocks.rs b/node/subsystem-util/src/determine_new_blocks.rs index 1566181bea04..d90cc765f120 100644 --- a/node/subsystem-util/src/determine_new_blocks.rs +++ b/node/subsystem-util/src/determine_new_blocks.rs @@ -19,7 +19,10 @@ use polkadot_node_subsystem::{ messages::ChainApiMessage, }; -use polkadot_overseer_gen::SubsystemSender; +use polkadot_overseer::{ + AllMessages, + gen::SubsystemSender, +}; use polkadot_primitives::v1::{Hash, Header, BlockNumber}; use futures::prelude::*; use futures::channel::oneshot; diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 1649bd09519c..866be758a7a7 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -45,7 +45,7 @@ pub use polkadot_overseer::{ gen::Timeout, }; -pub use subsystem_node_metrics::{ +pub use polkadot_node_metrics::{ Metronome, metrics, }; @@ -79,7 +79,7 @@ pub use error_handling::{Fault, unwrap_non_fatal}; /// These reexports are required so that external crates can use the `delegated_subsystem` macro properly. pub mod reexports { - pub use polkadot_overseer_gen::{ + pub use polkadot_overseer::gen::{ SpawnNamed, SpawnedSubsystem, Subsystem, diff --git a/node/subsystem-util/src/runtime/mod.rs b/node/subsystem-util/src/runtime/mod.rs index c2fb645d3ba2..77d0a07c4319 100644 --- a/node/subsystem-util/src/runtime/mod.rs +++ b/node/subsystem-util/src/runtime/mod.rs @@ -24,7 +24,7 @@ use sp_core::crypto::Public; use sp_keystore::{CryptoStore, SyncCryptoStorePtr}; use polkadot_primitives::v1::{CoreState, EncodeAs, GroupIndex, GroupRotationInfo, Hash, OccupiedCore, SessionIndex, SessionInfo, Signed, SigningContext, UncheckedSigned, ValidatorId, ValidatorIndex}; -use polkadot_overseer_gen::SubsystemContext; +use polkadot_overseer::SubsystemContext; use polkadot_node_subsystem::messages::RuntimeApiMessage; use polkadot_node_subsystem::OverseerSignal; diff --git a/node/subsystem/Cargo.toml b/node/subsystem/Cargo.toml index 61daa8a74d40..d83e41994a19 100644 --- a/node/subsystem/Cargo.toml +++ b/node/subsystem/Cargo.toml @@ -10,3 +10,9 @@ polkadot-overseer = { path = "../overseer" } polkadot-overseer-gen = { path = "../overseer/overseer-gen" } polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-jaeger = { path = "../jaeger" } +thiserror = "1.0.23" +smallvec = "1.6.1" +futures = "0.3" +polkadot-primitives = { path = "../../primitives" } +polkadot-node-primitives = { path = "../primitives" } +polkadot-node-metrics = { path = "../metrics" } diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 399b939085d3..6c13f2dcf205 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -26,8 +26,14 @@ pub use polkadot_node_subsystem_types::{errors, messages}; pub use polkadot_node_jaeger as jaeger; pub use jaeger::*; -use self::messages::AllMessages; - +use std::sync::Arc; +use std::fmt; +use smallvec::SmallVec; +use futures::prelude::*; +use futures::channel::{oneshot, mpsc}; +use futures::future::BoxFuture; +use polkadot_overseer::{AllMessages, OverseerSignal}; +use polkadot_primitives::v1::{Hash, BlockNumber}; /// How many slots are stack-reserved for active leaves updates /// /// If there are fewer than this number of slots, then we've wasted some stack space. @@ -135,40 +141,6 @@ impl fmt::Debug for ActiveLeavesUpdate { } } -/// Signals sent by an overseer to a subsystem. -#[derive(PartialEq, Clone, Debug)] -pub enum OverseerSignal { - /// Subsystems should adjust their jobs to start and stop work on appropriate block hashes. - ActiveLeaves(ActiveLeavesUpdate), - /// `Subsystem` is informed of a finalized block by its block hash and number. - BlockFinalized(Hash, BlockNumber), - /// Conclude the work of the `Overseer` and all `Subsystem`s. - Conclude, -} - -/// A message type that a subsystem receives from an overseer. -/// It wraps signals from an overseer and messages that are circulating -/// between subsystems. -/// -/// It is generic over over the message type `M` that a particular `Subsystem` may use. -#[derive(Debug)] -pub enum FromOverseer { - /// Signal from the `Overseer`. - Signal(OverseerSignal), - - /// Some other `Subsystem`'s message. - Communication { - /// Contained message - msg: M, - }, -} - -impl From for FromOverseer { - fn from(signal: OverseerSignal) -> Self { - FromOverseer::Signal(signal) - } -} - /// An error type that describes faults that may happen /// /// These are: @@ -192,7 +164,7 @@ pub enum SubsystemError { Infallible(#[from] std::convert::Infallible), #[error(transparent)] - Prometheus(#[from] substrate_prometheus_endpoint::PrometheusError), + Prometheus(#[from] polkadot_node_metrics::metrics::prometheus::PrometheusError), #[error(transparent)] Jaeger(#[from] JaegerError), @@ -235,91 +207,6 @@ pub struct SpawnedSubsystem { /// [`SubsystemError`]: struct.SubsystemError.html pub type SubsystemResult = Result; -/// A sender used by subsystems to communicate with other subsystems. -/// -/// Each clone of this type may add more capacity to the bounded buffer, so clones should -/// be used sparingly. -#[async_trait] -pub trait SubsystemSender: Send + Clone + 'static { - /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: AllMessages); - - /// Send multiple direct messages to other `Subsystem`s, routed based on message type. - async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send; - - /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message - /// type. - /// - /// This function should be used only when there is some other bounding factor on the messages - /// sent with it. Otherwise, it risks a memory leak. - fn send_unbounded_message(&mut self, msg: AllMessages); -} - -/// A context type that is given to the [`Subsystem`] upon spawning. -/// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s -/// or spawn jobs. -/// -/// [`Overseer`]: struct.Overseer.html -/// [`SubsystemJob`]: trait.SubsystemJob.html -#[async_trait] -pub trait SubsystemContext: Send + Sized + 'static { - /// The message type of this context. Subsystems launched with this context will expect - /// to receive messages of this type. - type Message: Send; - - /// The message sender type of this context. Clones of the sender should be used sparingly. - type Sender: SubsystemSender; - - /// Try to asynchronously receive a message. - /// - /// This has to be used with caution, if you loop over this without - /// using `pending!()` macro you will end up with a busy loop! - async fn try_recv(&mut self) -> Result>, ()>; - - /// Receive a message. - async fn recv(&mut self) -> SubsystemResult>; - - /// Spawn a child task on the executor. - fn spawn(&mut self, name: &'static str, s: Pin + Send>>) -> SubsystemResult<()>; - - /// Spawn a blocking child task on the executor's dedicated thread pool. - fn spawn_blocking( - &mut self, - name: &'static str, - s: Pin + Send>>, - ) -> SubsystemResult<()>; - - /// Get a mutable reference to the sender. - fn sender(&mut self) -> &mut Self::Sender; - - /// Send a direct message to some other `Subsystem`, routed based on message type. - async fn send_message(&mut self, msg: AllMessages) { - self.sender().send_message(msg).await - } - - /// Send multiple direct messages to other `Subsystem`s, routed based on message type. - async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send - { - self.sender().send_messages(msgs).await - } - - - /// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message - /// type. - /// - /// This function should be used only when there is some other bounding factor on the messages - /// sent with it. Otherwise, it risks a memory leak. - /// - /// Generally, for this method to be used, these conditions should be met: - /// * There is a communication cycle between subsystems - /// * One of the parts of the cycle has a clear bound on the number of messages produced. - fn send_unbounded_message(&mut self, msg: AllMessages) { - self.sender().send_unbounded_message(msg) - } -} - // Simplify usage without having to do large scale modifications of all // subsystems at once. pub type SubsystemSender = polkadot_overseer::gen::SubsystemSender; diff --git a/node/test/service/src/lib.rs b/node/test/service/src/lib.rs index d7425cde3d41..0eeebb705f45 100644 --- a/node/test/service/src/lib.rs +++ b/node/test/service/src/lib.rs @@ -22,7 +22,7 @@ pub mod chain_spec; pub use chain_spec::*; use futures::future::Future; -use polkadot_overseer::OverseerHandler; +use polkadot_overseer::Handler; use polkadot_primitives::v1::{ Id as ParaId, HeadData, ValidationCode, Balance, CollatorPair, }; @@ -290,7 +290,7 @@ pub struct PolkadotTestNode { /// Client's instance. pub client: Arc, /// The overseer handler. - pub overseer_handler: OverseerHandler, + pub overseer_handler: Handler, /// The `MultiaddrWithPeerId` to this node. This is useful if you want to pass it as "boot node" to other nodes. pub addr: MultiaddrWithPeerId, /// RPCHandlers to make RPC queries. From 24c3668507414eb01f83f4ca00acfd2377cd8eec Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 1 Jul 2021 13:10:48 +0200 Subject: [PATCH 075/161] error foo --- node/core/parachains-inherent/src/lib.rs | 6 +-- node/subsystem/src/lib.rs | 52 +----------------------- 2 files changed, 4 insertions(+), 54 deletions(-) diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs index 83e1b7718ac6..ab96741a37b1 100644 --- a/node/core/parachains-inherent/src/lib.rs +++ b/node/core/parachains-inherent/src/lib.rs @@ -57,7 +57,7 @@ impl ParachainsInherentDataProvider { let pid = async { let (sender, receiver) = futures::channel::oneshot::channel(); overseer.wait_for_activation(parent, sender).await; - receiver.await.map_err(|_| Error::ClosedChannelAwaitingActivation)?.map_err(Error::Subsystem)?; + receiver.await.map_err(|_| Error::ClosedChannelAwaitingActivation)?.map_err(|e| Error::Subsystem(e))?; let (sender, receiver) = futures::channel::oneshot::channel(); overseer.send_msg(AllMessages::Provisioner( @@ -127,7 +127,7 @@ impl sp_inherents::InherentDataProvider for ParachainsInherentDataProvider { #[derive(thiserror::Error, Debug)] pub enum Error { #[error("Blockchain error")] - Blockchain(sp_blockchain::Error), + Blockchain(#[from] sp_blockchain::Error), #[error("Timeout: provisioner did not return inherent data after {:?}", PROVISIONER_TIMEOUT)] Timeout, #[error("Could not find the parent header in the blockchain: {:?}", _0)] @@ -137,5 +137,5 @@ pub enum Error { #[error("Closed channel from provisioner when awaiting inherent data")] ClosedChannelAwaitingInherentData, #[error("Subsystem failed")] - Subsystem(SubsystemError), + Subsystem(#[from] SubsystemError), } diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 6c13f2dcf205..485820ab4539 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -32,6 +32,7 @@ use smallvec::SmallVec; use futures::prelude::*; use futures::channel::{oneshot, mpsc}; use futures::future::BoxFuture; +use polkadot_node_subsystem_types::errors::*; use polkadot_overseer::{AllMessages, OverseerSignal}; use polkadot_primitives::v1::{Hash, BlockNumber}; /// How many slots are stack-reserved for active leaves updates @@ -141,57 +142,6 @@ impl fmt::Debug for ActiveLeavesUpdate { } } -/// An error type that describes faults that may happen -/// -/// These are: -/// * Channels being closed -/// * Subsystems dying when they are not expected to -/// * Subsystems not dying when they are told to die -/// * etc. -#[derive(thiserror::Error, Debug)] -#[allow(missing_docs)] -pub enum SubsystemError { - #[error(transparent)] - NotifyCancellation(#[from] oneshot::Canceled), - - #[error(transparent)] - QueueError(#[from] mpsc::SendError), - - #[error("Failed to spawn a task: {0}")] - TaskSpawn(&'static str), - - #[error(transparent)] - Infallible(#[from] std::convert::Infallible), - - #[error(transparent)] - Prometheus(#[from] polkadot_node_metrics::metrics::prometheus::PrometheusError), - - #[error(transparent)] - Jaeger(#[from] JaegerError), - - #[error("Failed to {0}")] - Context(String), - - #[error("Subsystem stalled: {0}")] - SubsystemStalled(&'static str), - - /// Per origin (or subsystem) annotations to wrap an error. - #[error("Error originated in {origin}")] - FromOrigin { - /// An additional annotation tag for the origin of `source`. - origin: &'static str, - /// The wrapped error. Marked as source for tracking the error chain. - #[source] source: Box - }, -} - -impl SubsystemError { - /// Adds a `str` as `origin` to the given error `err`. - pub fn with_origin(origin: &'static str, err: E) -> Self { - Self::FromOrigin { origin, source: Box::new(err) } - } -} - /// An asynchronous subsystem task.. /// /// In essence it's just a newtype wrapping a `BoxFuture`. From d6c35be7c3cd8e551d67820ae23e5e4cbd53505f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 1 Jul 2021 14:16:53 +0200 Subject: [PATCH 076/161] dedup types, add convenience alias types --- Cargo.lock | 2 +- node/core/parachains-inherent/src/lib.rs | 6 +- node/malus/src/lib.rs | 8 +- node/overseer/src/lib.rs | 2 +- node/subsystem-types/src/lib.rs | 3 +- node/subsystem-util/Cargo.toml | 2 +- .../src/determine_new_blocks.rs | 4 +- node/subsystem-util/src/lib.rs | 7 +- .../src/rolling_session_window.rs | 18 ++- node/subsystem/src/lib.rs | 117 +++++++----------- 10 files changed, 74 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6701615b751d..22ff73711468 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6466,8 +6466,8 @@ dependencies = [ "polkadot-node-metrics", "polkadot-node-network-protocol", "polkadot-node-primitives", + "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", - "polkadot-node-subsystem-types", "polkadot-overseer", "polkadot-primitives", "rand 0.8.4", diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs index ab96741a37b1..ef68425bc37f 100644 --- a/node/core/parachains-inherent/src/lib.rs +++ b/node/core/parachains-inherent/src/lib.rs @@ -30,7 +30,7 @@ use polkadot_overseer::{ Handler, }; use polkadot_node_subsystem::{ - messages::ProvisionerMessage, SubsystemError, + messages::ProvisionerMessage, errors::SubsystemError, }; use polkadot_primitives::v1::{ Block, Hash, InherentData as ParachainsInherentData, @@ -60,9 +60,9 @@ impl ParachainsInherentDataProvider { receiver.await.map_err(|_| Error::ClosedChannelAwaitingActivation)?.map_err(|e| Error::Subsystem(e))?; let (sender, receiver) = futures::channel::oneshot::channel(); - overseer.send_msg(AllMessages::Provisioner( + overseer.send_msg( ProvisionerMessage::RequestInherentData(parent, sender), - )).await; + ).await; receiver.await.map_err(|_| Error::ClosedChannelAwaitingInherentData) }; diff --git a/node/malus/src/lib.rs b/node/malus/src/lib.rs index 1b8945a6c465..1161ced5a570 100644 --- a/node/malus/src/lib.rs +++ b/node/malus/src/lib.rs @@ -134,20 +134,20 @@ where } } - fn spawn( + async fn spawn( &mut self, name: &'static str, s: Pin + Send>>, ) -> SubsystemResult<()> { - self.inner.spawn(name, s) + self.inner.spawn(name, s).await } - fn spawn_blocking( + async fn spawn_blocking( &mut self, name: &'static str, s: Pin + Send>>, ) -> SubsystemResult<()> { - self.inner.spawn_blocking(name, s) + self.inner.spawn_blocking(name, s).await } fn sender(&mut self) -> &mut Self::Sender { diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 1685985fe46c..913e998a2af0 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -112,7 +112,7 @@ use polkadot_node_metrics::{ }, Metronome, }; -use polkadot_overseer_gen::{ +pub use polkadot_overseer_gen::{ OverseerError, TimeoutExt, SpawnNamed, diff --git a/node/subsystem-types/src/lib.rs b/node/subsystem-types/src/lib.rs index 982dacc8c2dd..95af2c2b373d 100644 --- a/node/subsystem-types/src/lib.rs +++ b/node/subsystem-types/src/lib.rs @@ -24,7 +24,7 @@ use std::{sync::Arc, fmt}; -use polkadot_primitives::v1::{Hash, BlockNumber}; +pub use polkadot_primitives::v1::{Hash, BlockNumber}; use smallvec::SmallVec; pub mod errors; @@ -69,6 +69,7 @@ impl LeafStatus { } } + /// Activated leaf. #[derive(Debug, Clone)] pub struct ActivatedLeaf { diff --git a/node/subsystem-util/Cargo.toml b/node/subsystem-util/Cargo.toml index 0d8b583e611a..f8414a1254b1 100644 --- a/node/subsystem-util/Cargo.toml +++ b/node/subsystem-util/Cargo.toml @@ -19,7 +19,7 @@ tracing = "0.1.26" lru = "0.6.5" polkadot-node-primitives = { path = "../primitives" } -polkadot-node-subsystem = { package = "polkadot-node-subsystem-types", path = "../subsystem-types" } +polkadot-node-subsystem = { package = "polkadot-node-subsystem", path = "../subsystem" } polkadot-node-jaeger = { path = "../jaeger" } polkadot-node-metrics = { path = "../metrics" } polkadot-node-network-protocol = { path = "../network/protocol" } diff --git a/node/subsystem-util/src/determine_new_blocks.rs b/node/subsystem-util/src/determine_new_blocks.rs index d90cc765f120..6253c2b38824 100644 --- a/node/subsystem-util/src/determine_new_blocks.rs +++ b/node/subsystem-util/src/determine_new_blocks.rs @@ -111,8 +111,8 @@ pub async fn determine_new_blocks( .map(|_| oneshot::channel()) .unzip::<_, _, Vec<_>, Vec<_>>(); - for (hash, sender) in batch_hashes.iter().cloned().zip(batch_senders) { - sender.send_message(ChainApiMessage::BlockHeader(hash, sender).into()).await; + for (hash, batched_sender) in batch_hashes.iter().cloned().zip(batch_senders) { + sender.send_message(ChainApiMessage::BlockHeader(hash, batched_sender).into()).await; } let mut requests = futures::stream::FuturesOrdered::new(); diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 866be758a7a7..5f5a1ac5c917 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -748,10 +748,13 @@ impl JobSubsystem { } } outgoing = jobs.next() => { + // TODO verify the introduced .await here is not a problem + // TODO it should only wait for the spawn to complete + // TODO but not for anything beyond that let res = match outgoing.expect("the Jobs stream never ends; qed") { - FromJobCommand::Spawn(name, task) => ctx.spawn(name, task), + FromJobCommand::Spawn(name, task) => ctx.spawn(name, task).await, FromJobCommand::SpawnBlocking(name, task) - => ctx.spawn_blocking(name, task), + => ctx.spawn_blocking(name, task).await, }; if let Err(e) = res { diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index 28484bcb1a79..bb36c2b9c3b2 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -20,10 +20,18 @@ //! care about the state of particular blocks. use polkadot_primitives::v1::{Hash, Header, SessionInfo, SessionIndex}; -use polkadot_overseer::SubsystemContext; + +use polkadot_overseer::AllMessages; use polkadot_node_subsystem::{ messages::{RuntimeApiMessage, RuntimeApiRequest}, errors::RuntimeApiError, + SubsystemContext, + Subsystem, + SubsystemSender, + SubsystemInstance, + SubsystemResult, + SpawnedSubsystem, + FromOverseer, }; use futures::channel::oneshot; @@ -159,10 +167,10 @@ impl RollingSessionWindow { // The genesis is guaranteed to be at the beginning of the session and its parent state // is non-existent. Therefore if we're at the genesis, we request using its state and // not the parent. - ctx.send_message(RuntimeApiMessage::Request( + ctx.send_message(AllMessages::RuntimeApiMessage(RuntimeApiMessage::Request( if block_header.number == 0 { block_hash } else { block_header.parent_hash }, RuntimeApiRequest::SessionIndexForChild(s_tx), - ).into()).await; + ))).await; match s_rx.await { Ok(Ok(s)) => s, @@ -271,10 +279,10 @@ async fn load_all_sessions( let mut v = Vec::new(); for i in start..=end_inclusive { let (tx, rx)= oneshot::channel(); - ctx.send_message(RuntimeApiMessage::Request( + ctx.send_message(AllMessages::from(RuntimeApiMessage::Request( block_hash, RuntimeApiRequest::SessionInfo(i, tx), - ).into()).await; + ))).await; let session_info = match rx.await { Ok(Ok(Some(s))) => s, diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 485820ab4539..628b09b6d1a3 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -33,7 +33,7 @@ use futures::prelude::*; use futures::channel::{oneshot, mpsc}; use futures::future::BoxFuture; use polkadot_node_subsystem_types::errors::*; -use polkadot_overseer::{AllMessages, OverseerSignal}; +pub use polkadot_overseer::{AllMessages, OverseerSignal, ActiveLeavesUpdate}; use polkadot_primitives::v1::{Hash, BlockNumber}; /// How many slots are stack-reserved for active leaves updates /// @@ -70,78 +70,6 @@ impl LeafStatus { } } } - -/// Activated leaf. -#[derive(Debug, Clone)] -pub struct ActivatedLeaf { - /// The block hash. - pub hash: Hash, - /// The block number. - pub number: BlockNumber, - /// The status of the leaf. - pub status: LeafStatus, - /// An associated [`jaeger::Span`]. - /// - /// NOTE: Each span should only be kept active as long as the leaf is considered active and should be dropped - /// when the leaf is deactivated. - pub span: Arc, -} - -/// Changes in the set of active leaves: the parachain heads which we care to work on. -/// -/// Note that the activated and deactivated fields indicate deltas, not complete sets. -#[derive(Clone, Default)] -pub struct ActiveLeavesUpdate { - /// New relay chain blocks of interest. - pub activated: SmallVec<[ActivatedLeaf; ACTIVE_LEAVES_SMALLVEC_CAPACITY]>, - /// Relay chain block hashes no longer of interest. - pub deactivated: SmallVec<[Hash; ACTIVE_LEAVES_SMALLVEC_CAPACITY]>, -} - -impl ActiveLeavesUpdate { - /// Create a ActiveLeavesUpdate with a single activated hash - pub fn start_work(activated: ActivatedLeaf) -> Self { - Self { activated: [activated][..].into(), ..Default::default() } - } - - /// Create a ActiveLeavesUpdate with a single deactivated hash - pub fn stop_work(hash: Hash) -> Self { - Self { deactivated: [hash][..].into(), ..Default::default() } - } - - /// Is this update empty and doesn't contain any information? - pub fn is_empty(&self) -> bool { - self.activated.is_empty() && self.deactivated.is_empty() - } -} - -impl PartialEq for ActiveLeavesUpdate { - /// Equality for `ActiveLeavesUpdate` doesnt imply bitwise equality. - /// - /// Instead, it means equality when `activated` and `deactivated` are considered as sets. - fn eq(&self, other: &Self) -> bool { - self.activated.len() == other.activated.len() && self.deactivated.len() == other.deactivated.len() - && self.activated.iter().all(|a| other.activated.iter().any(|o| a.hash == o.hash)) - && self.deactivated.iter().all(|a| other.deactivated.contains(a)) - } -} - -impl fmt::Debug for ActiveLeavesUpdate { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - struct Activated<'a>(&'a [ActivatedLeaf]); - impl fmt::Debug for Activated<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_list().entries(self.0.iter().map(|e| e.hash)).finish() - } - } - - f.debug_struct("ActiveLeavesUpdate") - .field("activated", &Activated(&self.activated)) - .field("deactivated", &self.deactivated) - .finish() - } -} - /// An asynchronous subsystem task.. /// /// In essence it's just a newtype wrapping a `BoxFuture`. @@ -159,10 +87,49 @@ pub type SubsystemResult = Result; // Simplify usage without having to do large scale modifications of all // subsystems at once. -pub type SubsystemSender = polkadot_overseer::gen::SubsystemSender; pub type FromOverseer = polkadot_overseer::gen::FromOverseer; -pub type Subsystem = polkadot_overseer::gen::Subsystem; pub type SubsystemInstance = polkadot_overseer::gen::SubsystemInstance; + +// Same for traits +// pub trait SubsystemSender: polkadot_overseer::gen::SubsystemSender {} + +// impl SubsystemSender for T where T: polkadot_overseer::gen::SubsystemSender { +// } + +pub use polkadot_overseer::gen::SubsystemSender; + + +pub trait Subsystem : polkadot_overseer::gen::Subsystem +where + Ctx: SubsystemContext, +{} + +impl Subsystem for T +where + T: polkadot_overseer::gen::Subsystem, + Ctx: SubsystemContext, +{} + + + + +pub trait SubsystemContext: polkadot_overseer::gen::SubsystemContext< + Signal=OverseerSignal, + AllMessages=AllMessages, + Error=SubsystemError, +> +{ +} + +impl SubsystemContext for T +where + T: polkadot_overseer::gen::SubsystemContext< + Signal=OverseerSignal, + AllMessages=AllMessages, + Error=SubsystemError, + >, +{ +} From 3323cd7e8567d266e18b8762594717ea07731bb4 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 1 Jul 2021 15:58:19 +0200 Subject: [PATCH 077/161] foo --- node/subsystem-util/src/lib.rs | 2 +- node/subsystem/src/lib.rs | 50 +++++++++++++++++----------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 5f5a1ac5c917..841ab98f5c93 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -26,7 +26,7 @@ use polkadot_node_subsystem::{ errors::RuntimeApiError, - messages::{RuntimeApiMessage, RuntimeApiRequest, RuntimeApiSender, BoundToRelayParent}, + messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest, RuntimeApiSender, BoundToRelayParent}, ActiveLeavesUpdate, OverseerSignal, errors::{ SubsystemError, diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 628b09b6d1a3..01b3f0bcb0f1 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -20,9 +20,6 @@ #![warn(missing_docs)] -use polkadot_node_subsystem_types::messages::AvailabilityStoreMessage; -pub use polkadot_node_subsystem_types::{errors, messages}; - pub use polkadot_node_jaeger as jaeger; pub use jaeger::*; @@ -33,7 +30,7 @@ use futures::prelude::*; use futures::channel::{oneshot, mpsc}; use futures::future::BoxFuture; use polkadot_node_subsystem_types::errors::*; -pub use polkadot_overseer::{AllMessages, OverseerSignal, ActiveLeavesUpdate}; +pub use polkadot_overseer::{OverseerSignal, ActiveLeavesUpdate}; use polkadot_primitives::v1::{Hash, BlockNumber}; /// How many slots are stack-reserved for active leaves updates /// @@ -41,6 +38,11 @@ use polkadot_primitives::v1::{Hash, BlockNumber}; /// If there are greater than this number of slots, then we fall back to a heap vector. const ACTIVE_LEAVES_SMALLVEC_CAPACITY: usize = 8; +pub use polkadot_node_subsystem_types::errors::{self, *}; +pub mod messages { + pub use polkadot_overseer::AllMessages; + pub use polkadot_node_subsystem_types::messages::*; +} /// The status of an activated leaf. #[derive(Debug, Clone)] @@ -70,15 +72,6 @@ impl LeafStatus { } } } -/// An asynchronous subsystem task.. -/// -/// In essence it's just a newtype wrapping a `BoxFuture`. -pub struct SpawnedSubsystem { - /// Name of the subsystem being spawned. - pub name: &'static str, - /// The task of the subsystem being spawned. - pub future: BoxFuture<'static, SubsystemResult<()>>, -} /// A `Result` type that wraps [`SubsystemError`]. /// @@ -94,42 +87,49 @@ pub type FromOverseer = polkadot_overseer::gen::FromOverseer = polkadot_overseer::gen::SubsystemInstance; // Same for traits -// pub trait SubsystemSender: polkadot_overseer::gen::SubsystemSender {} +pub trait SubsystemSender: polkadot_overseer::gen::SubsystemSender {} -// impl SubsystemSender for T where T: polkadot_overseer::gen::SubsystemSender { -// } - -pub use polkadot_overseer::gen::SubsystemSender; +impl SubsystemSender for T where T: polkadot_overseer::gen::SubsystemSender { +} +// pub use polkadot_overseer::gen::SubsystemSender; +pub type SpawnedSubsystem = polkadot_overseer::gen::SpawnedSubsystem; pub trait Subsystem : polkadot_overseer::gen::Subsystem where Ctx: SubsystemContext, -{} +{ + fn start(self, ctx: Ctx) -> SpawnedSubsystem; +} impl Subsystem for T where T: polkadot_overseer::gen::Subsystem, Ctx: SubsystemContext, -{} - - - +{ + fn start(self, ctx: Ctx) -> SpawnedSubsystem { + >::start(self, ctx) + } +} pub trait SubsystemContext: polkadot_overseer::gen::SubsystemContext< Signal=OverseerSignal, - AllMessages=AllMessages, + AllMessages=messages::AllMessages, Error=SubsystemError, > { + } impl SubsystemContext for T where T: polkadot_overseer::gen::SubsystemContext< Signal=OverseerSignal, - AllMessages=AllMessages, + AllMessages=messages::AllMessages, Error=SubsystemError, >, { + } From 601a5660ba7e4d9ea164f73844a1c756c3251f42 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 1 Jul 2021 17:19:25 +0200 Subject: [PATCH 078/161] make backing work --- node/core/backing/src/lib.rs | 32 +++--- node/core/candidate-validation/src/lib.rs | 11 ++- node/network/availability-recovery/src/lib.rs | 6 +- node/network/bridge/src/lib.rs | 14 +-- node/overseer/overseer-gen/src/lib.rs | 2 +- node/overseer/src/lib.rs | 4 + node/subsystem-test-helpers/src/lib.rs | 2 +- .../src/determine_new_blocks.rs | 8 +- node/subsystem-util/src/lib.rs | 99 ++++++++++--------- .../src/rolling_session_window.rs | 1 - node/subsystem-util/src/runtime/mod.rs | 30 +++--- node/subsystem/src/lib.rs | 30 ++---- 12 files changed, 117 insertions(+), 122 deletions(-) diff --git a/node/core/backing/src/lib.rs b/node/core/backing/src/lib.rs index 7fccfdd419ae..fa7b0bb3ee3d 100644 --- a/node/core/backing/src/lib.rs +++ b/node/core/backing/src/lib.rs @@ -37,6 +37,7 @@ use polkadot_node_primitives::{ use polkadot_subsystem::{ PerLeafSpan, Stage, SubsystemSender, jaeger, + overseer, messages::{ AllMessages, AvailabilityDistributionMessage, AvailabilityStoreMessage, CandidateBackingMessage, CandidateValidationMessage, CollatorProtocolMessage, @@ -308,7 +309,7 @@ async fn store_available_data( n_validators, available_data, tx, - ).into()).await; + )).await; let _ = rx.await.map_err(Error::StoreAvailableData)?; @@ -384,7 +385,7 @@ async fn request_pov( candidate_hash, pov_hash, tx, - }.into()).await; + }).await; let pov = rx.await.map_err(|_| Error::FetchPoV)?; Ok(Arc::new(pov)) @@ -397,13 +398,12 @@ async fn request_candidate_validation( ) -> Result { let (tx, rx) = oneshot::channel(); - sender.send_message(AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - candidate, - pov, - tx, - ) - ).into() + sender.send_message( + CandidateValidationMessage::ValidateFromChainState( + candidate, + pov, + tx, + ) ).await; match rx.await { @@ -415,7 +415,7 @@ async fn request_candidate_validation( type BackgroundValidationResult = Result<(CandidateReceipt, CandidateCommitments, Arc), CandidateReceipt>; -struct BackgroundValidationParams { +struct BackgroundValidationParams, F> { sender: JobSender, tx_command: mpsc::Sender, candidate: CandidateReceipt, @@ -600,14 +600,14 @@ impl CandidateBackingJob { root_span, ).await? { sender.send_message( - CollatorProtocolMessage::Seconded(self.parent, stmt).into() + CollatorProtocolMessage::Seconded(self.parent, stmt) ).await; } } } Err(candidate) => { sender.send_message( - CollatorProtocolMessage::Invalid(self.parent, candidate).into() + CollatorProtocolMessage::Invalid(self.parent, candidate) ).await; } } @@ -683,7 +683,7 @@ impl CandidateBackingJob { .map_or(false, |c| c != &candidate.descriptor().collator) { sender.send_message( - CollatorProtocolMessage::Invalid(self.parent, candidate.clone()).into() + CollatorProtocolMessage::Invalid(self.parent, candidate.clone()) ).await; return Ok(()); } @@ -732,7 +732,7 @@ impl CandidateBackingJob { if let Some(signed_statement) = self.sign_statement(statement).await { self.import_statement(sender, &signed_statement, root_span).await?; let smsg = StatementDistributionMessage::Share(self.parent, signed_statement.clone()); - sender.send_unbounded_message(smsg.into()); + sender.send_unbounded_message(smsg); Ok(Some(signed_statement)) } else { @@ -749,7 +749,7 @@ impl CandidateBackingJob { ProvisionerMessage::ProvisionableData( self.parent, ProvisionableData::MisbehaviorReport(self.parent, validator_id, report) - ).into() + ) ).await; } } @@ -801,7 +801,7 @@ impl CandidateBackingJob { self.parent, ProvisionableData::BackedCandidate(backed.receipt()), ); - sender.send_message(message.into()).await; + sender.send_message(message).await; span.as_ref().map(|s| s.child("backed")); span diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index aabdf55517ed..d58cd7b76e08 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -24,15 +24,16 @@ #![warn(missing_docs)] use polkadot_subsystem::{ - Subsystem, SubsystemContext, SpawnedSubsystem, SubsystemResult, SubsystemError, + overseer, + SubsystemContext, SpawnedSubsystem, SubsystemResult, SubsystemError, FromOverseer, OverseerSignal, messages::{ AllMessages, CandidateValidationMessage, RuntimeApiMessage, ValidationFailed, RuntimeApiRequest, }, + errors::RuntimeApiError, }; use polkadot_node_subsystem_util::metrics::{self, prometheus}; -use polkadot_subsystem::errors::RuntimeApiError; use polkadot_node_primitives::{ VALIDATION_CODE_BOMB_LIMIT, POV_BOMB_LIMIT, ValidationResult, InvalidCandidate, PoV, BlockData, }; @@ -84,7 +85,7 @@ impl CandidateValidationSubsystem { } } -impl Subsystem for CandidateValidationSubsystem where +impl overseer::Subsystem for CandidateValidationSubsystem where C: SubsystemContext, { fn start(self, ctx: C) -> SpawnedSubsystem { @@ -107,7 +108,7 @@ async fn run( let (mut validation_host, task) = polkadot_node_core_pvf::start( polkadot_node_core_pvf::Config::new(cache_path, program_path), ); - ctx.spawn_blocking("pvf-validation-host", task.boxed())?; + ctx.spawn_blocking("pvf-validation-host", task.boxed()).await?; loop { match ctx.recv().await? { @@ -181,7 +182,7 @@ async fn runtime_api_request( receiver: oneshot::Receiver>, ) -> SubsystemResult> { ctx.send_message( - AllMessages::RuntimeApi(RuntimeApiMessage::Request( + AllMessages::RuntimeApiMessage(RuntimeApiMessage::Request( relay_parent, request, )) diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index d9b206fcbf52..efd8d3712851 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -133,7 +133,7 @@ impl RequestFromBackersPhase { async fn run( &mut self, params: &InteractionParams, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) -> Result { tracing::trace!( target: LOG_TARGET, @@ -218,7 +218,7 @@ impl RequestChunksPhase { async fn launch_parallel_requests( &mut self, params: &InteractionParams, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) { let max_requests = std::cmp::min(N_PARALLEL, params.threshold); while self.requesting_chunks.len() < max_requests { @@ -330,7 +330,7 @@ impl RequestChunksPhase { async fn run( &mut self, params: &InteractionParams, - sender: &mut impl SubsystemSender, + sender: &mut impl SubsystemSender, ) -> Result { // First query the store for any chunks we've got. { diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index fb87a3bb6fa9..6a0e77fc99ce 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -605,7 +605,7 @@ where } async fn handle_network_messages( - mut sender: impl SubsystemSender, + mut sender: impl SubsystemSender, mut network_service: impl Network, network_stream: BoxStream<'static, NetworkEvent>, mut authority_discovery_service: AD, @@ -1104,21 +1104,21 @@ fn send_collation_message( async fn dispatch_validation_event_to_all( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) { dispatch_validation_events_to_all(std::iter::once(event), ctx).await } async fn dispatch_collation_event_to_all( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) { dispatch_collation_events_to_all(std::iter::once(event), ctx).await } fn dispatch_validation_event_to_all_unbounded( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) { for msg in AllMessages::dispatch_iter(event) { ctx.send_unbounded_message(msg); @@ -1127,7 +1127,7 @@ fn dispatch_validation_event_to_all_unbounded( fn dispatch_collation_event_to_all_unbounded( event: NetworkBridgeEvent, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) { if let Some(msg) = event.focus().ok().map(CollatorProtocolMessage::NetworkBridgeUpdateV1) { ctx.send_unbounded_message(msg.into()); @@ -1136,7 +1136,7 @@ fn dispatch_collation_event_to_all_unbounded( async fn dispatch_validation_events_to_all( events: I, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) where I: IntoIterator>, @@ -1147,7 +1147,7 @@ async fn dispatch_validation_events_to_all( async fn dispatch_collation_events_to_all( events: I, - ctx: &mut impl SubsystemSender + ctx: &mut impl SubsystemSender ) where I: IntoIterator>, diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index fcf39e7840a4..e38bc4c26b6e 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -368,7 +368,7 @@ pub trait SubsystemContext: Send + 'static { /// In some cases can be identical to `Self::Message`. type AllMessages: From + Send + 'static; /// The sender type as provided by `sender()` and underlying. - type Sender: SubsystemSender + SubsystemSender + std::fmt::Debug + Send; + type Sender: SubsystemSender + std::fmt::Debug + Send; /// The error type. type Error: ::std::error::Error + ::std::convert::From< OverseerError > + Sync + Send + 'static; diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 913e998a2af0..c233b193156e 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -391,6 +391,10 @@ pub struct Overseer { /// Various Prometheus metrics. pub metrics: Metrics, + + // Configuration for the candidate validation subsystem. + // FIXME !!! + // pub candidate_validation_config: CandidateValidationConfig, } impl Overseer diff --git a/node/subsystem-test-helpers/src/lib.rs b/node/subsystem-test-helpers/src/lib.rs index ee3ca4fe48cc..bd9342277f5b 100644 --- a/node/subsystem-test-helpers/src/lib.rs +++ b/node/subsystem-test-helpers/src/lib.rs @@ -171,7 +171,7 @@ pub fn sender_receiver() -> (TestSubsystemSender, mpsc::UnboundedReceiver for TestSubsystemSender { +impl SubsystemSender for TestSubsystemSender { async fn send_message(&mut self, msg: AllMessages) { self.tx .send(msg) diff --git a/node/subsystem-util/src/determine_new_blocks.rs b/node/subsystem-util/src/determine_new_blocks.rs index 6253c2b38824..1a92d2f20a08 100644 --- a/node/subsystem-util/src/determine_new_blocks.rs +++ b/node/subsystem-util/src/determine_new_blocks.rs @@ -19,9 +19,9 @@ use polkadot_node_subsystem::{ messages::ChainApiMessage, }; -use polkadot_overseer::{ - AllMessages, - gen::SubsystemSender, +use polkadot_node_subsystem::{ + messages::AllMessages, + SubsystemSender, }; use polkadot_primitives::v1::{Hash, Header, BlockNumber}; use futures::prelude::*; @@ -45,7 +45,7 @@ pub async fn determine_new_blocks( header: &Header, lower_bound_number: BlockNumber, ) -> Result, E> where - Sender: SubsystemSender, + Sender: SubsystemSender, { const ANCESTRY_STEP: usize = 4; diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 841ab98f5c93..4999f2629e19 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -25,22 +25,29 @@ #![warn(missing_docs)] use polkadot_node_subsystem::{ + overseer, errors::RuntimeApiError, - messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest, RuntimeApiSender, BoundToRelayParent}, + messages::{ + AllMessages, + RuntimeApiMessage, + RuntimeApiRequest, + RuntimeApiSender, + BoundToRelayParent, + }, ActiveLeavesUpdate, OverseerSignal, + SubsystemSender, errors::{ SubsystemError, SubsystemResult, }, + SubsystemContext, + SpawnedSubsystem, + FromOverseer, }; -pub use polkadot_overseer::{ - FromOverseer, +pub use overseer::{ Subsystem, - SubsystemContext, - SubsystemSender, TimeoutExt, - gen::SpawnedSubsystem, gen::OverseerError, gen::Timeout, }; @@ -142,15 +149,14 @@ impl From for Error { pub type RuntimeApiReceiver = oneshot::Receiver>; /// Request some data from the `RuntimeApi`. -pub async fn request_from_runtime( +pub async fn request_from_runtime( parent: Hash, sender: &mut Sender, request_builder: RequestBuilder, ) -> RuntimeApiReceiver where RequestBuilder: FnOnce(RuntimeApiSender) -> RuntimeApiRequest, - Sender: SubsystemSender, - M: From, + Sender: SubsystemSender, { let (tx, rx) = oneshot::channel(); @@ -175,7 +181,7 @@ macro_rules! specialize_requests { #[doc = "Request `"] #[doc = $doc_name] #[doc = "` from the runtime"] - pub async fn $func_name ( + pub async fn $func_name ( parent: Hash, $( $param_name: $param_ty, @@ -183,10 +189,9 @@ macro_rules! specialize_requests { sender: &mut Sender, ) -> RuntimeApiReceiver<$return_ty> where - M: From, - Sender: SubsystemSender, + Sender: SubsystemSender, { - request_from_runtime::<_, _, Sender, M>(parent, sender, |tx| RuntimeApiRequest::$request_variant( + request_from_runtime::<_, _, Sender>(parent, sender, |tx| RuntimeApiRequest::$request_variant( $( $param_name, )* tx )).await } @@ -295,20 +300,18 @@ pub struct Validator { impl Validator { /// Get a struct representing this node's validator if this node is in fact a validator in the context of the given block. - pub async fn new( + pub async fn new( parent: Hash, keystore: SyncCryptoStorePtr, - sender: &mut impl SubsystemSender , + sender: &mut impl SubsystemSender, ) -> Result - where - M: From, { // Note: request_validators and request_session_index_for_child do not and cannot // run concurrently: they both have a mutable handle to the same sender. // However, each of them returns a oneshot::Receiver, and those are resolved concurrently. let (validators, session_index) = futures::try_join!( request_validators(parent, sender).await, - request_session_index_for_child::(parent, sender).await, + request_session_index_for_child(parent, sender).await, )?; let signing_context = SigningContext { @@ -395,40 +398,40 @@ pub enum FromJobCommand { } /// A sender for messages from jobs, as well as commands to the overseer. -pub struct JobSender, M> { +pub struct JobSender { sender: S, from_job: mpsc::Sender, - _phantom: std::marker::PhantomData, } // A custom clone impl, since M does not need to impl `Clone` // which `#[derive(Clone)]` requires. -impl, M> Clone for JobSender { +impl Clone for JobSender { fn clone(&self) -> Self { Self { sender: self.sender.clone(), from_job: self.from_job.clone(), - _phantom: Default::default(), } } } -impl, M> JobSender { +impl JobSender { /// Get access to the underlying subsystem sender. pub fn subsystem_sender(&mut self) -> &mut S { &mut self.sender } /// Send a direct message to some other `Subsystem`, routed based on message type. - pub async fn send_message(&mut self, msg: M) { - self.sender.send_message(msg).await + pub async fn send_message(&mut self, msg: impl Into) { + self.sender.send_message(msg.into()).await } /// Send multiple direct messages to other `Subsystem`s, routed based on message type. - pub async fn send_messages(&mut self, msgs: T) - where T: IntoIterator + Send, T::IntoIter: Send + pub async fn send_messages(&mut self, msgs: T) + where + T: IntoIterator + Send, T::IntoIter: Send, + M: Into, { - self.sender.send_messages(msgs).await + self.sender.send_messages(msgs.into_iter().map(|m| m.into())).await } @@ -437,8 +440,8 @@ impl, M> JobSender { /// /// This function should be used only when there is some other bounding factor on the messages /// sent with it. Otherwise, it risks a memory leak. - pub fn send_unbounded_message(&mut self, msg: M) { - self.sender.send_unbounded_message(msg) + pub fn send_unbounded_message(&mut self, msg: impl Into) { + self.sender.send_unbounded_message(msg.into()) } /// Send a command to the subsystem, to be relayed onwards to the overseer. @@ -447,24 +450,25 @@ impl, M> JobSender { } } + #[async_trait::async_trait] -impl SubsystemSender for JobSender +impl overseer::SubsystemSender for JobSender where - M: Send + 'static, - S: SubsystemSender + Clone, + M: Send + 'static + Into, + S: SubsystemSender + Clone, { async fn send_message(&mut self, msg: M) { - self.sender.send_message(msg).await + self.sender.send_message(msg.into()).await } async fn send_messages(&mut self, msgs: T) where T: IntoIterator + Send, T::IntoIter: Send { - self.sender.send_messages(msgs).await + self.sender.send_messages(msgs.into_iter().map(|m| m.into())).await } fn send_unbounded_message(&mut self, msg: M) { - self.sender.send_unbounded_message(msg) + self.sender.send_unbounded_message(msg.into()) } } @@ -503,13 +507,13 @@ pub trait JobTrait: Unpin + Sized { /// Run a job for the given relay `parent`. /// /// The job should be ended when `receiver` returns `None`. - fn run>( + fn run( parent: Hash, span: Arc, run_args: Self::RunArgs, metrics: Self::Metrics, receiver: mpsc::Receiver, - sender: JobSender, + sender: JobSender, ) -> Pin> + Send>>; } @@ -565,7 +569,7 @@ where ) where Job: JobTrait, - Sender: SubsystemSender, + Sender: SubsystemSender, { let (to_job_tx, to_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY); let (from_job_tx, from_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY); @@ -580,7 +584,6 @@ where JobSender { sender, from_job: from_job_tx, - _phantom: Default::default(), }, ).await { tracing::error!( @@ -687,11 +690,11 @@ impl JobSubsystem { where Spawner: SpawnNamed + Send + Clone + Unpin + 'static, Context: SubsystemContext::ToJob, Signal=OverseerSignal>, - ::Sender: Clone, + ::Sender: SubsystemSender, Job: 'static + JobTrait + Send, - Job::RunArgs: Clone + Sync, - Job::ToJob: From, - Job::Metrics: Sync, + ::RunArgs: Clone + Sync, + ::ToJob: Sync + From<::Message>, + ::Metrics: Sync, { let JobSubsystem { params: JobSubsystemParams { @@ -713,7 +716,7 @@ impl JobSubsystem { deactivated, }))) => { for activated in activated { - let sender: Context::Sender = ctx.sender().clone(); + let sender = ctx.sender().clone(); jobs.spawn_job::( activated.hash, activated.span, @@ -733,7 +736,7 @@ impl JobSubsystem { } Ok(FromOverseer::Signal(OverseerSignal::BlockFinalized(..))) => {} Ok(FromOverseer::Communication { msg }) => { - if let Ok(to_job) = ::try_from(msg) { + if let Ok(to_job) = <::Message>::try_from(msg) { jobs.send_msg(to_job.relay_parent(), to_job).await; } } @@ -773,10 +776,10 @@ where Context: SubsystemContext, Job: 'static + JobTrait + Send, Job::RunArgs: Clone + Sync, - Job::ToJob: Sync + From, + ::ToJob: Sync + From<::Message>, Job::Metrics: Sync, { - fn start(self, ctx: Context) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { self.run(ctx).await; Ok(()) diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index bb36c2b9c3b2..1c012e965561 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -26,7 +26,6 @@ use polkadot_node_subsystem::{ messages::{RuntimeApiMessage, RuntimeApiRequest}, errors::RuntimeApiError, SubsystemContext, - Subsystem, SubsystemSender, SubsystemInstance, SubsystemResult, diff --git a/node/subsystem-util/src/runtime/mod.rs b/node/subsystem-util/src/runtime/mod.rs index 77d0a07c4319..4982b52d96f0 100644 --- a/node/subsystem-util/src/runtime/mod.rs +++ b/node/subsystem-util/src/runtime/mod.rs @@ -24,9 +24,11 @@ use sp_core::crypto::Public; use sp_keystore::{CryptoStore, SyncCryptoStorePtr}; use polkadot_primitives::v1::{CoreState, EncodeAs, GroupIndex, GroupRotationInfo, Hash, OccupiedCore, SessionIndex, SessionInfo, Signed, SigningContext, UncheckedSigned, ValidatorId, ValidatorIndex}; -use polkadot_overseer::SubsystemContext; -use polkadot_node_subsystem::messages::RuntimeApiMessage; -use polkadot_node_subsystem::OverseerSignal; +use polkadot_node_subsystem::{ + messages::RuntimeApiMessage, + OverseerSignal, + SubsystemContext, +}; use crate::{ @@ -96,13 +98,13 @@ impl RuntimeInfo { ) -> Result where M: From, - Context: SubsystemContext, + Context: SubsystemContext, { match self.session_index_cache.get(&parent) { Some(index) => Ok(*index), None => { let index = - recv_runtime(request_session_index_for_child::(parent, ctx.sender()).await) + recv_runtime(request_session_index_for_child(parent, ctx.sender()).await) .await?; self.session_index_cache.put(parent, index); Ok(index) @@ -118,7 +120,7 @@ impl RuntimeInfo { ) -> Result<&'a ExtendedSessionInfo> where M: From, - Context: SubsystemContext, + Context: SubsystemContext, { let session_index = self.get_session_index::<_,M>(ctx, parent).await?; @@ -137,11 +139,11 @@ impl RuntimeInfo { ) -> Result<&'a ExtendedSessionInfo> where M: From, - Context: SubsystemContext, + Context: SubsystemContext, { if !self.session_info_cache.contains(&session_index) { let session_info = - recv_runtime(request_session_info::(parent, session_index, ctx.sender()).await) + recv_runtime(request_session_info(parent, session_index, ctx.sender()).await) .await? .ok_or(NonFatal::NoSuchSession(session_index))?; let validator_info = self.get_validator_info(&session_info).await?; @@ -168,7 +170,7 @@ impl RuntimeInfo { ) -> Result, UncheckedSigned>> where M: From, - Context: SubsystemContext, + Context: SubsystemContext, Payload: EncodeAs + Clone, RealPayload: Encode + Clone, { @@ -253,9 +255,9 @@ pub async fn get_availability_cores(ctx: &mut Context, relay_parent: -> Result> where M: From, - Context: SubsystemContext, + Context: SubsystemContext, { - recv_runtime(request_availability_cores::(relay_parent, ctx.sender()).await).await + recv_runtime(request_availability_cores(relay_parent, ctx.sender()).await).await } /// Variant of `request_availability_cores` that only returns occupied ones. @@ -265,7 +267,7 @@ pub async fn get_occupied_cores( ) -> Result> where M: From, -Context: SubsystemContext, +Context: SubsystemContext, { let cores = get_availability_cores(ctx, relay_parent).await?; @@ -287,10 +289,10 @@ pub async fn get_group_rotation_info(ctx: &mut Context, relay_parent -> Result where M: From, - Context: SubsystemContext, + Context: SubsystemContext, { // We drop `groups` here as we don't need them, because of `RuntimeInfo`. Ideally we would not // fetch them in the first place. - let (_, info) = recv_runtime(request_validator_groups::(relay_parent, ctx.sender()).await).await?; + let (_, info) = recv_runtime(request_validator_groups(relay_parent, ctx.sender()).await).await?; Ok(info) } diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 01b3f0bcb0f1..f4d0ae84fae4 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -30,7 +30,7 @@ use futures::prelude::*; use futures::channel::{oneshot, mpsc}; use futures::future::BoxFuture; use polkadot_node_subsystem_types::errors::*; -pub use polkadot_overseer::{OverseerSignal, ActiveLeavesUpdate}; +pub use polkadot_overseer::{OverseerSignal, ActiveLeavesUpdate, self as overseer}; use polkadot_primitives::v1::{Hash, BlockNumber}; /// How many slots are stack-reserved for active leaves updates /// @@ -40,7 +40,7 @@ const ACTIVE_LEAVES_SMALLVEC_CAPACITY: usize = 8; pub use polkadot_node_subsystem_types::errors::{self, *}; pub mod messages { - pub use polkadot_overseer::AllMessages; + pub use super::overseer::AllMessages; pub use polkadot_node_subsystem_types::messages::*; } @@ -87,7 +87,8 @@ pub type FromOverseer = polkadot_overseer::gen::FromOverseer = polkadot_overseer::gen::SubsystemInstance; // Same for traits -pub trait SubsystemSender: polkadot_overseer::gen::SubsystemSender {} +pub trait SubsystemSender: polkadot_overseer::gen::SubsystemSender { +} impl SubsystemSender for T where T: polkadot_overseer::gen::SubsystemSender { } @@ -95,24 +96,7 @@ impl SubsystemSender for T where T: polkadot_overseer::gen::SubsystemSender; -pub trait Subsystem : polkadot_overseer::gen::Subsystem -where - Ctx: SubsystemContext, -{ - fn start(self, ctx: Ctx) -> SpawnedSubsystem; -} -impl Subsystem for T -where - T: polkadot_overseer::gen::Subsystem, - Ctx: SubsystemContext, -{ - fn start(self, ctx: Ctx) -> SpawnedSubsystem { - >::start(self, ctx) - } -} pub trait SubsystemContext: polkadot_overseer::gen::SubsystemContext< Signal=OverseerSignal, @@ -120,7 +104,8 @@ pub trait SubsystemContext: polkadot_overseer::gen::SubsystemContext< Error=SubsystemError, > { - + type Message: std::fmt::Debug + Send + 'static; + type Sender: SubsystemSender + Send + Clone + 'static; } impl SubsystemContext for T @@ -131,5 +116,6 @@ where Error=SubsystemError, >, { - + type Message = ::Message; + type Sender = ::Sender; } From 1844d1f6d434a3847b95782fe009b0eb82809c73 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 1 Jul 2021 17:49:31 +0200 Subject: [PATCH 079/161] make network bridge work --- node/network/bridge/src/lib.rs | 33 +++++++++++++++----------- node/network/bridge/src/multiplexer.rs | 2 +- node/overseer/src/lib.rs | 32 +++++++++++++++++++++++++ node/subsystem-types/src/messages.rs | 16 +++++++++++++ node/subsystem/src/lib.rs | 5 +++- 5 files changed, 72 insertions(+), 16 deletions(-) diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 6a0e77fc99ce..35cbc213eb20 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -28,20 +28,23 @@ use polkadot_overseer::Overseer; use sc_network::Event as NetworkEvent; use sp_consensus::SyncOracle; -use polkadot_overseer::{AllMessages, OverseerSignal}; use polkadot_overseer::gen::{ + Subsystem, + OverseerError, +}; +use polkadot_subsystem::{ + overseer, + OverseerSignal, FromOverseer, SpawnedSubsystem, - Subsystem, SubsystemContext, OverseerError, + SubsystemContext, SubsystemSender, -}; -use polkadot_subsystem::{ errors::{SubsystemError, SubsystemResult}, ActivatedLeaf, ActiveLeavesUpdate, - messages::StatementDistributionMessage -}; -use polkadot_subsystem::messages::{ - NetworkBridgeMessage, CollatorProtocolMessage, NetworkBridgeEvent, + messages::{ + AllMessages, StatementDistributionMessage, + NetworkBridgeMessage, CollatorProtocolMessage, NetworkBridgeEvent, + }, }; use polkadot_primitives::v1::{Hash, BlockNumber}; use polkadot_node_network_protocol::{ @@ -303,9 +306,9 @@ impl Subsystem for NetworkBridge, + Context: SubsystemContext + overseer::SubsystemContext, { - fn start(mut self, ctx: Context) -> SpawnedSubsystem { + fn start(mut self, ctx: Context) -> SpawnedSubsystem { // The stream of networking events has to be created at initialization, otherwise the // networking might open connections before the stream of events has been grabbed. let network_stream = self.network_service.event_stream(); @@ -375,7 +378,8 @@ async fn handle_subsystem_messages( metrics: Metrics, ) -> Result<(), UnexpectedAbort> where - Context: SubsystemContext, + Context: SubsystemContext, + Context: overseer::SubsystemContext, N: Network, AD: validator_discovery::AuthorityDiscovery, { @@ -867,14 +871,15 @@ async fn handle_network_messages( /// #fn is_send(); /// #is_send::(); /// ``` -async fn run_network( +async fn run_network( bridge: NetworkBridge, - mut ctx: impl SubsystemContext, + mut ctx: Context, network_stream: BoxStream<'static, NetworkEvent>, ) -> SubsystemResult<()> where N: Network, AD: validator_discovery::AuthorityDiscovery, + Context: SubsystemContext + overseer::SubsystemContext, { let shared = Shared::default(); @@ -900,7 +905,7 @@ where shared.clone(), ).remote_handle(); - ctx.spawn("network-bridge-network-worker", Box::pin(remote))?; + ctx.spawn("network-bridge-network-worker", Box::pin(remote)).await?; ctx.send_message( AllMessages::from(StatementDistributionMessage::StatementFetchingReceiver(statement_receiver)) diff --git a/node/network/bridge/src/multiplexer.rs b/node/network/bridge/src/multiplexer.rs index 0437ebeee786..b88cc4146959 100644 --- a/node/network/bridge/src/multiplexer.rs +++ b/node/network/bridge/src/multiplexer.rs @@ -172,7 +172,7 @@ fn multiplex_single( pending_response, )), Protocol::StatementFetching => { - panic!("Statement fetching requests are handled directly. qed."); + unreachable!("Statement fetching requests are handled directly. qed."); } }; Ok(r) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index c233b193156e..300344023be3 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -397,6 +397,38 @@ pub struct Overseer { // pub candidate_validation_config: CandidateValidationConfig, } + +// Additional `From` implementations, in order to deal with incoming network messages. +// Kept out of the proc macro, for sake of simplicity reduce the need to make even +// more types to the proc macro logic. + +use polkadot_node_network_protocol::{ + request_response::{request::IncomingRequest, v1 as req_res_v1, Requests}, + PeerId, UnifiedReputationChange, +}; + +impl From> for AllMessages { + fn from(req: IncomingRequest) -> Self { + From::::from(From::from(req)) + } +} +impl From> for AllMessages { + fn from(req: IncomingRequest) -> Self { + From::::from(From::from(req)) + } +} +impl From> for AllMessages { + fn from(req: IncomingRequest) -> Self { + From::::from(From::from(req)) + } +} +impl From> for AllMessages { + fn from(req: IncomingRequest) -> Self { + From::::from(From::from(req)) + } +} + + impl Overseer where SupportsParachains: HeadSupportsParachains, diff --git a/node/subsystem-types/src/messages.rs b/node/subsystem-types/src/messages.rs index 22c43eb68cf5..e820ab3569bb 100644 --- a/node/subsystem-types/src/messages.rs +++ b/node/subsystem-types/src/messages.rs @@ -801,3 +801,19 @@ pub enum ApprovalDistributionMessage { #[derive(Debug)] pub enum GossipSupportMessage { } + +impl From> for AvailabilityDistributionMessage { + fn from(req: IncomingRequest) -> Self { + Self::PoVFetchingRequest(req) + } +} +impl From> for AvailabilityDistributionMessage { + fn from(req: IncomingRequest) -> Self { + Self::ChunkFetchingRequest(req) + } +} +impl From> for CollatorProtocolMessage { + fn from(req: IncomingRequest) -> Self { + Self::CollationFetchingRequest(req) + } +} diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index f4d0ae84fae4..42c670f69d77 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -38,7 +38,10 @@ use polkadot_primitives::v1::{Hash, BlockNumber}; /// If there are greater than this number of slots, then we fall back to a heap vector. const ACTIVE_LEAVES_SMALLVEC_CAPACITY: usize = 8; -pub use polkadot_node_subsystem_types::errors::{self, *}; +pub use polkadot_node_subsystem_types::{ + errors::{self, *}, + ActivatedLeaf, +}; pub mod messages { pub use super::overseer::AllMessages; pub use polkadot_node_subsystem_types::messages::*; From 5c3377e0d597dd74d6ce34795e2c1bc97a930ad6 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 1 Jul 2021 17:54:01 +0200 Subject: [PATCH 080/161] fix bitfield dist --- node/core/bitfield-signing/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/core/bitfield-signing/src/lib.rs b/node/core/bitfield-signing/src/lib.rs index a36b2d8baa0b..4ac87cdd9989 100644 --- a/node/core/bitfield-signing/src/lib.rs +++ b/node/core/bitfield-signing/src/lib.rs @@ -299,7 +299,7 @@ impl JobTrait for BitfieldSigningJob { .send_message(BitfieldDistributionMessage::DistributeBitfield( relay_parent, signed_bitfield, - ).into()) + )) .await; Ok(()) From c19be05432375ec012e12afe07b535242cccff00 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 1 Jul 2021 17:54:15 +0200 Subject: [PATCH 081/161] split bitfield subsystem test --- node/core/bitfield-signing/src/lib.rs | 70 +------------------------ node/core/bitfield-signing/src/tests.rs | 67 +++++++++++++++++++++++ 2 files changed, 68 insertions(+), 69 deletions(-) create mode 100644 node/core/bitfield-signing/src/tests.rs diff --git a/node/core/bitfield-signing/src/lib.rs b/node/core/bitfield-signing/src/lib.rs index 4ac87cdd9989..4462b2d378c7 100644 --- a/node/core/bitfield-signing/src/lib.rs +++ b/node/core/bitfield-signing/src/lib.rs @@ -312,72 +312,4 @@ impl JobTrait for BitfieldSigningJob { pub type BitfieldSigningSubsystem = JobSubsystem; #[cfg(test)] -mod tests { - use super::*; - use futures::{pin_mut, executor::block_on}; - use polkadot_primitives::v1::{CandidateHash, OccupiedCore}; - use polkadot_node_subsystem::messages::AllMessages; - - fn occupied_core(para_id: u32, candidate_hash: CandidateHash) -> CoreState { - CoreState::Occupied(OccupiedCore { - group_responsible: para_id.into(), - next_up_on_available: None, - occupied_since: 100_u32, - time_out_at: 200_u32, - next_up_on_time_out: None, - availability: Default::default(), - candidate_hash, - candidate_descriptor: Default::default(), - }) - } - - #[test] - fn construct_availability_bitfield_works() { - block_on(async move { - let relay_parent = Hash::default(); - let validator_index = ValidatorIndex(1u32); - - let (mut sender, mut receiver) = polkadot_node_subsystem_test_helpers::sender_receiver(); - let future = construct_availability_bitfield( - relay_parent, - &jaeger::Span::Disabled, - validator_index, - &mut sender, - ).fuse(); - pin_mut!(future); - - let hash_a = CandidateHash(Hash::repeat_byte(1)); - let hash_b = CandidateHash(Hash::repeat_byte(2)); - - loop { - futures::select! { - m = receiver.next() => match m.unwrap() { - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(rp, RuntimeApiRequest::AvailabilityCores(tx)), - ) => { - assert_eq!(relay_parent, rp); - tx.send(Ok(vec![CoreState::Free, occupied_core(1, hash_a), occupied_core(2, hash_b)])).unwrap(); - } - AllMessages::AvailabilityStore( - AvailabilityStoreMessage::QueryChunkAvailability(c_hash, vidx, tx), - ) => { - assert_eq!(validator_index, vidx); - - tx.send(c_hash == hash_a).unwrap(); - }, - o => panic!("Unknown message: {:?}", o), - }, - r = future => match r { - Ok(r) => { - assert!(!r.0.get(0).unwrap()); - assert!(r.0.get(1).unwrap()); - assert!(!r.0.get(2).unwrap()); - break - }, - Err(e) => panic!("Failed: {:?}", e), - }, - } - } - }); - } -} +mod tests; diff --git a/node/core/bitfield-signing/src/tests.rs b/node/core/bitfield-signing/src/tests.rs new file mode 100644 index 000000000000..af76351ccfda --- /dev/null +++ b/node/core/bitfield-signing/src/tests.rs @@ -0,0 +1,67 @@ +use super::*; +use futures::{pin_mut, executor::block_on}; +use polkadot_primitives::v1::{CandidateHash, OccupiedCore}; +use polkadot_node_subsystem::messages::AllMessages; + +fn occupied_core(para_id: u32, candidate_hash: CandidateHash) -> CoreState { + CoreState::Occupied(OccupiedCore { + group_responsible: para_id.into(), + next_up_on_available: None, + occupied_since: 100_u32, + time_out_at: 200_u32, + next_up_on_time_out: None, + availability: Default::default(), + candidate_hash, + candidate_descriptor: Default::default(), + }) +} + +#[test] +fn construct_availability_bitfield_works() { + block_on(async move { + let relay_parent = Hash::default(); + let validator_index = ValidatorIndex(1u32); + + let (mut sender, mut receiver) = polkadot_node_subsystem_test_helpers::sender_receiver(); + let future = construct_availability_bitfield( + relay_parent, + &jaeger::Span::Disabled, + validator_index, + &mut sender, + ).fuse(); + pin_mut!(future); + + let hash_a = CandidateHash(Hash::repeat_byte(1)); + let hash_b = CandidateHash(Hash::repeat_byte(2)); + + loop { + futures::select! { + m = receiver.next() => match m.unwrap() { + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(rp, RuntimeApiRequest::AvailabilityCores(tx)), + ) => { + assert_eq!(relay_parent, rp); + tx.send(Ok(vec![CoreState::Free, occupied_core(1, hash_a), occupied_core(2, hash_b)])).unwrap(); + } + AllMessages::AvailabilityStore( + AvailabilityStoreMessage::QueryChunkAvailability(c_hash, vidx, tx), + ) => { + assert_eq!(validator_index, vidx); + + tx.send(c_hash == hash_a).unwrap(); + }, + o => panic!("Unknown message: {:?}", o), + }, + r = future => match r { + Ok(r) => { + assert!(!r.0.get(0).unwrap()); + assert!(r.0.get(1).unwrap()); + assert!(!r.0.get(2).unwrap()); + break + }, + Err(e) => panic!("Failed: {:?}", e), + }, + } + } + }); +} From 4fdebf84d140ee38aa68855921b9b5ee8455e34e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 1 Jul 2021 18:14:51 +0200 Subject: [PATCH 082/161] fixup collator protocol --- .../src/collator_side/mod.rs | 19 ++-- node/network/collator-protocol/src/lib.rs | 29 +++--- .../src/validator_side/mod.rs | 96 ++++++++++++------- 3 files changed, 91 insertions(+), 53 deletions(-) diff --git a/node/network/collator-protocol/src/collator_side/mod.rs b/node/network/collator-protocol/src/collator_side/mod.rs index 4cc52b33fde7..e7b46ab39e4f 100644 --- a/node/network/collator-protocol/src/collator_side/mod.rs +++ b/node/network/collator-protocol/src/collator_side/mod.rs @@ -917,13 +917,16 @@ async fn handle_our_view_change( } /// The collator protocol collator side main loop. -pub(crate) async fn run( - mut ctx: impl SubsystemContext, +pub(crate) async fn run( + mut ctx: Context, local_peer_id: PeerId, collator_pair: CollatorPair, metrics: Metrics, -) -> Result<()> { - use FromOverseer::*; +) -> Result<()> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext +{ use OverseerSignal::*; let mut state = State::new(local_peer_id, collator_pair, metrics); @@ -932,15 +935,15 @@ pub(crate) async fn run( loop { select! { msg = ctx.recv().fuse() => match msg.map_err(Fatal::SubsystemReceive)? { - Communication { msg } => { + FromOverseer::Communication { msg } => { log_error( process_msg(&mut ctx, &mut runtime, &mut state, msg).await, "Failed to process message" )?; }, - Signal(ActiveLeaves(_update)) => {} - Signal(BlockFinalized(..)) => {} - Signal(Conclude) => return Ok(()), + FromOverseer::Signal(ActiveLeaves(_update)) => {} + FromOverseer::Signal(BlockFinalized(..)) => {} + FromOverseer::Signal(Conclude) => return Ok(()), }, relay_parent = state.active_collation_fetches.select_next_some() => { let next = if let Some(waiting) = state.waiting_collation_fetches.get_mut(&relay_parent) { diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 78c255f85601..4794b335c0cc 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -28,16 +28,16 @@ use sp_keystore::SyncCryptoStorePtr; use polkadot_node_network_protocol::{PeerId, UnifiedReputationChange as Rep}; use polkadot_primitives::v1::CollatorPair; -use polkadot_subsystem::{ - messages::{CollatorProtocolMessage, NetworkBridgeMessage}, -}; -use polkadot_overseer::{ - AllMessages, +use polkadot_subsystem::{ OverseerSignal, - gen::{ - Subsystem, SpawnedSubsystem, OverseerError, SubsystemContext, - FromOverseer, + FromOverseer, + SpawnedSubsystem, + SubsystemContext, + SubsystemSender, + overseer::{self, OverseerError, }, + messages::{ + AllMessages, CollatorProtocolMessage, NetworkBridgeMessage, }, }; use polkadot_subsystem::{ @@ -106,7 +106,8 @@ impl CollatorProtocolSubsystem { async fn run(self, ctx: Context) -> Result<()> where - Context: SubsystemContext, + Context: overseer::SubsystemContext, + Context: SubsystemContext, { match self.protocol_side { ProtocolSide::Validator { keystore, eviction_policy, metrics } => validator_side::run( @@ -125,9 +126,11 @@ impl CollatorProtocolSubsystem { } } -impl Subsystem for CollatorProtocolSubsystem +impl overseer::Subsystem for CollatorProtocolSubsystem where - Context: SubsystemContext + Sync + Send, + Context: SubsystemContext, + Context: overseer::SubsystemContext, + ::Sender: SubsystemSender, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self @@ -154,7 +157,7 @@ where "reputation change for peer", ); - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message( NetworkBridgeMessage::ReportPeer(peer, rep), - )).await; + ).await; } diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs index 709855e353af..ed71449c8cdd 100644 --- a/node/network/collator-protocol/src/validator_side/mod.rs +++ b/node/network/collator-protocol/src/validator_side/mod.rs @@ -40,6 +40,7 @@ use polkadot_node_primitives::{SignedFullStatement, PoV}; use polkadot_node_subsystem_util::metrics::{self, prometheus}; use polkadot_primitives::v1::{CandidateReceipt, CollatorId, Hash, Id as ParaId}; use polkadot_subsystem::{ + overseer, jaeger, messages::{ AllMessages, CollatorProtocolMessage, IfDisconnected, @@ -583,19 +584,27 @@ fn collator_peer_id( ) } -async fn disconnect_peer(ctx: &mut impl SubsystemContext, peer_id: PeerId) { +async fn disconnect_peer(ctx: &mut Context, peer_id: PeerId) +where + Context: overseer::SubsystemContext, + Context: SubsystemContext, +{ ctx.send_message( - NetworkBridgeMessage::DisconnectPeer(peer_id, PeerSet::Collation).into() + NetworkBridgeMessage::DisconnectPeer(peer_id, PeerSet::Collation) ).await } /// Another subsystem has requested to fetch collations on a particular leaf for some para. -async fn fetch_collation( - ctx: &mut impl SubsystemContext, +async fn fetch_collation( + ctx: &mut Context, state: &mut State, pc: PendingCollation, id: CollatorId, -) { +) +where + Context: overseer::SubsystemContext, + Context: SubsystemContext, +{ let (tx, rx) = oneshot::channel(); let PendingCollation { relay_parent, para_id, peer_id, .. } = pc; @@ -627,7 +636,8 @@ async fn note_good_collation( id: CollatorId, ) where - Context: SubsystemContext + Context: overseer::SubsystemContext, + Context: SubsystemContext, { if let Some(peer_id) = collator_peer_id(peer_data, &id) { modify_reputation(ctx, peer_id, BENEFIT_NOTIFY_GOOD).await; @@ -635,19 +645,23 @@ where } /// Notify a collator that its collation got seconded. -async fn notify_collation_seconded( - ctx: &mut impl SubsystemContext, +async fn notify_collation_seconded( + ctx: &mut Context, peer_id: PeerId, relay_parent: Hash, statement: SignedFullStatement, -) { +) +where + Context: overseer::SubsystemContext, + Context: SubsystemContext, +{ let wire_message = protocol_v1::CollatorProtocolMessage::CollationSeconded(relay_parent, statement.into()); - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message( NetworkBridgeMessage::SendCollationMessage( vec![peer_id], protocol_v1::CollationProtocol::CollatorProtocol(wire_message), ) - )).await; + ).await; modify_reputation(ctx, peer_id, BENEFIT_NOTIFY_GOOD).await; } @@ -684,7 +698,8 @@ async fn request_collation( result: oneshot::Sender<(CandidateReceipt, PoV)>, ) where - Context: SubsystemContext + Context: overseer::SubsystemContext, + Context: SubsystemContext, { if !state.view.contains(&relay_parent) { tracing::debug!( @@ -737,8 +752,8 @@ where "Requesting collation", ); - ctx.send_message(AllMessages::NetworkBridge( - NetworkBridgeMessage::SendRequests(vec![requests], IfDisconnected::ImmediateError)) + ctx.send_message( + NetworkBridgeMessage::SendRequests(vec![requests], IfDisconnected::ImmediateError) ).await; } @@ -750,7 +765,8 @@ async fn process_incoming_peer_message( msg: protocol_v1::CollatorProtocolMessage, ) where - Context: SubsystemContext + Context: overseer::SubsystemContext, + Context: SubsystemContext, { use protocol_v1::CollatorProtocolMessage::*; use sp_runtime::traits::AppVerify; @@ -897,12 +913,16 @@ async fn remove_relay_parent( } /// Our view has changed. -async fn handle_our_view_change( - ctx: &mut impl SubsystemContext, +async fn handle_our_view_change( + ctx: &mut Context, state: &mut State, keystore: &SyncCryptoStorePtr, view: OurView, -) -> Result<()> { +) -> Result<()> +where + Context: overseer::SubsystemContext, + Context: SubsystemContext, +{ let old_view = std::mem::replace(&mut state.view, view); let added: HashMap> = state.view @@ -955,7 +975,8 @@ async fn handle_network_msg( bridge_message: NetworkBridgeEvent, ) -> Result<()> where - Context: SubsystemContext + Context: overseer::SubsystemContext, + Context: SubsystemContext, { use NetworkBridgeEvent::*; @@ -993,7 +1014,8 @@ async fn process_msg( state: &mut State, ) where - Context: SubsystemContext + Context: overseer::SubsystemContext, + Context: SubsystemContext, { use CollatorProtocolMessage::*; @@ -1101,9 +1123,10 @@ pub(crate) async fn run( eviction_policy: crate::CollatorEvictionPolicy, metrics: Metrics, ) -> Result<()> - where Context: SubsystemContext +where + Context: overseer::SubsystemContext, + Context: SubsystemContext, { - use FromOverseer::*; use OverseerSignal::*; let mut state = State { @@ -1122,7 +1145,7 @@ pub(crate) async fn run( select! { res = ctx.recv().fuse() => { match res { - Ok(Communication { msg }) => { + Ok(FromOverseer::Communication { msg }) => { tracing::trace!(target: LOG_TARGET, msg = ?msg, "received a message"); process_msg( &mut ctx, @@ -1131,7 +1154,7 @@ pub(crate) async fn run( &mut state, ).await; } - Ok(Signal(Conclude)) => break, + Ok(FromOverseer::Signal(Conclude)) => break, _ => {}, } } @@ -1161,11 +1184,15 @@ pub(crate) async fn run( } /// Handle a fetched collation result. -async fn handle_collation_fetched_result( - ctx: &mut impl SubsystemContext, +async fn handle_collation_fetched_result( + ctx: &mut Context, state: &mut State, (mut collation_event, res): PendingCollationFetch, -) { +) +where + Context: overseer::SubsystemContext, + Context: SubsystemContext, +{ // If no prior collation for this relay parent has been seconded, then // memoize the collation_event for that relay_parent, such that we may // notify the collator of their successful second backing @@ -1206,7 +1233,7 @@ async fn handle_collation_fetched_result( relay_parent.clone(), candidate_receipt, pov, - ).into() + ) ).await; entry.insert(collation_event); @@ -1223,11 +1250,15 @@ async fn handle_collation_fetched_result( // This issues `NetworkBridge` notifications to disconnect from all inactive peers at the // earliest possible point. This does not yet clean up any metadata, as that will be done upon // receipt of the `PeerDisconnected` event. -async fn disconnect_inactive_peers( - ctx: &mut impl SubsystemContext, +async fn disconnect_inactive_peers( + ctx: &mut Context, eviction_policy: &crate::CollatorEvictionPolicy, peers: &HashMap, -) { +) +where + Context: overseer::SubsystemContext, + Context: SubsystemContext, +{ for (peer, peer_data) in peers { if peer_data.is_inactive(&eviction_policy) { disconnect_peer(ctx, peer.clone()).await; @@ -1250,7 +1281,8 @@ async fn poll_collation_response( ) -> bool where - Context: SubsystemContext + Context: overseer::SubsystemContext, + Context: SubsystemContext, { if never!(per_req.from_collator.is_terminated()) { tracing::error!( From 15bc81471e06545f3b4d60706d770e9e0fa09f27 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 1 Jul 2021 18:21:28 +0200 Subject: [PATCH 083/161] fixup gossip support --- node/network/gossip-support/src/lib.rs | 79 +++++++++++++++++--------- 1 file changed, 53 insertions(+), 26 deletions(-) diff --git a/node/network/gossip-support/src/lib.rs b/node/network/gossip-support/src/lib.rs index 0818ffaddfe8..207d554161b7 100644 --- a/node/network/gossip-support/src/lib.rs +++ b/node/network/gossip-support/src/lib.rs @@ -29,14 +29,18 @@ use futures::{channel::oneshot, FutureExt as _}; use rand::{SeedableRng, seq::SliceRandom as _}; use rand_chacha::ChaCha20Rng; use polkadot_node_subsystem::{ + overseer, + SubsystemError, + FromOverseer, SpawnedSubsystem, SubsystemContext, messages::{ - GossipSupportMessage, NetworkBridgeMessage, + AllMessages, + GossipSupportMessage, + NetworkBridgeMessage, + RuntimeApiMessage, + RuntimeApiRequest, }, ActiveLeavesUpdate, OverseerSignal, }; -use polkadot_overseer::gen::{ - FromOverseer, AllMessages, Subsystem, SpawnedSubsystem, SubsystemContext, -}; use polkadot_node_subsystem_util as util; use polkadot_primitives::v1::{ Hash, SessionIndex, AuthorityDiscoveryId, @@ -78,6 +82,7 @@ impl GossipSupport { async fn run(self, ctx: Context) where Context: SubsystemContext, + Context: overseer::SubsystemContext, { let mut state = State::default(); self.run_inner(ctx, &mut state).await; @@ -86,6 +91,7 @@ impl GossipSupport { async fn run_inner(self, mut ctx: Context, state: &mut State) where Context: SubsystemContext, + Context: overseer::SubsystemContext, { let Self { keystore } = self; loop { @@ -122,10 +128,14 @@ impl GossipSupport { } } -async fn determine_relevant_authorities( - ctx: &mut impl SubsystemContext, +async fn determine_relevant_authorities( + ctx: &mut Context, relay_parent: Hash, -) -> Result, util::Error> { +) -> Result, util::Error> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let authorities = util::request_authorities(relay_parent, ctx.sender()).await.await??; tracing::debug!( target: LOG_TARGET, @@ -153,19 +163,23 @@ async fn ensure_i_am_an_authority( } /// A helper function for making a `ConnectToValidators` request. -async fn connect_to_authorities( - ctx: &mut impl SubsystemContext, +async fn connect_to_authorities( + ctx: &mut Context, validator_ids: Vec, peer_set: PeerSet, -) -> oneshot::Receiver { +) -> oneshot::Receiver +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let (failed, failed_rx) = oneshot::channel(); - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message( NetworkBridgeMessage::ConnectToValidators { validator_ids, peer_set, failed, } - )).await; + ).await; failed_rx } @@ -177,12 +191,16 @@ async fn connect_to_authorities( /// This limits the amount of gossip peers to 2 * sqrt(len) and ensures the diameter of 2. /// /// [web3]: https://research.web3.foundation/en/latest/polkadot/networking/3-avail-valid.html#topology -async fn update_gossip_topology( - ctx: &mut impl SubsystemContext, +async fn update_gossip_topology( + ctx: &mut Context, our_index: usize, authorities: Vec, relay_parent: Hash, -) -> Result<(), util::Error> { +) -> Result<(), util::Error> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ // retrieve BABE randomness let random_seed = { let (tx, rx) = oneshot::channel(); @@ -190,7 +208,7 @@ async fn update_gossip_topology( ctx.send_message(RuntimeApiMessage::Request( relay_parent, RuntimeApiRequest::CurrentBabeEpoch(tx), - ).into()).await; + )).await; let randomness = rx.await??.randomness; let mut subject = [0u8; 40]; @@ -211,11 +229,11 @@ async fn update_gossip_topology( let neighbors = matrix_neighbors(our_shuffled_position, len); let our_neighbors = neighbors.map(|i| authorities[indices[i]].clone()).collect(); - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message( NetworkBridgeMessage::NewGossipTopology { our_neighbors, } - )).await; + ).await; Ok(()) } @@ -246,12 +264,16 @@ impl State { /// 1. Determine if the current session index has changed. /// 2. If it has, determine relevant validators /// and issue a connection request. - async fn handle_active_leaves( + async fn handle_active_leaves( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut Context, keystore: &SyncCryptoStorePtr, leaves: impl Iterator, - ) -> Result<(), util::Error> { + ) -> Result<(), util::Error> + where + Context: SubsystemContext, + Context: overseer::SubsystemContext, + { for leaf in leaves { let current_index = util::request_session_index_for_child(leaf, ctx.sender()).await.await??; let since_failure = self.last_failure.map(|i| i.elapsed()).unwrap_or_default(); @@ -294,11 +316,15 @@ impl State { Ok(()) } - async fn issue_connection_request( + async fn issue_connection_request( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut Context, authorities: Vec, - ) -> Result<(), util::Error> { + ) -> Result<(), util::Error> + where + Context: SubsystemContext, + Context: overseer::SubsystemContext, + { let num = authorities.len(); tracing::debug!(target: LOG_TARGET, %num, "Issuing a connection request"); @@ -324,9 +350,10 @@ impl State { } } -impl Subsystem for GossipSupport +impl overseer::Subsystem for GossipSupport where - Context: SubsystemContext + Sync + Send, + Context: SubsystemContext, + Context: overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx) From d33c72db74019fd93f7a8f77f8ddc6dd57547350 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 1 Jul 2021 18:25:01 +0200 Subject: [PATCH 084/161] fixup candidate validation --- node/core/candidate-validation/src/lib.rs | 58 ++++++++++++++++------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index d58cd7b76e08..b46fb487c971 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -85,10 +85,12 @@ impl CandidateValidationSubsystem { } } -impl overseer::Subsystem for CandidateValidationSubsystem where - C: SubsystemContext, +impl overseer::Subsystem for CandidateValidationSubsystem +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, { - fn start(self, ctx: C) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run(ctx, self.metrics, self.config.artifacts_cache_path, self.config.program_path) .map_err(|e| SubsystemError::with_origin("candidate-validation", e)) .boxed(); @@ -99,12 +101,16 @@ impl overseer::Subsystem for CandidateValidationSubsystem } } -async fn run( - mut ctx: impl SubsystemContext, +async fn run( + mut ctx: Context, metrics: Metrics, cache_path: PathBuf, program_path: PathBuf, -) -> SubsystemResult<()> { +) -> SubsystemResult<()> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let (mut validation_host, task) = polkadot_node_core_pvf::start( polkadot_node_core_pvf::Config::new(cache_path, program_path), ); @@ -175,12 +181,16 @@ async fn run( } } -async fn runtime_api_request( - ctx: &mut impl SubsystemContext, +async fn runtime_api_request( + ctx: &mut Context, relay_parent: Hash, request: RuntimeApiRequest, receiver: oneshot::Receiver>, -) -> SubsystemResult> { +) -> SubsystemResult> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ ctx.send_message( AllMessages::RuntimeApiMessage(RuntimeApiMessage::Request( relay_parent, @@ -198,11 +208,15 @@ enum AssumptionCheckOutcome { BadRequest, } -async fn check_assumption_validation_data( - ctx: &mut impl SubsystemContext, +async fn check_assumption_validation_data( + ctx: &mut Context, descriptor: &CandidateDescriptor, assumption: OccupiedCoreAssumption, -) -> SubsystemResult { +) -> SubsystemResult +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let validation_data = { let (tx, rx) = oneshot::channel(); let d = runtime_api_request( @@ -248,10 +262,14 @@ async fn check_assumption_validation_data( }) } -async fn find_assumed_validation_data( - ctx: &mut impl SubsystemContext, +async fn find_assumed_validation_data( + ctx: &mut Context, descriptor: &CandidateDescriptor, -) -> SubsystemResult { +) -> SubsystemResult +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ // The candidate descriptor has a `persisted_validation_data_hash` which corresponds to // one of up to two possible values that we can derive from the state of the // relay-parent. We can fetch these values by getting the persisted validation data @@ -279,13 +297,17 @@ async fn find_assumed_validation_data( Ok(AssumptionCheckOutcome::DoesNotMatch) } -async fn spawn_validate_from_chain_state( - ctx: &mut impl SubsystemContext, +async fn spawn_validate_from_chain_state( + ctx: &mut Context, validation_host: &mut ValidationHost, descriptor: CandidateDescriptor, pov: Arc, metrics: &Metrics, -) -> SubsystemResult> { +) -> SubsystemResult> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let (validation_data, validation_code) = match find_assumed_validation_data(ctx, &descriptor).await? { AssumptionCheckOutcome::Matches(validation_data, validation_code) => { From 15672d3c880cc8141e3d9ef28fbd1bb78545b638 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 1 Jul 2021 18:27:54 +0200 Subject: [PATCH 085/161] yikes and quirks --- node/core/av-store/src/lib.rs | 4 +++- node/core/runtime-api/src/lib.rs | 2 +- node/network/bitfield-distribution/src/lib.rs | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index 4c646f5e2d5b..51f54177a220 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -37,7 +37,9 @@ use polkadot_node_primitives::{ ErasureChunk, AvailableData, }; use polkadot_subsystem::{ - FromOverseer, OverseerSignal, SubsystemError, Subsystem, SubsystemContext, SpawnedSubsystem, + FromOverseer, OverseerSignal, SubsystemError, + SubsystemContext, SpawnedSubsystem, + overseer::{self, Subsystem}, ActiveLeavesUpdate, errors::{ChainApiError, RuntimeApiError}, }; diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index 839cbfa3938c..42af77efc904 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -23,7 +23,7 @@ #![warn(missing_docs)] use polkadot_subsystem::{ - Subsystem, SpawnedSubsystem, SubsystemResult, SubsystemContext, + overseer{self, Subsystem}, SpawnedSubsystem, SubsystemResult, SubsystemContext, FromOverseer, OverseerSignal, messages::{ RuntimeApiMessage, RuntimeApiRequest as Request, diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index 728ace3cf649..d9279d668ce8 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -26,9 +26,10 @@ use futures::{channel::oneshot, FutureExt}; use polkadot_subsystem::messages::*; use polkadot_subsystem::{ - PerLeafSpan, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, Subsystem, + PerLeafSpan, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemResult, jaeger, + overseer::{self, Subsystem,}, }; use polkadot_node_subsystem_util::{ metrics::{self, prometheus}, From 372ebcbffc19dd3b4ca124f92ac8c7944041493d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 2 Jul 2021 11:36:43 +0200 Subject: [PATCH 086/161] fixup availability recovery --- node/network/availability-recovery/src/lib.rs | 57 ++++++++++++------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index efd8d3712851..5dd3a16aa2e9 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -34,7 +34,8 @@ use polkadot_primitives::v1::{ }; use polkadot_node_primitives::{ErasureChunk, AvailableData}; use polkadot_subsystem::{ - SubsystemContext, SubsystemResult, SubsystemError, Subsystem, SpawnedSubsystem, FromOverseer, + overseer::{self, Subsystem}, + SubsystemContext, SubsystemResult, SubsystemError, SpawnedSubsystem, FromOverseer, OverseerSignal, ActiveLeavesUpdate, SubsystemSender, errors::RecoveryError, jaeger, @@ -573,10 +574,12 @@ impl Default for State { } } -impl Subsystem for AvailabilityRecoverySubsystem - where C: SubsystemContext +impl Subsystem for AvailabilityRecoverySubsystem +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, { - fn start(self, ctx: C) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx) .map_err(|e| SubsystemError::with_origin("availability-recovery", e)) .boxed(); @@ -609,14 +612,18 @@ async fn handle_signal( } /// Machinery around launching interactions into the background. -async fn launch_interaction( +async fn launch_interaction( state: &mut State, - ctx: &mut impl SubsystemContext, + ctx: &mut Context, session_info: SessionInfo, receipt: CandidateReceipt, backing_group: Option, response_sender: oneshot::Sender>, -) -> error::Result<()> { +) -> error::Result<()> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let candidate_hash = receipt.hash(); let params = InteractionParams { @@ -650,7 +657,7 @@ async fn launch_interaction( awaiting: vec![response_sender], }); - if let Err(e) = ctx.spawn("recovery interaction", Box::pin(remote)) { + if let Err(e) = ctx.spawn("recovery interaction", Box::pin(remote)).await { tracing::warn!( target: LOG_TARGET, err = ?e, @@ -662,14 +669,18 @@ async fn launch_interaction( } /// Handles an availability recovery request. -async fn handle_recover( +async fn handle_recover( state: &mut State, - ctx: &mut impl SubsystemContext, + ctx: &mut Context, receipt: CandidateReceipt, session_index: SessionIndex, backing_group: Option, response_sender: oneshot::Sender>, -) -> error::Result<()> { +) -> error::Result<()> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let candidate_hash = receipt.hash(); let span = jaeger::Span::new(candidate_hash, "availbility-recovery") @@ -724,14 +735,18 @@ async fn handle_recover( } /// Queries a chunk from av-store. -async fn query_full_data( - ctx: &mut impl SubsystemContext, +async fn query_full_data( + ctx: &mut Context, candidate_hash: CandidateHash, -) -> error::Result> { +) -> error::Result> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let (tx, rx) = oneshot::channel(); - ctx.send_message(AllMessages::AvailabilityStore( + ctx.send_message( AvailabilityStoreMessage::QueryAvailableData(candidate_hash, tx), - )).await; + ).await; Ok(rx.await.map_err(error::Error::CanceledQueryFullData)?) } @@ -747,10 +762,14 @@ impl AvailabilityRecoverySubsystem { Self { fast_path: false } } - async fn run( + async fn run( self, - mut ctx: impl SubsystemContext, - ) -> SubsystemResult<()> { + mut ctx: Context, + ) -> SubsystemResult<()> + where + Context: SubsystemContext, + Context: overseer::SubsystemContext, + { let mut state = State::default(); loop { From 821024bd4fb8fa7603f59bdb7c83209a0836ff0c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 2 Jul 2021 11:50:03 +0200 Subject: [PATCH 087/161] fixup av-store --- node/core/av-store/src/lib.rs | 47 +++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index 51f54177a220..50bb7de5ab5c 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -524,9 +524,10 @@ impl KnownUnfinalizedBlocks { } } -impl Subsystem for AvailabilityStoreSubsystem +impl overseer::Subsystem for AvailabilityStoreSubsystem where Context: SubsystemContext, + Context: overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run(self, ctx) @@ -542,7 +543,8 @@ where async fn run(mut subsystem: AvailabilityStoreSubsystem, mut ctx: Context) where - Context: SubsystemContext, + Context: SubsystemContext, + Context: overseer::SubsystemContext, { let mut next_pruning = Delay::new(subsystem.pruning_config.pruning_interval).fuse(); @@ -572,7 +574,8 @@ async fn run_iteration( ) -> Result where - Context: SubsystemContext, + Context: SubsystemContext, + Context: overseer::SubsystemContext, { select! { incoming = ctx.recv().fuse() => { @@ -617,18 +620,22 @@ where Ok(false) } -async fn process_block_activated( - ctx: &mut impl SubsystemContext, +async fn process_block_activated( + ctx: &mut Context, subsystem: &mut AvailabilityStoreSubsystem, activated: Hash, -) -> Result<(), Error> { +) -> Result<(), Error> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let now = subsystem.clock.now()?; let block_header = { let (tx, rx) = oneshot::channel(); ctx.send_message( - ChainApiMessage::BlockHeader(activated, tx).into() + ChainApiMessage::BlockHeader(activated, tx) ).await; match rx.await?? { @@ -668,8 +675,8 @@ async fn process_block_activated( Ok(()) } -async fn process_new_head( - ctx: &mut impl SubsystemContext, +async fn process_new_head( + ctx: &mut Context, db: &Arc, db_transaction: &mut DBTransaction, config: &Config, @@ -677,12 +684,16 @@ async fn process_new_head( now: Duration, hash: Hash, header: Header, -) -> Result<(), Error> { +) -> Result<(), Error> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let candidate_events = { let (tx, rx) = oneshot::channel(); ctx.send_message( - RuntimeApiMessage::Request(hash, RuntimeApiRequest::CandidateEvents(tx)).into() + RuntimeApiMessage::Request(hash, RuntimeApiRequest::CandidateEvents(tx)) ).await; rx.await?? @@ -693,7 +704,7 @@ async fn process_new_head( let n_validators = { let (tx, rx) = oneshot::channel(); ctx.send_message( - RuntimeApiMessage::Request(header.parent_hash, RuntimeApiRequest::Validators(tx)).into() + RuntimeApiMessage::Request(header.parent_hash, RuntimeApiRequest::Validators(tx)) ).await; rx.await??.len() @@ -837,12 +848,16 @@ macro_rules! peek_num { } } -async fn process_block_finalized( - ctx: &mut impl SubsystemContext, +async fn process_block_finalized( + ctx: &mut Context, subsystem: &AvailabilityStoreSubsystem, finalized_hash: Hash, finalized_number: BlockNumber, -) -> Result<(), Error> { +) -> Result<(), Error> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let now = subsystem.clock.now()?; let mut next_possible_batch = 0; @@ -871,7 +886,7 @@ async fn process_block_finalized( finalized_hash } else { let (tx, rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::FinalizedBlockHash(batch_num, tx).into()).await; + ctx.send_message(ChainApiMessage::FinalizedBlockHash(batch_num, tx)).await; match rx.await?? { None => { From 12d5dd333ea6e13ec2625fa8ecf4f15d0d522226 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 2 Jul 2021 12:09:35 +0200 Subject: [PATCH 088/161] collator and runtime msg --- .../src/collator_side/mod.rs | 119 ++++++++++++------ node/subsystem-util/src/lib.rs | 8 +- node/subsystem-util/src/runtime/mod.rs | 45 +++---- 3 files changed, 100 insertions(+), 72 deletions(-) diff --git a/node/network/collator-protocol/src/collator_side/mod.rs b/node/network/collator-protocol/src/collator_side/mod.rs index e7b46ab39e4f..895999e28782 100644 --- a/node/network/collator-protocol/src/collator_side/mod.rs +++ b/node/network/collator-protocol/src/collator_side/mod.rs @@ -24,6 +24,7 @@ use polkadot_primitives::v1::{ Id as ParaId, }; use polkadot_subsystem::{ + overseer, FromOverseer, OverseerSignal, PerLeafSpan, SubsystemContext, jaeger, messages::{ AllMessages, CollatorProtocolMessage, NetworkBridgeEvent, NetworkBridgeMessage, @@ -301,15 +302,19 @@ impl State { /// or the relay-parent isn't in the active-leaves set, we ignore the message /// as it must be invalid in that case - although this indicates a logic error /// elsewhere in the node. -async fn distribute_collation( - ctx: &mut impl SubsystemContext, +async fn distribute_collation( + ctx: &mut Context, runtime: &mut RuntimeInfo, state: &mut State, id: ParaId, receipt: CandidateReceipt, pov: PoV, result_sender: Option>, -) -> Result<()> { +) -> Result<()> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let relay_parent = receipt.descriptor.relay_parent; // This collation is not in the active-leaves set. @@ -400,11 +405,15 @@ async fn distribute_collation( /// Get the Id of the Core that is assigned to the para being collated on if any /// and the total number of cores. -async fn determine_core( - ctx: &mut impl SubsystemContext, +async fn determine_core( + ctx: &mut Context, para_id: ParaId, relay_parent: Hash, -) -> Result> { +) -> Result> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let cores = get_availability_cores(ctx, relay_parent).await?; for (idx, core) in cores.iter().enumerate() { @@ -430,13 +439,17 @@ struct GroupValidators { /// Figure out current and next group of validators assigned to the para being collated on. /// /// Returns [`ValidatorId`]'s of current and next group as determined based on the `relay_parent`. -async fn determine_our_validators( - ctx: &mut impl SubsystemContext, +async fn determine_our_validators( + ctx: &mut Context, runtime: &mut RuntimeInfo, core_index: CoreIndex, cores: usize, relay_parent: Hash, -) -> Result<(GroupValidators, GroupValidators)> { +) -> Result<(GroupValidators, GroupValidators)> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let session_index = runtime.get_session_index(ctx, relay_parent).await?; let info = &runtime.get_session_info_by_index(ctx, relay_parent, session_index) .await? @@ -469,11 +482,15 @@ async fn determine_our_validators( } /// Issue a `Declare` collation message to the given `peer`. -async fn declare( - ctx: &mut impl SubsystemContext, +async fn declare( + ctx: &mut Context, state: &mut State, peer: PeerId, -) { +) +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let declare_signature_payload = protocol_v1::declare_signature_payload(&state.local_peer_id); if let Some(para_id) = state.collating_on { @@ -483,39 +500,47 @@ async fn declare( state.collator_pair.sign(&declare_signature_payload), ); - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message( NetworkBridgeMessage::SendCollationMessage( vec![peer], protocol_v1::CollationProtocol::CollatorProtocol(wire_message), ) - )).await; + ).await; } } /// Issue a connection request to a set of validators and /// revoke the previous connection request. -async fn connect_to_validators( - ctx: &mut impl SubsystemContext, +async fn connect_to_validators( + ctx: &mut Context, validator_ids: Vec, -) { +) +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ // ignore address resolution failure // will reissue a new request on new collation let (failed, _) = oneshot::channel(); - ctx.send_message(AllMessages::NetworkBridge(NetworkBridgeMessage::ConnectToValidators { + ctx.send_message(NetworkBridgeMessage::ConnectToValidators { validator_ids, peer_set: PeerSet::Collation, failed, - })).await; + }).await; } /// Advertise collation to the given `peer`. /// /// This will only advertise a collation if there exists one for the given `relay_parent` and the given `peer` is /// set as validator for our para at the given `relay_parent`. -async fn advertise_collation( - ctx: &mut impl SubsystemContext, +async fn advertise_collation( + ctx: &mut Context, state: &mut State, relay_parent: Hash, peer: PeerId, -) { +) +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let should_advertise = state.our_validators_groups .get(&relay_parent) .map(|g| g.should_advertise_to(&state.peer_ids, &peer)) @@ -555,12 +580,12 @@ async fn advertise_collation( relay_parent, ); - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message( NetworkBridgeMessage::SendCollationMessage( vec![peer.clone()], protocol_v1::CollationProtocol::CollatorProtocol(wire_message), ) - )).await; + ).await; if let Some(validators) = state.our_validators_groups.get_mut(&relay_parent) { validators.advertised_to_peer(&state.peer_ids, &peer); @@ -570,12 +595,16 @@ async fn advertise_collation( } /// The main incoming message dispatching switch. -async fn process_msg( - ctx: &mut impl SubsystemContext, +async fn process_msg( + ctx: &mut Context, runtime: &mut RuntimeInfo, state: &mut State, msg: CollatorProtocolMessage, -) -> Result<()> { +) -> Result<()> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ use CollatorProtocolMessage::*; let _timer = state.metrics.time_process_msg(); @@ -718,13 +747,17 @@ async fn send_collation( } /// A networking messages switch. -async fn handle_incoming_peer_message( - ctx: &mut impl SubsystemContext, +async fn handle_incoming_peer_message( + ctx: &mut Context, runtime: &mut RuntimeInfo, state: &mut State, origin: PeerId, msg: protocol_v1::CollatorProtocolMessage, -) -> Result<()> { +) -> Result<()> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ use protocol_v1::CollatorProtocolMessage::*; match msg { @@ -737,7 +770,7 @@ async fn handle_incoming_peer_message( // If we are declared to, this is another collator, and we should disconnect. ctx.send_message( - NetworkBridgeMessage::DisconnectPeer(origin, PeerSet::Collation).into() + NetworkBridgeMessage::DisconnectPeer(origin, PeerSet::Collation) ).await; } AdvertiseCollation(_) => { @@ -748,12 +781,12 @@ async fn handle_incoming_peer_message( ); ctx.send_message( - NetworkBridgeMessage::ReportPeer(origin.clone(), COST_UNEXPECTED_MESSAGE).into() + NetworkBridgeMessage::ReportPeer(origin.clone(), COST_UNEXPECTED_MESSAGE) ).await; // If we are advertised to, this is another collator, and we should disconnect. ctx.send_message( - NetworkBridgeMessage::DisconnectPeer(origin, PeerSet::Collation).into() + NetworkBridgeMessage::DisconnectPeer(origin, PeerSet::Collation) ).await; } CollationSeconded(relay_parent, statement) => { @@ -789,12 +822,16 @@ async fn handle_incoming_peer_message( } /// Our view has changed. -async fn handle_peer_view_change( - ctx: &mut impl SubsystemContext, +async fn handle_peer_view_change( + ctx: &mut Context, state: &mut State, peer_id: PeerId, view: View, -) { +) +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ let current = state.peer_views.entry(peer_id.clone()).or_default(); let added: Vec = view.difference(&*current).cloned().collect(); @@ -807,12 +844,16 @@ async fn handle_peer_view_change( } /// Bridge messages switch. -async fn handle_network_msg( - ctx: &mut impl SubsystemContext, +async fn handle_network_msg( + ctx: &mut Context, runtime: &mut RuntimeInfo, state: &mut State, bridge_message: NetworkBridgeEvent, -) -> Result<()> { +) -> Result<()> +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, +{ use NetworkBridgeEvent::*; match bridge_message { diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 4999f2629e19..7d361d88de04 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -181,17 +181,15 @@ macro_rules! specialize_requests { #[doc = "Request `"] #[doc = $doc_name] #[doc = "` from the runtime"] - pub async fn $func_name ( + pub async fn $func_name ( parent: Hash, $( $param_name: $param_ty, )* - sender: &mut Sender, + sender: &mut impl SubsystemSender, ) -> RuntimeApiReceiver<$return_ty> - where - Sender: SubsystemSender, { - request_from_runtime::<_, _, Sender>(parent, sender, |tx| RuntimeApiRequest::$request_variant( + request_from_runtime(parent, sender, |tx| RuntimeApiRequest::$request_variant( $( $param_name, )* tx )).await } diff --git a/node/subsystem-util/src/runtime/mod.rs b/node/subsystem-util/src/runtime/mod.rs index 4982b52d96f0..8bed1a2c0f43 100644 --- a/node/subsystem-util/src/runtime/mod.rs +++ b/node/subsystem-util/src/runtime/mod.rs @@ -91,14 +91,13 @@ impl RuntimeInfo { } /// Retrieve the current session index. - pub async fn get_session_index( + pub async fn get_session_index( &mut self, ctx: &mut Context, parent: Hash, ) -> Result where - M: From, - Context: SubsystemContext, + Context: SubsystemContext, { match self.session_index_cache.get(&parent) { Some(index) => Ok(*index), @@ -113,16 +112,13 @@ impl RuntimeInfo { } /// Get `ExtendedSessionInfo` by relay parent hash. - pub async fn get_session_info<'a, Context, M>( + pub async fn get_session_info<'a>( &'a mut self, - ctx: &mut Context, + ctx: &mut impl SubsystemContext, parent: Hash, ) -> Result<&'a ExtendedSessionInfo> - where - M: From, - Context: SubsystemContext, { - let session_index = self.get_session_index::<_,M>(ctx, parent).await?; + let session_index = self.get_session_index(ctx, parent).await?; self.get_session_info_by_index(ctx, parent, session_index).await } @@ -131,15 +127,12 @@ impl RuntimeInfo { /// /// `request_session_info` still requires the parent to be passed in, so we take the parent /// in addition to the `SessionIndex`. - pub async fn get_session_info_by_index<'a, Context, M>( + pub async fn get_session_info_by_index<'a>( &'a mut self, - ctx: &mut Context, + ctx: &mut impl SubsystemContext, parent: Hash, session_index: SessionIndex, ) -> Result<&'a ExtendedSessionInfo> - where - M: From, - Context: SubsystemContext, { if !self.session_info_cache.contains(&session_index) { let session_info = @@ -162,20 +155,19 @@ impl RuntimeInfo { } /// Convenience function for checking the signature of something signed. - pub async fn check_signature( + pub async fn check_signature( &mut self, ctx: &mut Context, parent: Hash, signed: UncheckedSigned, ) -> Result, UncheckedSigned>> where - M: From, - Context: SubsystemContext, + Context: SubsystemContext, Payload: EncodeAs + Clone, RealPayload: Encode + Clone, { - let session_index = self.get_session_index::<_,M>(ctx, parent).await?; - let info = self.get_session_info_by_index::<_,M>(ctx, parent, session_index).await?; + let session_index = self.get_session_index(ctx, parent).await?; + let info = self.get_session_info_by_index(ctx, parent, session_index).await?; Ok(check_signature(session_index, &info.session_info, parent, signed)) } @@ -251,23 +243,21 @@ where } /// Request availability cores from the runtime. -pub async fn get_availability_cores(ctx: &mut Context, relay_parent: Hash) +pub async fn get_availability_cores(ctx: &mut Context, relay_parent: Hash) -> Result> where - M: From, - Context: SubsystemContext, + Context: SubsystemContext, { recv_runtime(request_availability_cores(relay_parent, ctx.sender()).await).await } /// Variant of `request_availability_cores` that only returns occupied ones. -pub async fn get_occupied_cores( +pub async fn get_occupied_cores( ctx: &mut Context, relay_parent: Hash, ) -> Result> where -M: From, -Context: SubsystemContext, + Context: SubsystemContext, { let cores = get_availability_cores(ctx, relay_parent).await?; @@ -285,11 +275,10 @@ Context: SubsystemContext, } /// Get group rotation info based on the given relay_parent. -pub async fn get_group_rotation_info(ctx: &mut Context, relay_parent: Hash) +pub async fn get_group_rotation_info(ctx: &mut Context, relay_parent: Hash) -> Result where - M: From, - Context: SubsystemContext, + Context: SubsystemContext, { // We drop `groups` here as we don't need them, because of `RuntimeInfo`. Ideally we would not // fetch them in the first place. From 868b6121c159759034403b717044abbe7bfe5786 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 2 Jul 2021 12:16:54 +0200 Subject: [PATCH 089/161] fixup runtime API --- node/core/runtime-api/src/lib.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index 42af77efc904..f0844875d224 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -23,12 +23,14 @@ #![warn(missing_docs)] use polkadot_subsystem::{ - overseer{self, Subsystem}, SpawnedSubsystem, SubsystemResult, SubsystemContext, - FromOverseer, OverseerSignal, + SubsystemError, SubsystemResult, + FromOverseer, OverseerSignal, SpawnedSubsystem, + SubsystemContext, + errors::RuntimeApiError, messages::{ - RuntimeApiMessage, RuntimeApiRequest as Request, + AllMessages, RuntimeApiMessage, RuntimeApiRequest as Request, }, - errors::RuntimeApiError, + overseer, }; use polkadot_node_subsystem_util::metrics::{self, prometheus}; use polkadot_primitives::v1::{Block, BlockId, Hash, ParachainHost}; @@ -85,10 +87,11 @@ impl RuntimeApiSubsystem { } } -impl Subsystem for RuntimeApiSubsystem where +impl overseer::Subsystem for RuntimeApiSubsystem where Client: ProvideRuntimeApi + Send + 'static + Sync, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, - Context: SubsystemContext + Context: SubsystemContext, + Context: overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { SpawnedSubsystem { @@ -265,12 +268,14 @@ impl RuntimeApiSubsystem where } } -async fn run( - mut ctx: impl SubsystemContext, +async fn run( + mut ctx: Context, mut subsystem: RuntimeApiSubsystem, ) -> SubsystemResult<()> where Client: ProvideRuntimeApi + Send + Sync + 'static, Client::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, + Context: SubsystemContext, + Context: overseer::SubsystemContext, { loop { select! { From 686bb7f1a69be95f3631f58ab4937762be06d165 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 2 Jul 2021 12:22:54 +0200 Subject: [PATCH 090/161] fixup availabilty distribution --- node/network/availability-distribution/src/lib.rs | 13 ++++++++----- .../src/pov_requester/mod.rs | 4 ++-- .../src/requester/fetch_task/mod.rs | 6 +++--- .../availability-distribution/src/requester/mod.rs | 1 + .../availability-distribution/src/responder.rs | 8 ++++---- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/node/network/availability-distribution/src/lib.rs b/node/network/availability-distribution/src/lib.rs index 65580ea41d88..b42024472791 100644 --- a/node/network/availability-distribution/src/lib.rs +++ b/node/network/availability-distribution/src/lib.rs @@ -19,8 +19,9 @@ use futures::{future::Either, FutureExt, StreamExt, TryFutureExt}; use sp_keystore::SyncCryptoStorePtr; use polkadot_subsystem::{ - messages::AvailabilityDistributionMessage, FromOverseer, OverseerSignal, SpawnedSubsystem, - Subsystem, SubsystemContext, SubsystemError, + messages::{AllMessages, AvailabilityDistributionMessage}, FromOverseer, OverseerSignal, SpawnedSubsystem, + SubsystemContext, SubsystemError, + overseer, }; /// Error and [`Result`] type for this subsystem. @@ -58,9 +59,10 @@ pub struct AvailabilityDistributionSubsystem { metrics: Metrics, } -impl Subsystem for AvailabilityDistributionSubsystem +impl overseer::Subsystem for AvailabilityDistributionSubsystem where - Context: SubsystemContext + Sync + Send, + Context: SubsystemContext, + Context: overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self @@ -86,7 +88,8 @@ impl AvailabilityDistributionSubsystem { /// Start processing work as passed on from the Overseer. async fn run(mut self, mut ctx: Context) -> std::result::Result<(), Fatal> where - Context: SubsystemContext + Sync + Send, + Context: SubsystemContext, + Context: overseer::SubsystemContext, { let mut requester = Requester::new(self.metrics.clone()).fuse(); loop { diff --git a/node/network/availability-distribution/src/pov_requester/mod.rs b/node/network/availability-distribution/src/pov_requester/mod.rs index 3d7e41e6ae6b..c690049cc84f 100644 --- a/node/network/availability-distribution/src/pov_requester/mod.rs +++ b/node/network/availability-distribution/src/pov_requester/mod.rs @@ -62,7 +62,7 @@ where let full_req = Requests::PoVFetching(req); ctx.send_message( - AllMessages::NetworkBridge( + AllMessages::NetworkBridgeMessage( NetworkBridgeMessage::SendRequests( vec![full_req], // We are supposed to be connected to validators of our group via `PeerSet`, @@ -75,7 +75,7 @@ where let span = jaeger::Span::new(candidate_hash, "fetch-pov") .with_validator_index(from_validator) .with_relay_parent(parent); - ctx.spawn("pov-fetcher", fetch_pov_job(pov_hash, pending_response.boxed(), span, tx).boxed()) + ctx.spawn("pov-fetcher", fetch_pov_job(pov_hash, pending_response.boxed(), span, tx).boxed()).await .map_err(|e| Fatal::SpawnTask(e))?; Ok(()) } diff --git a/node/network/availability-distribution/src/requester/fetch_task/mod.rs b/node/network/availability-distribution/src/requester/fetch_task/mod.rs index c936d443fc6b..450abcc98e6e 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/mod.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/mod.rs @@ -188,7 +188,7 @@ impl FetchTask { if let Some(running) = prepared_running { let (handle, kill) = oneshot::channel(); - ctx.spawn("chunk-fetcher", running.run(kill).boxed()) + ctx.spawn("chunk-fetcher", running.run(kill).boxed()).await .map_err(|e| Fatal::SpawnTask(e))?; Ok(FetchTask { @@ -336,7 +336,7 @@ impl RunningTask { let requests = Requests::ChunkFetching(full_request); self.sender - .send(FromFetchTask::Message(AllMessages::NetworkBridge( + .send(FromFetchTask::Message(AllMessages::NetworkBridgeMessage( NetworkBridgeMessage::SendRequests(vec![requests], IfDisconnected::TryConnect) ))) .await @@ -399,7 +399,7 @@ impl RunningTask { let (tx, rx) = oneshot::channel(); let r = self .sender - .send(FromFetchTask::Message(AllMessages::AvailabilityStore( + .send(FromFetchTask::Message(AllMessages::AvailabilityStoreMessage( AvailabilityStoreMessage::StoreChunk { candidate_hash: self.request.candidate_hash, chunk, diff --git a/node/network/availability-distribution/src/requester/mod.rs b/node/network/availability-distribution/src/requester/mod.rs index 4553cb1707c1..68ebe90ca0b1 100644 --- a/node/network/availability-distribution/src/requester/mod.rs +++ b/node/network/availability-distribution/src/requester/mod.rs @@ -33,6 +33,7 @@ use futures::{ use polkadot_node_subsystem_util::runtime::{RuntimeInfo, get_occupied_cores}; use polkadot_primitives::v1::{CandidateHash, Hash, OccupiedCore}; use polkadot_subsystem::{ + messages::AllMessages, ActiveLeavesUpdate, SubsystemContext, ActivatedLeaf, }; diff --git a/node/network/availability-distribution/src/responder.rs b/node/network/availability-distribution/src/responder.rs index 9a20510eb354..27a662be8126 100644 --- a/node/network/availability-distribution/src/responder.rs +++ b/node/network/availability-distribution/src/responder.rs @@ -158,9 +158,9 @@ where Context: SubsystemContext, { let (tx, rx) = oneshot::channel(); - ctx.send_message(AllMessages::AvailabilityStore( + ctx.send_message( AvailabilityStoreMessage::QueryChunk(candidate_hash, validator_index, tx), - )) + ) .await; let result = rx.await.map_err(|e| { @@ -185,9 +185,9 @@ where Context: SubsystemContext, { let (tx, rx) = oneshot::channel(); - ctx.send_message(AllMessages::AvailabilityStore( + ctx.send_message( AvailabilityStoreMessage::QueryAvailableData(candidate_hash, tx), - )) + ) .await; let result = rx.await.map_err(|e| NonFatal::QueryAvailableDataResponseChannel(e))?; From 161a639fb0f90d6a4b37aff2541bf31b87fef692 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 2 Jul 2021 12:27:29 +0200 Subject: [PATCH 091/161] fixup bitfield distribution --- node/network/bitfield-distribution/src/lib.rs | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index d9279d668ce8..9997dab270be 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -27,9 +27,9 @@ use futures::{channel::oneshot, FutureExt}; use polkadot_subsystem::messages::*; use polkadot_subsystem::{ PerLeafSpan, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, - SubsystemContext, SubsystemResult, + SubsystemContext, SubsystemResult, SubsystemError, jaeger, - overseer::{self, Subsystem,}, + overseer, }; use polkadot_node_subsystem_util::{ metrics::{self, prometheus}, @@ -163,6 +163,7 @@ impl BitfieldDistribution { async fn run(self, mut ctx: Context) where Context: SubsystemContext, + Context: overseer::SubsystemContext, { // work: process incoming messages from the overseer and process accordingly. let mut state = ProtocolState::default(); @@ -251,9 +252,9 @@ where { tracing::trace!(target: LOG_TARGET, ?rep, peer_id = %peer, "reputation change"); - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message( NetworkBridgeMessage::ReportPeer(peer, rep), - )) + ) .await } @@ -329,7 +330,7 @@ where let _span = span.child("provisionable"); // notify the overseer about a new and valid signed bitfield - ctx.send_message(AllMessages::Provisioner( + ctx.send_message( ProvisionerMessage::ProvisionableData( message.relay_parent, ProvisionableData::Bitfield( @@ -337,7 +338,7 @@ where message.signed_availability.clone(), ), ), - )) + ) .await; drop(_span); @@ -384,12 +385,12 @@ where ); } else { let _span = span.child("gossip"); - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message( NetworkBridgeMessage::SendValidationMessage( interested_peers, message.into_validation_protocol(), ), - )) + ) .await; } } @@ -688,19 +689,20 @@ where .or_default() .insert(validator.clone()); - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message( NetworkBridgeMessage::SendValidationMessage( vec![dest], message.into_validation_protocol(), ), - )).await; + ).await; } -impl Subsystem for BitfieldDistribution +impl overseer::Subsystem for BitfieldDistribution where - C: SubsystemContext + Sync + Send, + Context: SubsystemContext, + Context: overseer::SubsystemContext, { - fn start(self, ctx: C) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx) .map(|_| Ok(())) .boxed(); @@ -723,12 +725,12 @@ where let (validators_tx, validators_rx) = oneshot::channel(); let (session_tx, session_rx) = oneshot::channel(); - let query_validators = AllMessages::RuntimeApi(RuntimeApiMessage::Request( + let query_validators = AllMessages::RuntimeApiMessage(RuntimeApiMessage::Request( relay_parent.clone(), RuntimeApiRequest::Validators(validators_tx), )); - let query_signing = AllMessages::RuntimeApi(RuntimeApiMessage::Request( + let query_signing = AllMessages::RuntimeApiMessage(RuntimeApiMessage::Request( relay_parent.clone(), RuntimeApiRequest::SessionIndexForChild(session_tx), )); From 8583c969437d7ca3073d092dd66b4a9a2253ebab Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 2 Jul 2021 12:35:41 +0200 Subject: [PATCH 092/161] fixup collation generation --- node/collation-generation/src/lib.rs | 32 +++++++++++++++------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index 49682972c362..735a4697a184 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -30,8 +30,11 @@ use polkadot_node_primitives::{ CollationGenerationConfig, AvailableData, PoV, }; use polkadot_node_subsystem::{ - messages::{CollationGenerationMessage, CollatorProtocolMessage}, - FromOverseer, SpawnedSubsystem, Subsystem, SubsystemContext, SubsystemResult, + ActiveLeavesUpdate, + messages::{AllMessages, CollationGenerationMessage, CollatorProtocolMessage}, + SpawnedSubsystem, SubsystemContext, SubsystemResult, + SubsystemError, FromOverseer, OverseerSignal, + overseer, }; use polkadot_node_subsystem_util::{ request_availability_cores, request_persisted_validation_data, @@ -82,7 +85,8 @@ impl CollationGenerationSubsystem { /// Otherwise, most are logged and then discarded. async fn run(mut self, mut ctx: Context) where - Context: SubsystemContext, + Context: SubsystemContext, + Context: overseer::SubsystemContext, { // when we activate new leaves, we spawn a bunch of sub-tasks, each of which is // expected to generate precisely one message. We don't want to block the main loop @@ -114,19 +118,16 @@ impl CollationGenerationSubsystem { // it should hopefully therefore be ok that it's an async function mutably borrowing self. async fn handle_incoming( &mut self, - incoming: SubsystemResult>, + incoming: SubsystemResult::Message>>, ctx: &mut Context, sender: &mpsc::Sender, ) -> bool where Context: SubsystemContext, + Context: overseer::SubsystemContext, { - use polkadot_node_subsystem::ActiveLeavesUpdate; - use polkadot_node_subsystem::FromOverseer::{Communication, Signal}; - use polkadot_node_subsystem::OverseerSignal::{ActiveLeaves, BlockFinalized, Conclude}; - match incoming { - Ok(Signal(ActiveLeaves(ActiveLeavesUpdate { activated, .. }))) => { + Ok(FromOverseer::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { activated, .. }))) => { // follow the procedure from the guide if let Some(config) = &self.config { let metrics = self.metrics.clone(); @@ -143,8 +144,8 @@ impl CollationGenerationSubsystem { false } - Ok(Signal(Conclude)) => true, - Ok(Communication { + Ok(FromOverseer::Signal(OverseerSignal::Conclude)) => true, + Ok(FromOverseer::Communication { msg: CollationGenerationMessage::Initialize(config), }) => { if self.config.is_some() { @@ -154,7 +155,7 @@ impl CollationGenerationSubsystem { } false } - Ok(Signal(BlockFinalized(..))) => false, + Ok(FromOverseer::Signal(OverseerSignal::BlockFinalized(..))) => false, Err(err) => { tracing::error!( target: LOG_TARGET, @@ -168,9 +169,10 @@ impl CollationGenerationSubsystem { } } -impl Subsystem for CollationGenerationSubsystem +impl overseer::Subsystem for CollationGenerationSubsystem where Context: SubsystemContext, + Context: overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = async move { @@ -402,7 +404,7 @@ async fn handle_new_activations( ); metrics.on_collation_generated(); - if let Err(err) = task_sender.send(AllMessages::CollatorProtocol( + if let Err(err) = task_sender.send(AllMessages::CollatorProtocolMessage( CollatorProtocolMessage::DistributeCollation(ccr, pov, result_sender) )).await { tracing::warn!( @@ -412,7 +414,7 @@ async fn handle_new_activations( "failed to send collation result", ); } - }))?; + })).await?; } } From 9b0bb57391239d469fa40fbc1821cab0b852a826 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 2 Jul 2021 13:05:19 +0200 Subject: [PATCH 093/161] chore --- behavior-tests/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/behavior-tests/Cargo.toml b/behavior-tests/Cargo.toml index b9aee7671bf7..0d79bafa2a34 100644 --- a/behavior-tests/Cargo.toml +++ b/behavior-tests/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "behavior-tests" version = "0.1.0" -authors = ["Bernhard Schuster "] +authors = ["Parity Technologies "] edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 5da3e345a163cda739b5531b54ccfe5565cd6a8c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 2 Jul 2021 13:16:06 +0200 Subject: [PATCH 094/161] fixup approval voting , part of chain-api --- node/core/approval-voting/src/import.rs | 23 ++++++------ node/core/approval-voting/src/lib.rs | 48 +++++++++++++------------ node/core/chain-api/src/lib.rs | 18 +++++++--- node/overseer/overseer-gen/src/lib.rs | 9 +++++ 4 files changed, 61 insertions(+), 37 deletions(-) diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index bcb59f6ba7b0..17f60678b289 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -29,9 +29,11 @@ //! We maintain a rolling window of session indices. This starts as empty use polkadot_node_subsystem::{ + overseer, messages::{ - RuntimeApiMessage, RuntimeApiRequest, ChainApiMessage, ApprovalDistributionMessage, + RuntimeApiMessage, RuntimeApiRequest, ChainApiMessage, ApprovalDistributionMessage, ApprovalVotingMessage, }, + SubsystemSender, SubsystemContext, SubsystemError, SubsystemResult, }; use polkadot_node_subsystem_util::determine_new_blocks; @@ -83,7 +85,7 @@ struct ImportedBlockInfoEnv<'a> { // Computes information about the imported block. Returns `None` if the info couldn't be extracted - // failure to communicate with overseer, async fn imported_block_info( - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), env: ImportedBlockInfoEnv<'_>, block_hash: Hash, block_header: &Header, @@ -97,7 +99,7 @@ async fn imported_block_info( ctx.send_message(RuntimeApiMessage::Request( block_hash, RuntimeApiRequest::CandidateEvents(c_tx), - ).into()).await; + )).await; let events: Vec = match c_rx.await { Ok(Ok(events)) => events, @@ -119,7 +121,7 @@ async fn imported_block_info( ctx.send_message(RuntimeApiMessage::Request( block_header.parent_hash, RuntimeApiRequest::SessionIndexForChild(s_tx), - ).into()).await; + )).await; let session_index = match s_rx.await { Ok(Ok(s)) => s, @@ -160,7 +162,7 @@ async fn imported_block_info( ctx.send_message(RuntimeApiMessage::Request( block_hash, RuntimeApiRequest::CurrentBabeEpoch(s_tx), - ).into()).await; + )).await; match s_rx.await { Ok(Ok(s)) => s, @@ -284,20 +286,21 @@ pub struct BlockImportedCandidates { /// /// It is the responsibility of the caller to schedule wakeups for each block. pub(crate) async fn handle_new_head( - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), state: &mut State, db_writer: &dyn KeyValueDB, db_config: DatabaseConfig, head: Hash, finalized_number: &Option, -) -> SubsystemResult> { +) -> SubsystemResult> +{ // Update session info based on most recent head. let mut span = jaeger::Span::new(head, "approval-checking-import"); let header = { let (h_tx, h_rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::BlockHeader(head, h_tx).into()).await; + ctx.send_message(ChainApiMessage::BlockHeader(head, h_tx)).await; match h_rx.await? { Err(e) => { @@ -375,7 +378,7 @@ pub(crate) async fn handle_new_head( // It's possible that we've lost a race with finality. let (tx, rx) = oneshot::channel(); ctx.send_message( - ChainApiMessage::FinalizedBlockHash(block_header.number.clone(), tx).into() + ChainApiMessage::FinalizedBlockHash(block_header.number.clone(), tx) ).await; let lost_to_finality = match rx.await { @@ -541,7 +544,7 @@ pub(crate) async fn handle_new_head( "Informing distribution of newly imported chain", ); - ctx.send_unbounded_message(ApprovalDistributionMessage::NewBlocks(approval_meta).into()); + ctx.send_unbounded_message(ApprovalDistributionMessage::NewBlocks(approval_meta)); Ok(imported_candidates) } diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 54943c40b660..db57bf81f42c 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -23,13 +23,14 @@ use polkadot_node_subsystem::{ messages::{ + AllMessages, AssignmentCheckError, AssignmentCheckResult, ApprovalCheckError, ApprovalCheckResult, ApprovalVotingMessage, RuntimeApiMessage, RuntimeApiRequest, ChainApiMessage, ApprovalDistributionMessage, CandidateValidationMessage, AvailabilityRecoveryMessage, }, errors::RecoveryError, - Subsystem, SubsystemContext, SubsystemError, SubsystemResult, SpawnedSubsystem, + overseer::{self, SubsystemSender as _}, SubsystemContext, SubsystemError, SubsystemResult, SpawnedSubsystem, FromOverseer, OverseerSignal, SubsystemSender, }; use polkadot_node_subsystem_util::{ @@ -326,11 +327,13 @@ impl ApprovalVotingSubsystem { } } -impl Subsystem for ApprovalVotingSubsystem - where C: SubsystemContext +impl overseer::Subsystem for ApprovalVotingSubsystem +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, { - fn start(self, ctx: C) -> SpawnedSubsystem { - let future = run::( + fn start(self, ctx: Context) -> SpawnedSubsystem { + let future = run::( ctx, self, Box::new(SystemClock), @@ -722,13 +725,15 @@ enum Action { Conclude, } -async fn run( - mut ctx: C, +async fn run( + mut ctx: Context, mut subsystem: ApprovalVotingSubsystem, clock: Box, assignment_criteria: Box, ) -> SubsystemResult<()> - where C: SubsystemContext +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, { let mut state = State { session_window: RollingSessionWindow::new(APPROVAL_SESSIONS), @@ -849,8 +854,7 @@ async fn run( // // returns `true` if any of the actions was a `Conclude` command. async fn handle_actions( - ctx: &mut impl SubsystemContext, - state: &mut State, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), state: &mut State, metrics: &Metrics, wakeups: &mut Wakeups, currently_checking_set: &mut CurrentlyCheckingSet, @@ -922,7 +926,7 @@ async fn handle_actions( ctx.send_unbounded_message(ApprovalDistributionMessage::DistributeAssignment( indirect_cert, candidate_index, - ).into()); + )); match approvals_cache.get(&candidate_hash) { Some(ApprovalOutcome::Approved) => { @@ -969,7 +973,7 @@ async fn handle_actions( ApprovalDBV1Reader::new(db, db_config) )?; - ctx.send_messages(messages.into_iter().map(Into::into)).await; + ctx.send_messages(messages.into_iter()).await; } Action::Conclude => { conclude = true; } } @@ -1084,7 +1088,7 @@ fn distribution_messages_for_activation<'a>( // Handle an incoming signal from the overseer. Returns true if execution should conclude. async fn handle_from_overseer( - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), state: &mut State, metrics: &Metrics, db_writer: &dyn KeyValueDB, @@ -1198,7 +1202,7 @@ async fn handle_from_overseer( } async fn handle_approved_ancestor( - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), db: &impl DBReader, target: Hash, lower_bound: BlockNumber, @@ -1217,7 +1221,7 @@ async fn handle_approved_ancestor( let target_number = { let (tx, rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::BlockNumber(target, tx).into()).await; + ctx.send_message(ChainApiMessage::BlockNumber(target, tx)).await; match rx.await { Ok(Ok(Some(n))) => n, @@ -1241,7 +1245,7 @@ async fn handle_approved_ancestor( hash: target, k: (target_number - (lower_bound + 1)) as usize, response_channel: tx, - }.into()).await; + }).await; match rx.await { Ok(Ok(a)) => a, @@ -2054,7 +2058,7 @@ fn process_wakeup( // spawned. When the background work is no longer needed, the `AbortHandle` should be dropped // to cancel the background work and any requests it has spawned. async fn launch_approval( - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), metrics: Metrics, session_index: SessionIndex, candidate: CandidateReceipt, @@ -2103,7 +2107,7 @@ async fn launch_approval( session_index, Some(backing_group), a_tx, - ).into()).await; + )).await; ctx.send_message( RuntimeApiMessage::Request( @@ -2112,7 +2116,7 @@ async fn launch_approval( candidate.descriptor.validation_code_hash, code_tx, ), - ).into() + ) ).await; let candidate = candidate.clone(); @@ -2196,13 +2200,13 @@ async fn launch_approval( let para_id = candidate.descriptor.para_id; - sender.send_message(CandidateValidationMessage::ValidateFromExhaustive( + sender.send_message(AllMessages::from(CandidateValidationMessage::ValidateFromExhaustive( available_data.validation_data, validation_code, candidate.descriptor, available_data.pov, val_tx, - ).into()).await; + ))).await; match val_rx.await { Err(_) => @@ -2261,7 +2265,7 @@ async fn launch_approval( }; let (background, remote_handle) = background.remote_handle(); - ctx.spawn("approval-checks", Box::pin(background)) + ctx.spawn("approval-checks", Box::pin(background)).await .map(move |()| remote_handle) } diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index b7c152686afa..fa5b34e2a11a 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -40,7 +40,12 @@ use sp_blockchain::HeaderBackend; use polkadot_node_subsystem_util::metrics::{self, prometheus}; use polkadot_primitives::v1::{Block, BlockId}; use polkadot_subsystem::{ - messages::ChainApiMessage, FromOverseer, OverseerSignal, SpawnedSubsystem, Subsystem, + overseer, + messages::{ + AllMessages, + ChainApiMessage, + }, + FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, SubsystemResult, }; @@ -62,13 +67,14 @@ impl ChainApiSubsystem { } } -impl Subsystem for ChainApiSubsystem +impl overseer::Subsystem for ChainApiSubsystem where Client: HeaderBackend + AuxStore + 'static, Context: SubsystemContext, + Context: overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { - let future = run(ctx, self) + let future = run::(ctx, self) .map_err(|e| SubsystemError::with_origin("chain-api", e)) .boxed(); SpawnedSubsystem { @@ -78,12 +84,14 @@ where } } -async fn run( - mut ctx: impl SubsystemContext, +async fn run( + mut ctx: Context, subsystem: ChainApiSubsystem, ) -> SubsystemResult<()> where Client: HeaderBackend + AuxStore, + Context: SubsystemContext, + Context: overseer::SubsystemContext, { loop { match ctx.recv().await? { diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index e38bc4c26b6e..b150ee8bbbcf 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -415,6 +415,15 @@ pub trait SubsystemContext: Send + 'static { self.sender().send_messages(msgs.into_iter().map(|x| ::from(x))).await } + /// Send a message using the unbounded connection. + fn send_unbounded_message(&mut self, msg: X) + where + Self::AllMessages: From, + X: Send, + { + self.sender().send_unbounded_message(Self::AllMessages::from(msg)) + } + /// Obtain the sender. fn sender(&mut self) -> &mut Self::Sender; } From c90657cd0704a595b0ff81b8de1baefde691045e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 2 Jul 2021 13:16:37 +0200 Subject: [PATCH 095/161] split tests off in chain-api --- node/core/chain-api/src/lib.rs | 307 +------------------------------ node/core/chain-api/src/tests.rs | 304 ++++++++++++++++++++++++++++++ 2 files changed, 305 insertions(+), 306 deletions(-) create mode 100644 node/core/chain-api/src/tests.rs diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index fa5b34e2a11a..73949a2e66b2 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -301,309 +301,4 @@ impl metrics::Metrics for Metrics { } #[cfg(test)] -mod tests { - use super::*; - - use std::collections::BTreeMap; - use futures::{future::BoxFuture, channel::oneshot}; - use parity_scale_codec::Encode; - - use polkadot_primitives::v1::{Hash, BlockNumber, BlockId, Header}; - use polkadot_node_primitives::BlockWeight; - use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle}; - use sp_blockchain::Info as BlockInfo; - use sp_core::testing::TaskExecutor; - - #[derive(Clone)] - struct TestClient { - blocks: BTreeMap, - block_weights: BTreeMap, - finalized_blocks: BTreeMap, - headers: BTreeMap, - } - - const ONE: Hash = Hash::repeat_byte(0x01); - const TWO: Hash = Hash::repeat_byte(0x02); - const THREE: Hash = Hash::repeat_byte(0x03); - const FOUR: Hash = Hash::repeat_byte(0x04); - const ERROR_PATH: Hash = Hash::repeat_byte(0xFF); - - fn default_header() -> Header { - Header { - parent_hash: Hash::zero(), - number: 100500, - state_root: Hash::zero(), - extrinsics_root: Hash::zero(), - digest: Default::default(), - } - } - - impl Default for TestClient { - fn default() -> Self { - Self { - blocks: maplit::btreemap! { - ONE => 1, - TWO => 2, - THREE => 3, - FOUR => 4, - }, - block_weights: maplit::btreemap! { - ONE => 0, - TWO => 1, - THREE => 1, - FOUR => 2, - }, - finalized_blocks: maplit::btreemap! { - 1 => ONE, - 3 => THREE, - }, - headers: maplit::btreemap! { - TWO => Header { - parent_hash: ONE, - number: 2, - ..default_header() - }, - THREE => Header { - parent_hash: TWO, - number: 3, - ..default_header() - }, - FOUR => Header { - parent_hash: THREE, - number: 4, - ..default_header() - }, - ERROR_PATH => Header { - ..default_header() - } - }, - } - } - } - - fn last_key_value(map: &BTreeMap) -> (K, V) { - assert!(!map.is_empty()); - map.iter() - .last() - .map(|(k, v)| (k.clone(), v.clone())) - .unwrap() - } - - impl HeaderBackend for TestClient { - fn info(&self) -> BlockInfo { - let genesis_hash = self.blocks.iter().next().map(|(h, _)| *h).unwrap(); - let (best_hash, best_number) = last_key_value(&self.blocks); - let (finalized_number, finalized_hash) = last_key_value(&self.finalized_blocks); - - BlockInfo { - best_hash, - best_number, - genesis_hash, - finalized_hash, - finalized_number, - number_leaves: 0, - finalized_state: None, - } - } - fn number(&self, hash: Hash) -> sp_blockchain::Result> { - Ok(self.blocks.get(&hash).copied()) - } - fn hash(&self, number: BlockNumber) -> sp_blockchain::Result> { - Ok(self.finalized_blocks.get(&number).copied()) - } - fn header(&self, id: BlockId) -> sp_blockchain::Result> { - match id { - // for error path testing - BlockId::Hash(hash) if hash.is_zero() => { - Err(sp_blockchain::Error::Backend("Zero hashes are illegal!".into())) - } - BlockId::Hash(hash) => { - Ok(self.headers.get(&hash).cloned()) - } - _ => unreachable!(), - } - } - fn status(&self, _id: BlockId) -> sp_blockchain::Result { - unimplemented!() - } - } - - fn test_harness( - test: impl FnOnce(Arc, TestSubsystemContextHandle) - -> BoxFuture<'static, ()>, - ) { - let (ctx, ctx_handle) = make_subsystem_context(TaskExecutor::new()); - let client = Arc::new(TestClient::default()); - - let subsystem = ChainApiSubsystem::new(client.clone(), Metrics(None)); - let chain_api_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = test(client, ctx_handle); - - futures::executor::block_on(future::join(chain_api_task, test_task)); - } - - impl AuxStore for TestClient { - fn insert_aux< - 'a, - 'b: 'a, - 'c: 'a, - I: IntoIterator, - D: IntoIterator, - >( - &self, - _insert: I, - _delete: D, - ) -> sp_blockchain::Result<()> { - unimplemented!() - } - - fn get_aux(&self, key: &[u8]) -> sp_blockchain::Result>> { - Ok(self - .block_weights - .iter() - .find(|(hash, _)| sc_consensus_babe::aux_schema::block_weight_key(hash) == key) - .map(|(_, weight)| weight.encode())) - } - } - - #[test] - fn request_block_number() { - test_harness(|client, mut sender| { - async move { - let zero = Hash::zero(); - let test_cases = [ - (TWO, client.number(TWO).unwrap()), - (zero, client.number(zero).unwrap()), // not here - ]; - for (hash, expected) in &test_cases { - let (tx, rx) = oneshot::channel(); - - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::BlockNumber(*hash, tx), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), *expected); - } - - sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }.boxed() - }) - } - - #[test] - fn request_block_header() { - test_harness(|client, mut sender| { - async move { - const NOT_HERE: Hash = Hash::repeat_byte(0x5); - let test_cases = [ - (TWO, client.header(BlockId::Hash(TWO)).unwrap()), - (NOT_HERE, client.header(BlockId::Hash(NOT_HERE)).unwrap()), - ]; - for (hash, expected) in &test_cases { - let (tx, rx) = oneshot::channel(); - - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::BlockHeader(*hash, tx), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), *expected); - } - - sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }.boxed() - }) - } - - #[test] - fn request_block_weight() { - test_harness(|client, mut sender| { - async move { - const NOT_HERE: Hash = Hash::repeat_byte(0x5); - let test_cases = [ - (TWO, sc_consensus_babe::block_weight(&*client, TWO).unwrap()), - (FOUR, sc_consensus_babe::block_weight(&*client, FOUR).unwrap()), - (NOT_HERE, sc_consensus_babe::block_weight(&*client, NOT_HERE).unwrap()), - ]; - for (hash, expected) in &test_cases { - let (tx, rx) = oneshot::channel(); - - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::BlockWeight(*hash, tx), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), *expected); - } - - sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }.boxed() - }) - } - - #[test] - fn request_finalized_hash() { - test_harness(|client, mut sender| { - async move { - let test_cases = [ - (1, client.hash(1).unwrap()), // not here - (2, client.hash(2).unwrap()), - ]; - for (number, expected) in &test_cases { - let (tx, rx) = oneshot::channel(); - - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::FinalizedBlockHash(*number, tx), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), *expected); - } - - sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }.boxed() - }) - } - - #[test] - fn request_last_finalized_number() { - test_harness(|client, mut sender| { - async move { - let (tx, rx) = oneshot::channel(); - - let expected = client.info().finalized_number; - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::FinalizedBlockNumber(tx), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), expected); - - sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }.boxed() - }) - } - - #[test] - fn request_ancestors() { - test_harness(|_client, mut sender| { - async move { - let (tx, rx) = oneshot::channel(); - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::Ancestors { hash: THREE, k: 4, response_channel: tx }, - }).await; - assert_eq!(rx.await.unwrap().unwrap(), vec![TWO, ONE]); - - let (tx, rx) = oneshot::channel(); - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::Ancestors { hash: TWO, k: 1, response_channel: tx }, - }).await; - assert_eq!(rx.await.unwrap().unwrap(), vec![ONE]); - - let (tx, rx) = oneshot::channel(); - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::Ancestors { hash: ERROR_PATH, k: 2, response_channel: tx }, - }).await; - assert!(rx.await.unwrap().is_err()); - - sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }.boxed() - }) - } -} +mod tests; diff --git a/node/core/chain-api/src/tests.rs b/node/core/chain-api/src/tests.rs new file mode 100644 index 000000000000..015030cfab24 --- /dev/null +++ b/node/core/chain-api/src/tests.rs @@ -0,0 +1,304 @@ +use super::*; + +use std::collections::BTreeMap; +use futures::{future::BoxFuture, channel::oneshot}; +use parity_scale_codec::Encode; + +use polkadot_primitives::v1::{Hash, BlockNumber, BlockId, Header}; +use polkadot_node_primitives::BlockWeight; +use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle}; +use sp_blockchain::Info as BlockInfo; +use sp_core::testing::TaskExecutor; + +#[derive(Clone)] +struct TestClient { + blocks: BTreeMap, + block_weights: BTreeMap, + finalized_blocks: BTreeMap, + headers: BTreeMap, +} + +const ONE: Hash = Hash::repeat_byte(0x01); +const TWO: Hash = Hash::repeat_byte(0x02); +const THREE: Hash = Hash::repeat_byte(0x03); +const FOUR: Hash = Hash::repeat_byte(0x04); +const ERROR_PATH: Hash = Hash::repeat_byte(0xFF); + +fn default_header() -> Header { + Header { + parent_hash: Hash::zero(), + number: 100500, + state_root: Hash::zero(), + extrinsics_root: Hash::zero(), + digest: Default::default(), + } +} + +impl Default for TestClient { + fn default() -> Self { + Self { + blocks: maplit::btreemap! { + ONE => 1, + TWO => 2, + THREE => 3, + FOUR => 4, + }, + block_weights: maplit::btreemap! { + ONE => 0, + TWO => 1, + THREE => 1, + FOUR => 2, + }, + finalized_blocks: maplit::btreemap! { + 1 => ONE, + 3 => THREE, + }, + headers: maplit::btreemap! { + TWO => Header { + parent_hash: ONE, + number: 2, + ..default_header() + }, + THREE => Header { + parent_hash: TWO, + number: 3, + ..default_header() + }, + FOUR => Header { + parent_hash: THREE, + number: 4, + ..default_header() + }, + ERROR_PATH => Header { + ..default_header() + } + }, + } + } +} + +fn last_key_value(map: &BTreeMap) -> (K, V) { + assert!(!map.is_empty()); + map.iter() + .last() + .map(|(k, v)| (k.clone(), v.clone())) + .unwrap() +} + +impl HeaderBackend for TestClient { + fn info(&self) -> BlockInfo { + let genesis_hash = self.blocks.iter().next().map(|(h, _)| *h).unwrap(); + let (best_hash, best_number) = last_key_value(&self.blocks); + let (finalized_number, finalized_hash) = last_key_value(&self.finalized_blocks); + + BlockInfo { + best_hash, + best_number, + genesis_hash, + finalized_hash, + finalized_number, + number_leaves: 0, + finalized_state: None, + } + } + fn number(&self, hash: Hash) -> sp_blockchain::Result> { + Ok(self.blocks.get(&hash).copied()) + } + fn hash(&self, number: BlockNumber) -> sp_blockchain::Result> { + Ok(self.finalized_blocks.get(&number).copied()) + } + fn header(&self, id: BlockId) -> sp_blockchain::Result> { + match id { + // for error path testing + BlockId::Hash(hash) if hash.is_zero() => { + Err(sp_blockchain::Error::Backend("Zero hashes are illegal!".into())) + } + BlockId::Hash(hash) => { + Ok(self.headers.get(&hash).cloned()) + } + _ => unreachable!(), + } + } + fn status(&self, _id: BlockId) -> sp_blockchain::Result { + unimplemented!() + } +} + +fn test_harness( + test: impl FnOnce(Arc, TestSubsystemContextHandle) + -> BoxFuture<'static, ()>, +) { + let (ctx, ctx_handle) = make_subsystem_context(TaskExecutor::new()); + let client = Arc::new(TestClient::default()); + + let subsystem = ChainApiSubsystem::new(client.clone(), Metrics(None)); + let chain_api_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = test(client, ctx_handle); + + futures::executor::block_on(future::join(chain_api_task, test_task)); +} + +impl AuxStore for TestClient { + fn insert_aux< + 'a, + 'b: 'a, + 'c: 'a, + I: IntoIterator, + D: IntoIterator, + >( + &self, + _insert: I, + _delete: D, + ) -> sp_blockchain::Result<()> { + unimplemented!() + } + + fn get_aux(&self, key: &[u8]) -> sp_blockchain::Result>> { + Ok(self + .block_weights + .iter() + .find(|(hash, _)| sc_consensus_babe::aux_schema::block_weight_key(hash) == key) + .map(|(_, weight)| weight.encode())) + } +} + +#[test] +fn request_block_number() { + test_harness(|client, mut sender| { + async move { + let zero = Hash::zero(); + let test_cases = [ + (TWO, client.number(TWO).unwrap()), + (zero, client.number(zero).unwrap()), // not here + ]; + for (hash, expected) in &test_cases { + let (tx, rx) = oneshot::channel(); + + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::BlockNumber(*hash, tx), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), *expected); + } + + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) +} + +#[test] +fn request_block_header() { + test_harness(|client, mut sender| { + async move { + const NOT_HERE: Hash = Hash::repeat_byte(0x5); + let test_cases = [ + (TWO, client.header(BlockId::Hash(TWO)).unwrap()), + (NOT_HERE, client.header(BlockId::Hash(NOT_HERE)).unwrap()), + ]; + for (hash, expected) in &test_cases { + let (tx, rx) = oneshot::channel(); + + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::BlockHeader(*hash, tx), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), *expected); + } + + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) +} + +#[test] +fn request_block_weight() { + test_harness(|client, mut sender| { + async move { + const NOT_HERE: Hash = Hash::repeat_byte(0x5); + let test_cases = [ + (TWO, sc_consensus_babe::block_weight(&*client, TWO).unwrap()), + (FOUR, sc_consensus_babe::block_weight(&*client, FOUR).unwrap()), + (NOT_HERE, sc_consensus_babe::block_weight(&*client, NOT_HERE).unwrap()), + ]; + for (hash, expected) in &test_cases { + let (tx, rx) = oneshot::channel(); + + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::BlockWeight(*hash, tx), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), *expected); + } + + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) +} + +#[test] +fn request_finalized_hash() { + test_harness(|client, mut sender| { + async move { + let test_cases = [ + (1, client.hash(1).unwrap()), // not here + (2, client.hash(2).unwrap()), + ]; + for (number, expected) in &test_cases { + let (tx, rx) = oneshot::channel(); + + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::FinalizedBlockHash(*number, tx), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), *expected); + } + + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) +} + +#[test] +fn request_last_finalized_number() { + test_harness(|client, mut sender| { + async move { + let (tx, rx) = oneshot::channel(); + + let expected = client.info().finalized_number; + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::FinalizedBlockNumber(tx), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), expected); + + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) +} + +#[test] +fn request_ancestors() { + test_harness(|_client, mut sender| { + async move { + let (tx, rx) = oneshot::channel(); + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::Ancestors { hash: THREE, k: 4, response_channel: tx }, + }).await; + assert_eq!(rx.await.unwrap().unwrap(), vec![TWO, ONE]); + + let (tx, rx) = oneshot::channel(); + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::Ancestors { hash: TWO, k: 1, response_channel: tx }, + }).await; + assert_eq!(rx.await.unwrap().unwrap(), vec![ONE]); + + let (tx, rx) = oneshot::channel(); + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::Ancestors { hash: ERROR_PATH, k: 2, response_channel: tx }, + }).await; + assert!(rx.await.unwrap().is_err()); + + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) +} From 096ea780477853c5b8cb4c0e6999c1f613350427 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 2 Jul 2021 13:30:11 +0200 Subject: [PATCH 096/161] fixup remaining subsystems --- node/network/approval-distribution/src/lib.rs | 58 +++++++++---------- .../network/statement-distribution/src/lib.rs | 58 +++++++++---------- 2 files changed, 57 insertions(+), 59 deletions(-) diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 160d45957b0e..8c7c2774a389 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -29,11 +29,13 @@ use polkadot_node_primitives::{ approval::{AssignmentCert, BlockApprovalMeta, IndirectSignedApprovalVote, IndirectAssignmentCert}, }; use polkadot_node_subsystem::{ + overseer, messages::{ AllMessages, ApprovalDistributionMessage, ApprovalVotingMessage, NetworkBridgeMessage, AssignmentCheckResult, ApprovalCheckResult, NetworkBridgeEvent, }, - ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, Subsystem, SubsystemContext, + SubsystemError, + ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, }; use polkadot_node_subsystem_util::{ metrics::{self, prometheus}, @@ -187,7 +189,7 @@ enum PendingMessage { impl State { async fn handle_network_msg( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), metrics: &Metrics, event: NetworkBridgeEvent, ) { @@ -257,8 +259,7 @@ impl State { async fn handle_new_blocks( &mut self, - ctx: &mut impl SubsystemContext, - metrics: &Metrics, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), metrics: &Metrics, metas: Vec, ) { let mut new_hashes = HashSet::new(); @@ -360,8 +361,7 @@ impl State { async fn process_incoming_peer_message( &mut self, - ctx: &mut impl SubsystemContext, - metrics: &Metrics, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), metrics: &Metrics, peer_id: PeerId, msg: protocol_v1::ApprovalDistributionMessage, ) { @@ -448,8 +448,7 @@ impl State { async fn handle_peer_view_change( &mut self, - ctx: &mut impl SubsystemContext, - metrics: &Metrics, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), metrics: &Metrics, peer_id: PeerId, view: View, ) { @@ -512,8 +511,7 @@ impl State { async fn import_and_circulate_assignment( &mut self, - ctx: &mut impl SubsystemContext, - metrics: &Metrics, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), metrics: &Metrics, source: MessageSource, assignment: IndirectAssignmentCert, claimed_candidate_index: CandidateIndex, @@ -592,11 +590,11 @@ impl State { let (tx, rx) = oneshot::channel(); - ctx.send_message(AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportAssignment( + ctx.send_message(ApprovalVotingMessage::CheckAndImportAssignment( assignment.clone(), claimed_candidate_index, tx, - ))).await; + )).await; let timer = metrics.time_awaiting_approval_voting(); let result = match rx.await { @@ -743,14 +741,13 @@ impl State { protocol_v1::ValidationProtocol::ApprovalDistribution( protocol_v1::ApprovalDistributionMessage::Assignments(assignments) ), - ).into()).await; + )).await; } } async fn import_and_circulate_approval( &mut self, - ctx: &mut impl SubsystemContext, - metrics: &Metrics, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), metrics: &Metrics, source: MessageSource, vote: IndirectSignedApprovalVote, ) { @@ -840,10 +837,10 @@ impl State { let (tx, rx) = oneshot::channel(); - ctx.send_message(AllMessages::ApprovalVoting(ApprovalVotingMessage::CheckAndImportApproval( + ctx.send_message(ApprovalVotingMessage::CheckAndImportApproval( vote.clone(), tx, - ))).await; + )).await; let timer = metrics.time_awaiting_approval_voting(); let result = match rx.await { @@ -989,13 +986,12 @@ impl State { protocol_v1::ValidationProtocol::ApprovalDistribution( protocol_v1::ApprovalDistributionMessage::Approvals(approvals) ), - ).into()).await; + )).await; } } async fn unify_with_peer( - ctx: &mut impl SubsystemContext, - gossip_peers: &HashSet, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), gossip_peers: &HashSet, metrics: &Metrics, entries: &mut HashMap, peer_id: PeerId, @@ -1060,8 +1056,7 @@ impl State { async fn send_gossip_messages_to_peer( entries: &HashMap, - ctx: &mut impl SubsystemContext, - peer_id: PeerId, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), peer_id: PeerId, blocks: Vec, ) { let mut assignments = Vec::new(); @@ -1130,7 +1125,7 @@ impl State { protocol_v1::ValidationProtocol::ApprovalDistribution( protocol_v1::ApprovalDistributionMessage::Assignments(assignments) ), - ).into()).await; + )).await; } if !approvals.is_empty() { @@ -1147,7 +1142,7 @@ impl State { protocol_v1::ValidationProtocol::ApprovalDistribution( protocol_v1::ApprovalDistributionMessage::Approvals(approvals) ), - ).into()).await; + )).await; } } } @@ -1155,7 +1150,7 @@ impl State { /// Modify the reputation of a peer based on its behavior. async fn modify_reputation( - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), peer_id: PeerId, rep: Rep, ) { @@ -1166,9 +1161,9 @@ async fn modify_reputation( "Reputation change for peer", ); - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message( NetworkBridgeMessage::ReportPeer(peer_id, rep), - )).await; + ).await; } impl ApprovalDistribution { @@ -1180,6 +1175,7 @@ impl ApprovalDistribution { async fn run(self, ctx: Context) where Context: SubsystemContext, + Context: overseer::SubsystemContext, { let mut state = State::default(); self.run_inner(ctx, &mut state).await @@ -1189,6 +1185,7 @@ impl ApprovalDistribution { async fn run_inner(self, mut ctx: Context, state: &mut State) where Context: SubsystemContext, + Context: overseer::SubsystemContext, { loop { let message = match ctx.recv().await { @@ -1261,11 +1258,12 @@ impl ApprovalDistribution { } } -impl Subsystem for ApprovalDistribution +impl overseer::Subsystem for ApprovalDistribution where - C: SubsystemContext + Sync + Send, + Context: SubsystemContext, + Context: overseer::SubsystemContext, { - fn start(self, ctx: C) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { let future = self.run(ctx) .map(|_| Ok(())) .boxed(); diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index b390017d2a06..026c4ab02356 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -26,7 +26,8 @@ use error::{FatalResult, NonFatalResult, log_error}; use parity_scale_codec::Encode; use polkadot_subsystem::{ - ActiveLeavesUpdate, FromOverseer, OverseerSignal, PerLeafSpan, SpawnedSubsystem, Subsystem, + overseer, + ActiveLeavesUpdate, FromOverseer, OverseerSignal, PerLeafSpan, SpawnedSubsystem, SubsystemContext, SubsystemError, jaeger, messages::{ AllMessages, NetworkBridgeMessage, StatementDistributionMessage, @@ -107,10 +108,12 @@ pub struct StatementDistribution { metrics: Metrics, } -impl Subsystem for StatementDistribution - where C: SubsystemContext +impl overseer::Subsystem for StatementDistribution +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, { - fn start(self, ctx: C) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { // Swallow error because failure is fatal to the node and we log with more precision // within `run`. SpawnedSubsystem { @@ -588,7 +591,7 @@ enum Message { impl Message { async fn receive( - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), from_requester: &mut mpsc::Receiver, from_responder: &mut mpsc::Receiver, ) -> Message { @@ -846,7 +849,7 @@ async fn circulate_statement_and_dependents( gossip_peers: &HashSet, peers: &mut HashMap, active_heads: &mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), relay_parent: Hash, statement: SignedFullStatement, priority_peers: Vec, @@ -953,7 +956,7 @@ fn is_statement_large(statement: &SignedFullStatement) -> bool { async fn circulate_statement<'a>( gossip_peers: &HashSet, peers: &mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), relay_parent: Hash, stored: StoredStatement<'a>, mut priority_peers: Vec, @@ -1017,7 +1020,7 @@ async fn circulate_statement<'a>( statement = ?stored.statement, "Sending statement", ); - ctx.send_message(AllMessages::NetworkBridge(NetworkBridgeMessage::SendValidationMessage( + ctx.send_message(AllMessages::NetworkBridgeMessage(NetworkBridgeMessage::SendValidationMessage( peers_to_send.iter().map(|(p, _)| p.clone()).collect(), payload, ))).await; @@ -1034,7 +1037,7 @@ async fn circulate_statement<'a>( async fn send_statements_about( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), relay_parent: Hash, candidate_hash: CandidateHash, active_head: &ActiveHeadData, @@ -1059,7 +1062,7 @@ async fn send_statements_about( statement = ?statement.statement, "Sending statement", ); - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message(AllMessages::NetworkBridgeMessage( NetworkBridgeMessage::SendValidationMessage(vec![peer.clone()], payload) )).await; @@ -1071,7 +1074,7 @@ async fn send_statements_about( async fn send_statements( peer: PeerId, peer_data: &mut PeerData, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), relay_parent: Hash, active_head: &ActiveHeadData, metrics: &Metrics, @@ -1094,7 +1097,7 @@ async fn send_statements( statement = ?statement.statement, "Sending statement" ); - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message(AllMessages::NetworkBridgeMessage( NetworkBridgeMessage::SendValidationMessage(vec![peer.clone()], payload) )).await; @@ -1103,11 +1106,11 @@ async fn send_statements( } async fn report_peer( - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), peer: PeerId, rep: Rep, ) { - ctx.send_message(AllMessages::NetworkBridge( + ctx.send_message(AllMessages::NetworkBridgeMessage( NetworkBridgeMessage::ReportPeer(peer, rep) )).await } @@ -1123,7 +1126,7 @@ async fn retrieve_statement_from_message<'a>( peer: PeerId, message: protocol_v1::StatementDistributionMessage, active_head: &'a mut ActiveHeadData, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), req_sender: &mpsc::Sender, metrics: &Metrics, ) -> Option { @@ -1225,7 +1228,7 @@ async fn launch_request( meta: StatementMetadata, peer: PeerId, req_sender: mpsc::Sender, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), metrics: &Metrics, ) -> Option { @@ -1238,7 +1241,7 @@ async fn launch_request( ) .remote_handle(); - let result = ctx.spawn("large-statement-fetcher", task.boxed()); + let result = ctx.spawn("large-statement-fetcher", task.boxed()).await; if let Err(err) = result { tracing::error!(target: LOG_TARGET, ?err, "Spawning task failed."); return None @@ -1263,7 +1266,7 @@ async fn handle_incoming_message_and_circulate<'a>( gossip_peers: &HashSet, peers: &mut HashMap, active_heads: &'a mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, @@ -1312,7 +1315,7 @@ async fn handle_incoming_message<'a>( peer: PeerId, peer_data: &mut PeerData, active_heads: &'a mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), message: protocol_v1::StatementDistributionMessage, req_sender: &mpsc::Sender, metrics: &Metrics, @@ -1447,10 +1450,7 @@ async fn handle_incoming_message<'a>( // When we receive a new message from a peer, we forward it to the // candidate backing subsystem. - let message = AllMessages::CandidateBacking( - CandidateBackingMessage::Statement(relay_parent, statement.statement.clone()) - ); - ctx.send_message(message).await; + ctx.send_message(CandidateBackingMessage::Statement(relay_parent, statement.statement.clone())).await; Some((relay_parent, statement)) } @@ -1462,7 +1462,7 @@ async fn update_peer_view_and_maybe_send_unlocked( peer: PeerId, gossip_peers: &HashSet, peer_data: &mut PeerData, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), active_heads: &HashMap, new_view: View, metrics: &Metrics, @@ -1506,7 +1506,7 @@ async fn handle_network_update( gossip_peers: &mut HashSet, authorities: &mut HashMap, active_heads: &mut HashMap, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), req_sender: &mpsc::Sender, update: NetworkBridgeEvent, metrics: &Metrics, @@ -1599,7 +1599,7 @@ async fn handle_network_update( impl StatementDistribution { async fn run( self, - mut ctx: impl SubsystemContext, + mut ctx: (impl SubsystemContext + overseer::SubsystemContext), ) -> std::result::Result<(), Fatal> { let mut peers: HashMap = HashMap::new(); let mut gossip_peers: HashSet = HashSet::new(); @@ -1774,7 +1774,7 @@ impl StatementDistribution { } RequesterMessage::SendRequest(req) => { ctx.send_message( - AllMessages::NetworkBridge( + AllMessages::NetworkBridgeMessage( NetworkBridgeMessage::SendRequests( vec![req], IfDisconnected::ImmediateError, @@ -1831,7 +1831,7 @@ impl StatementDistribution { async fn handle_subsystem_message( &self, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), runtime: &mut RuntimeInfo, peers: &mut HashMap, gossip_peers: &mut HashSet, @@ -1951,7 +1951,7 @@ impl StatementDistribution { ctx.spawn( "large-statement-responder", respond(receiver, res_sender.clone()).boxed() - ).map_err(Fatal::SpawnTask)?; + ).await.map_err(Fatal::SpawnTask)?; } } } From 4bed82372b4c66d763a2ee9b54d86b0d7da83499 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 2 Jul 2021 18:10:03 +0200 Subject: [PATCH 097/161] add new subsystems --- node/overseer/src/lib.rs | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 300344023be3..4f0f90f488fb 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -90,6 +90,7 @@ use polkadot_node_subsystem::messages::{ CollatorProtocolMessage, AvailabilityRecoveryMessage, ApprovalDistributionMessage, ApprovalVotingMessage, GossipSupportMessage, NetworkBridgeEvent, + DisputeParticipationMessage, DisputeCoordinatorMessage, ChainSelectionMessage, }; pub use polkadot_node_subsystem::{ OverseerSignal, @@ -100,7 +101,7 @@ pub use polkadot_node_subsystem::{ /// TODO legacy, to be deleted, left for easier integration mod subsystems; -use self::subsystems::AllSubsystems; +pub use self::subsystems::AllSubsystems; mod metrics; use self::metrics::Metrics; @@ -211,6 +212,24 @@ impl Handler { tracing::info!(target: LOG_TARGET, "Failed to send an event to Overseer"); } } + + /// Whether the overseer handler is connected to an overseer. + #[deprecated(note ="Makes no sense imho")] + pub fn is_connected(&self) -> bool { + true + } + + /// Whether the handler is disconnected. + #[deprecated(note="Makes no sense imho")] + pub fn is_disconnected(&self) -> bool { + false + } + + /// Using this handler, connect another handler to the same + /// overseer, if any. + pub fn connect_other(&self, other: &mut Handler) { + *other = self.clone(); + } } /// An event telling the `Overseer` on the particular block @@ -369,6 +388,15 @@ pub struct Overseer { #[subsystem(no_dispatch, GossipSupportMessage)] gossip_support: GossipSupport, + #[subsystem(no_dispatch, DisputeCoordinatorMessage)] + dipute_coordinator: DisputeCoordinator, + + #[subsystem(no_dispatch, DisputeParticipationMessage)] + dispute_participation: DisputeParticipation, + + #[subsystem(no_dispatch, ChainSelectionMessage)] + chain_selection: ChainSelection, + /// External listeners waiting for a hash to be in the active-leave set. pub activation_external_listeners: HashMap>>>, From 39e0008d3666a661fe26fa2978dd546c6e864fd6 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 5 Jul 2021 10:54:19 +0200 Subject: [PATCH 098/161] license header foo --- .../proc-macro/src/impl_builder.rs | 18 ++++++++++++++++++ .../proc-macro/src/impl_channels_out.rs | 16 ++++++++++++++++ .../proc-macro/src/impl_message_wrapper.rs | 16 ++++++++++++++++ .../overseer-gen/proc-macro/src/impl_misc.rs | 16 ++++++++++++++++ .../proc-macro/src/impl_overseer.rs | 16 ++++++++++++++++ .../proc-macro/src/impl_replace.rs | 19 ++++++++++++++++++- .../overseer-gen/proc-macro/src/parse_attr.rs | 16 ++++++++++++++++ .../proc-macro/src/parse_struct.rs | 16 ++++++++++++++++ .../overseer-gen/proc-macro/src/tests.rs | 16 ++++++++++++++++ node/overseer/overseer-gen/src/lib.rs | 2 +- node/overseer/src/subsystems.rs | 15 +++++++++++++++ 11 files changed, 164 insertions(+), 2 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 8bd103b48fcb..f603740ed38a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -1,3 +1,19 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + use quote::quote; use syn::{Ident, Result}; @@ -5,6 +21,8 @@ use super::*; /// Implement a builder pattern for the `Overseer`-type, /// which acts as the gateway to constructing the overseer. +/// +/// Elements tagged with `wip` are not covered here. pub(crate) fn impl_builder(info: &OverseerInfo) -> Result { let overseer_name = info.overseer_name.clone(); let builder = Ident::new(&(overseer_name.to_string() + "Builder"), overseer_name.span()); diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 9ea2b24fa97b..ccef2c5d16df 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -1,3 +1,19 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + use quote::quote; use syn::Result; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 72085e4f6a0e..88f6d6ccc472 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -1,6 +1,22 @@ use quote::quote; use syn::Result; use syn::Path; +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + use syn::spanned::Spanned; use proc_macro2::Ident; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index c63298e3ffeb..aebdd84323a1 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -1,3 +1,19 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + use quote::quote; use syn::{Ident, Result}; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index b6943ff2438f..f592e36ed035 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -1,3 +1,19 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + use quote::quote; use syn::Result; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs index 72f99d09062b..ae60743a519f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs @@ -1,4 +1,21 @@ -//! Replace a subsystem +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Replace a subsystem with a different implementation. + use proc_macro2::TokenStream; use quote::quote; use syn::{Ident, Result}; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs index dd0231549138..a4a3450a0610 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs @@ -1,3 +1,19 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + use proc_macro2::Span; use std::collections::{hash_map::RandomState, HashMap}; use syn::parse::Parse; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index ab7763a15d82..784e76cf3df8 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -1,3 +1,19 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + use proc_macro2::{Span, TokenStream}; use quote::quote; use syn::Visibility; diff --git a/node/overseer/overseer-gen/proc-macro/src/tests.rs b/node/overseer/overseer-gen/proc-macro/src/tests.rs index 56b230786d37..10adb2fd17ab 100644 --- a/node/overseer/overseer-gen/proc-macro/src/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/tests.rs @@ -1,3 +1,19 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + use super::*; use assert_matches::assert_matches; use quote::quote; diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index b150ee8bbbcf..3b62bd889252 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. +// Copyright 2021 Parity Technologies (UK) Ltd. // This file is part of Polkadot. // Polkadot is free software: you can redistribute it and/or modify diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index b3808f7f021f..85df1292e995 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -1,3 +1,18 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . use polkadot_node_subsystem::errors::SubsystemError; use ::polkadot_overseer_gen::{ From 5b5e114ae552f250bc7fa9aa9366b8e90cd7a260 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 5 Jul 2021 10:54:41 +0200 Subject: [PATCH 099/161] introduce wip --- .../proc-macro/src/parse_struct.rs | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 784e76cf3df8..a2aeb07ca36a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -49,6 +49,10 @@ pub(crate) struct SubSysField { /// If the subsystem implementation is blocking execution and hence /// has to be spawned on a separate thread or thread pool. pub(crate) blocking: bool, + /// The subsystem is a work in progress. + /// Avoids dispatching `Wrapper` type messages, but generates the variants. + /// Does not require the subsystem to be instanciated with the builder pattern. + pub(crate) wip: bool, } fn try_type_to_path(ty: Type, span: Span) -> Result { @@ -63,6 +67,9 @@ pub(crate) struct SubSystemTag { pub(crate) attrs: Vec, #[allow(dead_code)] pub(crate) no_dispatch: bool, + /// The subsystem is WIP, only generate the `Wrapper` variant, but do not forward messages + /// and also not include the subsystem in the list of subsystems. + pub(crate) wip: bool, pub(crate) blocking: bool, pub(crate) consumes: Punctuated, } @@ -86,7 +93,7 @@ impl Parse for SubSystemTag { let mut unique = HashSet::<_, RandomState>::default(); while let Some(ident) = parse_tags()? { - if ident != "no_dispatch" && ident != "blocking" { + if ident != "no_dispatch" && ident != "blocking" && ident != "wip" { return Err(Error::new(ident.span(), "Allowed tags are only `no_dispatch` or `blocking`.")); } if !unique.insert(ident.to_string()) { @@ -95,10 +102,11 @@ impl Parse for SubSystemTag { } let no_dispatch = unique.take("no_dispatch").is_some(); let blocking = unique.take("blocking").is_some(); + let wip = unique.take("wip").is_some(); let consumes = content.parse_terminated(Path::parse)?; - Ok(Self { attrs, no_dispatch, blocking, consumes }) + Ok(Self { attrs, no_dispatch, blocking, consumes, wip}) } } @@ -152,7 +160,11 @@ impl OverseerInfo { } pub(crate) fn subsystem_names(&self) -> Vec { - self.subsystems.iter().map(|ssf| ssf.name.clone()).collect::>() + self.subsystems + .iter() + .filter(|ssf| !ssf.wip) + .map(|ssf| ssf.name.clone()) + .collect::>() } #[allow(dead_code)] @@ -169,15 +181,18 @@ impl OverseerInfo { self.baggage.iter().map(|bag| bag.field_ty.clone()).collect::>() } pub(crate) fn baggage_decl(&self) -> Vec { - self.baggage.iter().map(|bag| { - let BaggageField { - vis, - field_ty, - field_name, - .. - } = bag; - quote!{ #vis #field_name: #field_ty } - }).collect::>() + self.baggage + .iter() + .map(|bag| { + let BaggageField { + vis, + field_ty, + field_name, + .. + } = bag; + quote!{ #vis #field_name: #field_ty } + }) + .collect::>() } /// Generic types per subsystem, in the form `Sub#N`. @@ -254,6 +269,7 @@ impl OverseerGuts { ty: try_type_to_path(ty, span)?, consumes: consumes_paths[0].clone(), no_dispatch: variant.no_dispatch, + wip: variant.wip, blocking: variant.blocking, }); } else { From 6fbece0e43c04b891539542ab417805d8ce5be64 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 5 Jul 2021 11:21:46 +0200 Subject: [PATCH 100/161] finalize `wip` tag --- .../proc-macro/src/impl_dispatch.rs | 1 + .../proc-macro/src/impl_overseer.rs | 9 ++++- .../proc-macro/src/parse_struct.rs | 37 ++++++++++++++++++- node/overseer/src/lib.rs | 6 +-- 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index c27a77e321b6..7f8f270455a5 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -43,6 +43,7 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { ::std::iter::once( extern_msg // focuses on a `NetworkBridgeEvent< protocol_v1::* >` + // TODO do not require this to be hardcoded, either externalize or ... .focus() .ok() .map(|event: NetworkBridgeEvent< _ >| { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index f592e36ed035..ae29fd871da3 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -22,7 +22,7 @@ use super::*; pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result { let message_wrapper = &info.message_wrapper.clone(); let overseer_name = info.overseer_name.clone(); - let subsystem_name = &info.subsystem_names(); + let subsystem_name = &info.subsystem_names_without_wip(); let baggage_decl = &info.baggage_decl(); @@ -38,7 +38,8 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result Result self. #subsystem_name .send_message( inner ).await?, )* + // subsystems that are still work in progress + #( + #message_wrapper :: #unconsumes ( inner ) => {} + )* } Ok(()) } diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index a2aeb07ca36a..552079995dbf 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -167,6 +167,14 @@ impl OverseerInfo { .collect::>() } + pub(crate) fn subsystem_names_without_wip(&self) -> Vec { + self.subsystems + .iter() + .filter(|ssf| !ssf.wip) + .map(|ssf| ssf.name.clone()) + .collect::>() + } + #[allow(dead_code)] // TODO use as the defaults, if no subsystem is specified // TODO or drop the type argument. @@ -197,7 +205,11 @@ impl OverseerInfo { /// Generic types per subsystem, in the form `Sub#N`. pub(crate) fn builder_generic_types(&self) -> Vec { - self.subsystems.iter().map(|sff| sff.generic.clone()).collect::>() + self.subsystems + .iter() + .filter(|ssf| !ssf.wip) + .map(|sff| sff.generic.clone()) + .collect::>() } pub(crate) fn baggage_generic_types(&self) -> Vec { @@ -218,6 +230,29 @@ impl OverseerInfo { pub(crate) fn consumes(&self) -> Vec { self.subsystems.iter().map(|ssf| ssf.consumes.clone()).collect::>() } + + pub(crate) fn channel_names_without_wip(&self, suffix: &'static str) -> Vec { + self.subsystems + .iter() + .filter(|ssf| !ssf.wip) + .map(|ssf| Ident::new(&(ssf.name.to_string() + suffix), ssf.name.span())) + .collect::>() + } + + pub(crate) fn consumes_without_wip(&self) -> Vec { + self.subsystems + .iter() + .filter(|ssf| !ssf.wip) + .map(|ssf| ssf.consumes.clone()) + .collect::>() + } + pub(crate) fn consumes_only_wip(&self) -> Vec { + self.subsystems + .iter() + .filter(|ssf| ssf.wip) + .map(|ssf| ssf.consumes.clone()) + .collect::>() + } } /// Internals of the overseer. diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 4f0f90f488fb..8c8a958b5ea9 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -388,13 +388,13 @@ pub struct Overseer { #[subsystem(no_dispatch, GossipSupportMessage)] gossip_support: GossipSupport, - #[subsystem(no_dispatch, DisputeCoordinatorMessage)] + #[subsystem(no_dispatch, wip, DisputeCoordinatorMessage)] dipute_coordinator: DisputeCoordinator, - #[subsystem(no_dispatch, DisputeParticipationMessage)] + #[subsystem(no_dispatch, wip, DisputeParticipationMessage)] dispute_participation: DisputeParticipation, - #[subsystem(no_dispatch, ChainSelectionMessage)] + #[subsystem(no_dispatch, wip, ChainSelectionMessage)] chain_selection: ChainSelection, /// External listeners waiting for a hash to be in the active-leave set. From 322341ef5f5f9242760f36994f57fd59a5e3b52b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 5 Jul 2021 12:15:07 +0200 Subject: [PATCH 101/161] missing baggage type --- node/overseer/overseer-gen/proc-macro/src/impl_builder.rs | 2 +- node/overseer/src/lib.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index f603740ed38a..bb8fb5432f02 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -233,7 +233,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Date: Mon, 5 Jul 2021 15:04:59 +0200 Subject: [PATCH 102/161] fixup merge artifact labeled as TODO --- .../proc-macro/src/impl_dispatch.rs | 2 +- node/overseer/src/lib.rs | 4 --- node/subsystem-types/src/lib.rs | 1 - node/subsystem/src/lib.rs | 29 +------------------ 4 files changed, 2 insertions(+), 34 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index 7f8f270455a5..fbb215a4656d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -46,7 +46,7 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { // TODO do not require this to be hardcoded, either externalize or ... .focus() .ok() - .map(|event: NetworkBridgeEvent< _ >| { + .map(|event: NetworkBridgeEvent<_>| { #message_wrapper :: #dispatchable ( // the inner type of the enum variant #dispatchable :: from( event ) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index eb1d81317c58..c5c65cfd35c2 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -419,10 +419,6 @@ pub struct Overseer { /// Various Prometheus metrics. pub metrics: Metrics, - - // Configuration for the candidate validation subsystem. - // FIXME !!! - // pub candidate_validation_config: CandidateValidationConfig, } diff --git a/node/subsystem-types/src/lib.rs b/node/subsystem-types/src/lib.rs index 95af2c2b373d..f3d350898c06 100644 --- a/node/subsystem-types/src/lib.rs +++ b/node/subsystem-types/src/lib.rs @@ -69,7 +69,6 @@ impl LeafStatus { } } - /// Activated leaf. #[derive(Debug, Clone)] pub struct ActivatedLeaf { diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 42c670f69d77..04bb503b7cdd 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -41,40 +41,13 @@ const ACTIVE_LEAVES_SMALLVEC_CAPACITY: usize = 8; pub use polkadot_node_subsystem_types::{ errors::{self, *}, ActivatedLeaf, + LeafStatus, }; pub mod messages { pub use super::overseer::AllMessages; pub use polkadot_node_subsystem_types::messages::*; } -/// The status of an activated leaf. -#[derive(Debug, Clone)] -pub enum LeafStatus { - /// A leaf is fresh when it's the first time the leaf has been encountered. - /// Most leaves should be fresh. - Fresh, - /// A leaf is stale when it's encountered for a subsequent time. This will happen - /// when the chain is reverted or the fork-choice rule abandons some chain. - Stale, -} - -impl LeafStatus { - /// Returns a bool indicating fresh status. - pub fn is_fresh(&self) -> bool { - match *self { - LeafStatus::Fresh => true, - LeafStatus::Stale => false, - } - } - - /// Returns a bool indicating stale status. - pub fn is_stale(&self) -> bool { - match *self { - LeafStatus::Fresh => false, - LeafStatus::Stale => true, - } - } -} /// A `Result` type that wraps [`SubsystemError`]. /// From 2e9ba173699d65375ebad3dc02b290944c2dcca7 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 5 Jul 2021 15:54:35 +0200 Subject: [PATCH 103/161] fixins --- Cargo.lock | 2 -- node/core/approval-voting/src/lib.rs | 5 ++--- node/core/av-store/src/lib.rs | 2 +- node/core/candidate-validation/src/lib.rs | 6 +++--- node/core/chain-api/src/lib.rs | 5 +---- node/core/dispute-participation/src/lib.rs | 4 ++-- node/core/parachains-inherent/Cargo.toml | 1 - node/core/parachains-inherent/src/lib.rs | 5 +---- node/core/runtime-api/src/lib.rs | 2 +- node/core/runtime-api/src/tests.rs | 1 + node/network/approval-distribution/src/lib.rs | 2 +- node/network/availability-distribution/src/lib.rs | 2 +- .../src/pov_requester/mod.rs | 5 ++--- .../availability-distribution/src/responder.rs | 2 +- node/network/availability-recovery/src/lib.rs | 2 +- node/network/bitfield-distribution/src/lib.rs | 13 ++++++------- node/network/bridge/src/lib.rs | 3 +-- node/network/bridge/src/network.rs | 1 - node/network/collator-protocol/Cargo.toml | 1 - .../collator-protocol/src/collator_side/mod.rs | 2 +- .../collator-protocol/src/collator_side/tests.rs | 2 +- node/network/collator-protocol/src/error.rs | 5 ----- node/network/collator-protocol/src/lib.rs | 13 +++---------- .../collator-protocol/src/validator_side/mod.rs | 2 +- .../collator-protocol/src/validator_side/tests.rs | 2 +- node/network/gossip-support/src/lib.rs | 1 - node/overseer/examples/minimal-example.rs | 4 ++-- node/overseer/src/lib.rs | 1 - node/overseer/src/subsystems.rs | 3 ++- node/subsystem-types/src/errors.rs | 2 +- .../src/messages/network_bridge_event.rs | 2 -- node/subsystem-util/src/determine_new_blocks.rs | 1 - node/subsystem-util/src/lib.rs | 4 +--- node/subsystem-util/src/rolling_session_window.rs | 14 ++++---------- node/subsystem-util/src/runtime/mod.rs | 6 +----- node/subsystem-util/src/tests.rs | 3 +-- node/subsystem/src/lib.rs | 12 +----------- 37 files changed, 45 insertions(+), 98 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22ff73711468..3ad5749b0261 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5921,7 +5921,6 @@ dependencies = [ "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", - "polkadot-overseer", "polkadot-primitives", "sp-core", "sp-keyring", @@ -6232,7 +6231,6 @@ dependencies = [ "futures 0.3.15", "futures-timer 3.0.2", "polkadot-node-subsystem", - "polkadot-overseer", "polkadot-primitives", "sp-blockchain", "sp-inherents", diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index db57bf81f42c..a199d39fc3c9 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -23,7 +23,6 @@ use polkadot_node_subsystem::{ messages::{ - AllMessages, AssignmentCheckError, AssignmentCheckResult, ApprovalCheckError, ApprovalCheckResult, ApprovalVotingMessage, RuntimeApiMessage, RuntimeApiRequest, ChainApiMessage, ApprovalDistributionMessage, CandidateValidationMessage, @@ -2200,13 +2199,13 @@ async fn launch_approval( let para_id = candidate.descriptor.para_id; - sender.send_message(AllMessages::from(CandidateValidationMessage::ValidateFromExhaustive( + sender.send_message(CandidateValidationMessage::ValidateFromExhaustive( available_data.validation_data, validation_code, candidate.descriptor, available_data.pov, val_tx, - ))).await; + ).into()).await; match val_rx.await { Err(_) => diff --git a/node/core/av-store/src/lib.rs b/node/core/av-store/src/lib.rs index 50bb7de5ab5c..0160e87c434e 100644 --- a/node/core/av-store/src/lib.rs +++ b/node/core/av-store/src/lib.rs @@ -39,7 +39,7 @@ use polkadot_node_primitives::{ use polkadot_subsystem::{ FromOverseer, OverseerSignal, SubsystemError, SubsystemContext, SpawnedSubsystem, - overseer::{self, Subsystem}, + overseer, ActiveLeavesUpdate, errors::{ChainApiError, RuntimeApiError}, }; diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index b46fb487c971..d986fcc876b2 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -28,7 +28,7 @@ use polkadot_subsystem::{ SubsystemContext, SpawnedSubsystem, SubsystemResult, SubsystemError, FromOverseer, OverseerSignal, messages::{ - AllMessages, CandidateValidationMessage, RuntimeApiMessage, + CandidateValidationMessage, RuntimeApiMessage, ValidationFailed, RuntimeApiRequest, }, errors::RuntimeApiError, @@ -192,10 +192,10 @@ where Context: overseer::SubsystemContext, { ctx.send_message( - AllMessages::RuntimeApiMessage(RuntimeApiMessage::Request( + RuntimeApiMessage::Request( relay_parent, request, - )) + ) ).await; receiver.await.map_err(Into::into) diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index 73949a2e66b2..856f7f823e71 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -41,10 +41,7 @@ use polkadot_node_subsystem_util::metrics::{self, prometheus}; use polkadot_primitives::v1::{Block, BlockId}; use polkadot_subsystem::{ overseer, - messages::{ - AllMessages, - ChainApiMessage, - }, + messages::ChainApiMessage, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, SubsystemResult, }; diff --git a/node/core/dispute-participation/src/lib.rs b/node/core/dispute-participation/src/lib.rs index 19bc56a020a6..373b2b4ea806 100644 --- a/node/core/dispute-participation/src/lib.rs +++ b/node/core/dispute-participation/src/lib.rs @@ -360,13 +360,13 @@ async fn issue_local_statement( session: SessionIndex, valid: bool, ) { - ctx.send_message(AllMessages::DisputeCoordinator( + ctx.send_message( DisputeCoordinatorMessage::IssueLocalStatement( session, candidate_hash, candidate_receipt, valid, ), - )) + ) .await } diff --git a/node/core/parachains-inherent/Cargo.toml b/node/core/parachains-inherent/Cargo.toml index 6dd3d3bbdab6..bfd1e84c415d 100644 --- a/node/core/parachains-inherent/Cargo.toml +++ b/node/core/parachains-inherent/Cargo.toml @@ -11,7 +11,6 @@ tracing = "0.1.26" thiserror = "1.0.23" async-trait = "0.1.47" polkadot-node-subsystem = { path = "../../subsystem" } -polkadot-overseer = { path = "../../overseer" } polkadot-primitives = { path = "../../../primitives" } sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-inherents = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs index ef68425bc37f..4dfc0d4c85bf 100644 --- a/node/core/parachains-inherent/src/lib.rs +++ b/node/core/parachains-inherent/src/lib.rs @@ -25,11 +25,8 @@ #![deny(unused_crate_dependencies, unused_results)] use futures::{select, FutureExt}; -use polkadot_overseer::{ - AllMessages, - Handler, -}; use polkadot_node_subsystem::{ + overseer::Handler, messages::ProvisionerMessage, errors::SubsystemError, }; use polkadot_primitives::v1::{ diff --git a/node/core/runtime-api/src/lib.rs b/node/core/runtime-api/src/lib.rs index f0844875d224..f58a17fdaa71 100644 --- a/node/core/runtime-api/src/lib.rs +++ b/node/core/runtime-api/src/lib.rs @@ -28,7 +28,7 @@ use polkadot_subsystem::{ SubsystemContext, errors::RuntimeApiError, messages::{ - AllMessages, RuntimeApiMessage, RuntimeApiRequest as Request, + RuntimeApiMessage, RuntimeApiRequest as Request, }, overseer, }; diff --git a/node/core/runtime-api/src/tests.rs b/node/core/runtime-api/src/tests.rs index 787dc8884b0b..1f6078f73e4f 100644 --- a/node/core/runtime-api/src/tests.rs +++ b/node/core/runtime-api/src/tests.rs @@ -29,6 +29,7 @@ use futures::channel::oneshot; use polkadot_node_primitives::{ BabeEpoch, BabeEpochConfiguration, BabeAllowedSlots, }; +use polkadot_subsystem::messages::AllMessages; #[derive(Default, Clone)] struct MockRuntimeApi { diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 8c7c2774a389..c0848a328173 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -31,7 +31,7 @@ use polkadot_node_primitives::{ use polkadot_node_subsystem::{ overseer, messages::{ - AllMessages, ApprovalDistributionMessage, ApprovalVotingMessage, NetworkBridgeMessage, + ApprovalDistributionMessage, ApprovalVotingMessage, NetworkBridgeMessage, AssignmentCheckResult, ApprovalCheckResult, NetworkBridgeEvent, }, SubsystemError, diff --git a/node/network/availability-distribution/src/lib.rs b/node/network/availability-distribution/src/lib.rs index b42024472791..67de44ccdcb8 100644 --- a/node/network/availability-distribution/src/lib.rs +++ b/node/network/availability-distribution/src/lib.rs @@ -19,7 +19,7 @@ use futures::{future::Either, FutureExt, StreamExt, TryFutureExt}; use sp_keystore::SyncCryptoStorePtr; use polkadot_subsystem::{ - messages::{AllMessages, AvailabilityDistributionMessage}, FromOverseer, OverseerSignal, SpawnedSubsystem, + messages::AvailabilityDistributionMessage, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, overseer, }; diff --git a/node/network/availability-distribution/src/pov_requester/mod.rs b/node/network/availability-distribution/src/pov_requester/mod.rs index c690049cc84f..e6aa9cfa55e2 100644 --- a/node/network/availability-distribution/src/pov_requester/mod.rs +++ b/node/network/availability-distribution/src/pov_requester/mod.rs @@ -29,7 +29,7 @@ use polkadot_primitives::v1::{ use polkadot_node_primitives::PoV; use polkadot_subsystem::{ SubsystemContext, - messages::{AllMessages, NetworkBridgeMessage, IfDisconnected} + messages::{NetworkBridgeMessage, IfDisconnected} }; use polkadot_node_subsystem_util::runtime::RuntimeInfo; @@ -62,7 +62,6 @@ where let full_req = Requests::PoVFetching(req); ctx.send_message( - AllMessages::NetworkBridgeMessage( NetworkBridgeMessage::SendRequests( vec![full_req], // We are supposed to be connected to validators of our group via `PeerSet`, @@ -70,7 +69,7 @@ where // longer to get established, so we try to connect in any case. IfDisconnected::TryConnect ) - )).await; + ).await; let span = jaeger::Span::new(candidate_hash, "fetch-pov") .with_validator_index(from_validator) diff --git a/node/network/availability-distribution/src/responder.rs b/node/network/availability-distribution/src/responder.rs index 27a662be8126..e45574b5a6ed 100644 --- a/node/network/availability-distribution/src/responder.rs +++ b/node/network/availability-distribution/src/responder.rs @@ -24,7 +24,7 @@ use polkadot_node_network_protocol::request_response::{request::IncomingRequest, use polkadot_primitives::v1::{CandidateHash, ValidatorIndex}; use polkadot_node_primitives::{AvailableData, ErasureChunk}; use polkadot_subsystem::{ - messages::{AllMessages, AvailabilityStoreMessage}, + messages::AvailabilityStoreMessage, SubsystemContext, jaeger, }; diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index 5dd3a16aa2e9..937da4e8f5c0 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -40,7 +40,7 @@ use polkadot_subsystem::{ errors::RecoveryError, jaeger, messages::{ - AvailabilityStoreMessage, AvailabilityRecoveryMessage, AllMessages, NetworkBridgeMessage, + AvailabilityStoreMessage, AvailabilityRecoveryMessage, NetworkBridgeMessage, }, }; use polkadot_node_network_protocol::{ diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index 9997dab270be..a0c66b18fec1 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -725,18 +725,17 @@ where let (validators_tx, validators_rx) = oneshot::channel(); let (session_tx, session_rx) = oneshot::channel(); - let query_validators = AllMessages::RuntimeApiMessage(RuntimeApiMessage::Request( + // query validators + ctx.send_message(RuntimeApiMessage::Request( relay_parent.clone(), RuntimeApiRequest::Validators(validators_tx), - )); + )).await; - let query_signing = AllMessages::RuntimeApiMessage(RuntimeApiMessage::Request( + // query signing context + ctx.send_message(RuntimeApiMessage::Request( relay_parent.clone(), RuntimeApiRequest::SessionIndexForChild(session_tx), - )); - - ctx.send_messages(std::iter::once(query_validators).chain(std::iter::once(query_signing))) - .await; + )).await; match (validators_rx.await?, session_rx.await?) { (Ok(v), Ok(s)) => Ok(Some(( diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 35cbc213eb20..606bd73cefc3 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -24,7 +24,6 @@ use parity_scale_codec::{Encode, Decode}; use parking_lot::Mutex; use futures::prelude::*; use futures::stream::BoxStream; -use polkadot_overseer::Overseer; use sc_network::Event as NetworkEvent; use sp_consensus::SyncOracle; @@ -908,7 +907,7 @@ where ctx.spawn("network-bridge-network-worker", Box::pin(remote)).await?; ctx.send_message( - AllMessages::from(StatementDistributionMessage::StatementFetchingReceiver(statement_receiver)) + StatementDistributionMessage::StatementFetchingReceiver(statement_receiver) ).await; let subsystem_event_handler = handle_subsystem_messages( diff --git a/node/network/bridge/src/network.rs b/node/network/bridge/src/network.rs index c7522bde2799..3ffdb4baa114 100644 --- a/node/network/bridge/src/network.rs +++ b/node/network/bridge/src/network.rs @@ -34,7 +34,6 @@ use polkadot_node_network_protocol::{ PeerId, UnifiedReputationChange as Rep, }; use polkadot_primitives::v1::{AuthorityDiscoveryId, Block, Hash}; -use polkadot_subsystem::errors::{SubsystemError, SubsystemResult}; use crate::validator_discovery::AuthorityDiscovery; diff --git a/node/network/collator-protocol/Cargo.toml b/node/network/collator-protocol/Cargo.toml index 84e1788e9693..971e4903f22a 100644 --- a/node/network/collator-protocol/Cargo.toml +++ b/node/network/collator-protocol/Cargo.toml @@ -19,7 +19,6 @@ polkadot-primitives = { path = "../../../primitives" } polkadot-node-network-protocol = { path = "../../network/protocol" } polkadot-node-primitives = { path = "../../primitives" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } -polkadot-overseer = { path = "../../overseer" } polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } [dev-dependencies] diff --git a/node/network/collator-protocol/src/collator_side/mod.rs b/node/network/collator-protocol/src/collator_side/mod.rs index 895999e28782..643429b9c060 100644 --- a/node/network/collator-protocol/src/collator_side/mod.rs +++ b/node/network/collator-protocol/src/collator_side/mod.rs @@ -27,7 +27,7 @@ use polkadot_subsystem::{ overseer, FromOverseer, OverseerSignal, PerLeafSpan, SubsystemContext, jaeger, messages::{ - AllMessages, CollatorProtocolMessage, NetworkBridgeEvent, NetworkBridgeMessage, + CollatorProtocolMessage, NetworkBridgeEvent, NetworkBridgeMessage, }, }; use polkadot_node_network_protocol::{ diff --git a/node/network/collator-protocol/src/collator_side/tests.rs b/node/network/collator-protocol/src/collator_side/tests.rs index a716dbc16a78..96319d8c151b 100644 --- a/node/network/collator-protocol/src/collator_side/tests.rs +++ b/node/network/collator-protocol/src/collator_side/tests.rs @@ -39,7 +39,7 @@ use polkadot_primitives::v1::{ use polkadot_node_primitives::BlockData; use polkadot_subsystem::{ jaeger, - messages::{RuntimeApiMessage, RuntimeApiRequest}, + messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest}, ActiveLeavesUpdate, ActivatedLeaf, LeafStatus, }; use polkadot_subsystem_testhelpers as test_helpers; diff --git a/node/network/collator-protocol/src/error.rs b/node/network/collator-protocol/src/error.rs index e2e636aed9e4..4fa6ac1959cc 100644 --- a/node/network/collator-protocol/src/error.rs +++ b/node/network/collator-protocol/src/error.rs @@ -19,7 +19,6 @@ use polkadot_node_primitives::UncheckedSignedFullStatement; use polkadot_subsystem::errors::SubsystemError; -use polkadot_overseer::OverseerError; use thiserror::Error; use polkadot_node_subsystem_util::{Fault, runtime, unwrap_non_fatal}; @@ -62,10 +61,6 @@ pub enum Fatal { #[error("Receiving message from overseer failed")] SubsystemReceive(#[source] SubsystemError), - /// Receiving subsystem message from overseer failed. - #[error("Receiving message from overseer failed")] - Overseer(#[source] OverseerError), - /// Errors coming from runtime::Runtime. #[error("Error while accessing runtime information")] Runtime(#[from] #[source] runtime::Fatal), diff --git a/node/network/collator-protocol/src/lib.rs b/node/network/collator-protocol/src/lib.rs index 4794b335c0cc..96af19aa1e58 100644 --- a/node/network/collator-protocol/src/lib.rs +++ b/node/network/collator-protocol/src/lib.rs @@ -30,21 +30,14 @@ use polkadot_node_network_protocol::{PeerId, UnifiedReputationChange as Rep}; use polkadot_primitives::v1::CollatorPair; use polkadot_subsystem::{ - OverseerSignal, - FromOverseer, SpawnedSubsystem, SubsystemContext, SubsystemSender, - overseer::{self, OverseerError, }, + overseer, messages::{ - AllMessages, CollatorProtocolMessage, NetworkBridgeMessage, - }, -}; -use polkadot_subsystem::{ - errors::{RuntimeApiError, SubsystemError}, - messages::{ - RuntimeApiMessage, RuntimeApiRequest as Request, + CollatorProtocolMessage, NetworkBridgeMessage, }, + errors::SubsystemError, }; mod error; diff --git a/node/network/collator-protocol/src/validator_side/mod.rs b/node/network/collator-protocol/src/validator_side/mod.rs index ed71449c8cdd..5a5208130c53 100644 --- a/node/network/collator-protocol/src/validator_side/mod.rs +++ b/node/network/collator-protocol/src/validator_side/mod.rs @@ -43,7 +43,7 @@ use polkadot_subsystem::{ overseer, jaeger, messages::{ - AllMessages, CollatorProtocolMessage, IfDisconnected, + CollatorProtocolMessage, IfDisconnected, NetworkBridgeEvent, NetworkBridgeMessage, CandidateBackingMessage, }, FromOverseer, OverseerSignal, PerLeafSpan, SubsystemContext, SubsystemSender, diff --git a/node/network/collator-protocol/src/validator_side/tests.rs b/node/network/collator-protocol/src/validator_side/tests.rs index fb49bdfa2c56..86076407c776 100644 --- a/node/network/collator-protocol/src/validator_side/tests.rs +++ b/node/network/collator-protocol/src/validator_side/tests.rs @@ -31,7 +31,7 @@ use polkadot_primitives::v1::{ use polkadot_node_primitives::BlockData; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_subsystem_testhelpers as test_helpers; -use polkadot_subsystem::messages::{RuntimeApiMessage, RuntimeApiRequest}; +use polkadot_subsystem::messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest}; use polkadot_node_network_protocol::{ our_view, ObservedRole, request_response::{Requests, ResponseSender}, }; diff --git a/node/network/gossip-support/src/lib.rs b/node/network/gossip-support/src/lib.rs index 207d554161b7..5d20f7b03420 100644 --- a/node/network/gossip-support/src/lib.rs +++ b/node/network/gossip-support/src/lib.rs @@ -33,7 +33,6 @@ use polkadot_node_subsystem::{ SubsystemError, FromOverseer, SpawnedSubsystem, SubsystemContext, messages::{ - AllMessages, GossipSupportMessage, NetworkBridgeMessage, RuntimeApiMessage, diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index 1eaeac205429..ae8ee8cea985 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -63,7 +63,7 @@ impl Subsystem1 { Delay::new(Duration::from_secs(1)).await; let (tx, _) = oneshot::channel(); - ctx.send_message(AllMessages::CandidateValidation( + ctx.send_message( CandidateValidationMessage::ValidateFromChainState( Default::default(), PoV { @@ -71,7 +71,7 @@ impl Subsystem1 { }.into(), tx, ) - )).await; + ).await; } } } diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index c5c65cfd35c2..f0253cb1c185 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -114,7 +114,6 @@ use polkadot_node_metrics::{ Metronome, }; pub use polkadot_overseer_gen::{ - OverseerError, TimeoutExt, SpawnNamed, Subsystem, diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index 85df1292e995..462340dd5181 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -18,7 +18,6 @@ use polkadot_node_subsystem::errors::SubsystemError; use ::polkadot_overseer_gen::{ MapSubsystem, SubsystemContext, Subsystem, - OverseerError, SpawnedSubsystem, FromOverseer, }; @@ -165,6 +164,7 @@ impl } } + /// Reference every indidviudal subsystem. pub fn as_ref(&self) -> AllSubsystems<&'_ CV, &'_ CB, &'_ SD, &'_ AD, &'_ AR, &'_ BS, &'_ BD, &'_ P, &'_ RA, &'_ AS, &'_ NB, &'_ CA, &'_ CG, &'_ CP, &'_ ApD, &'_ ApV, &'_ GS> { AllSubsystems { candidate_validation: &self.candidate_validation, @@ -187,6 +187,7 @@ impl } } + /// Map each subsystem. pub fn map_subsystems(self, mapper: Mapper) -> AllSubsystems< >::Output, diff --git a/node/subsystem-types/src/errors.rs b/node/subsystem-types/src/errors.rs index 5097435f2766..de2ca01b658a 100644 --- a/node/subsystem-types/src/errors.rs +++ b/node/subsystem-types/src/errors.rs @@ -117,7 +117,7 @@ pub enum SubsystemError { /// Generated by the `#[overseer(..)]` proc-macro #[error(transparent)] - Generated(#[from] polkadot_overseer_gen::OverseerError), + Generated(#[from] ::polkadot_overseer_gen::OverseerError), /// Per origin (or subsystem) annotations to wrap an error. #[error("Error originated in {origin}")] diff --git a/node/subsystem-types/src/messages/network_bridge_event.rs b/node/subsystem-types/src/messages/network_bridge_event.rs index 864f3b9bada1..84f2d69479ae 100644 --- a/node/subsystem-types/src/messages/network_bridge_event.rs +++ b/node/subsystem-types/src/messages/network_bridge_event.rs @@ -78,8 +78,6 @@ impl NetworkBridgeEvent { => NetworkBridgeEvent::PeerDisconnected(peer.clone()), NetworkBridgeEvent::NewGossipTopology(ref peers) => NetworkBridgeEvent::NewGossipTopology(peers.clone()), - NetworkBridgeEvent::PeerMessage(ref peer, ref msg) - => NetworkBridgeEvent::PeerMessage(peer.clone(), <&'a T>::try_from(msg)?.clone()), NetworkBridgeEvent::PeerViewChange(ref peer, ref view) => NetworkBridgeEvent::PeerViewChange(peer.clone(), view.clone()), NetworkBridgeEvent::OurViewChange(ref view) diff --git a/node/subsystem-util/src/determine_new_blocks.rs b/node/subsystem-util/src/determine_new_blocks.rs index 1a92d2f20a08..6379d01c3570 100644 --- a/node/subsystem-util/src/determine_new_blocks.rs +++ b/node/subsystem-util/src/determine_new_blocks.rs @@ -20,7 +20,6 @@ use polkadot_node_subsystem::{ messages::ChainApiMessage, }; use polkadot_node_subsystem::{ - messages::AllMessages, SubsystemSender, }; use polkadot_primitives::v1::{Hash, Header, BlockNumber}; diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 7d361d88de04..1e3943c65755 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -38,7 +38,6 @@ use polkadot_node_subsystem::{ SubsystemSender, errors::{ SubsystemError, - SubsystemResult, }, SubsystemContext, SpawnedSubsystem, @@ -59,14 +58,13 @@ pub use polkadot_node_metrics::{ use polkadot_node_jaeger as jaeger; use futures::{channel::{mpsc, oneshot}, prelude::*, select, stream::{Stream, SelectAll}}; -use futures_timer::Delay; use parity_scale_codec::Encode; use pin_project::pin_project; use polkadot_primitives::v1::{ CandidateEvent, CommittedCandidateReceipt, CoreState, EncodeAs, PersistedValidationData, GroupRotationInfo, Hash, Id as ParaId, OccupiedCoreAssumption, SessionIndex, Signed, SigningContext, ValidationCode, ValidatorId, ValidatorIndex, SessionInfo, - AuthorityDiscoveryId, GroupIndex, +AuthorityDiscoveryId, GroupIndex, }; use sp_core::{traits::SpawnNamed, Public}; use sp_application_crypto::AppKey; diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index 1c012e965561..68bfdedd7ffa 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -21,16 +21,10 @@ use polkadot_primitives::v1::{Hash, Header, SessionInfo, SessionIndex}; -use polkadot_overseer::AllMessages; use polkadot_node_subsystem::{ messages::{RuntimeApiMessage, RuntimeApiRequest}, errors::RuntimeApiError, SubsystemContext, - SubsystemSender, - SubsystemInstance, - SubsystemResult, - SpawnedSubsystem, - FromOverseer, }; use futures::channel::oneshot; @@ -166,10 +160,10 @@ impl RollingSessionWindow { // The genesis is guaranteed to be at the beginning of the session and its parent state // is non-existent. Therefore if we're at the genesis, we request using its state and // not the parent. - ctx.send_message(AllMessages::RuntimeApiMessage(RuntimeApiMessage::Request( + ctx.send_message(RuntimeApiMessage::Request( if block_header.number == 0 { block_hash } else { block_header.parent_hash }, RuntimeApiRequest::SessionIndexForChild(s_tx), - ))).await; + )).await; match s_rx.await { Ok(Ok(s)) => s, @@ -278,10 +272,10 @@ async fn load_all_sessions( let mut v = Vec::new(); for i in start..=end_inclusive { let (tx, rx)= oneshot::channel(); - ctx.send_message(AllMessages::from(RuntimeApiMessage::Request( + ctx.send_message(RuntimeApiMessage::Request( block_hash, RuntimeApiRequest::SessionInfo(i, tx), - ))).await; + )).await; let session_info = match rx.await { Ok(Ok(Some(s))) => s, diff --git a/node/subsystem-util/src/runtime/mod.rs b/node/subsystem-util/src/runtime/mod.rs index 8bed1a2c0f43..882ecd122cb7 100644 --- a/node/subsystem-util/src/runtime/mod.rs +++ b/node/subsystem-util/src/runtime/mod.rs @@ -24,11 +24,7 @@ use sp_core::crypto::Public; use sp_keystore::{CryptoStore, SyncCryptoStorePtr}; use polkadot_primitives::v1::{CoreState, EncodeAs, GroupIndex, GroupRotationInfo, Hash, OccupiedCore, SessionIndex, SessionInfo, Signed, SigningContext, UncheckedSigned, ValidatorId, ValidatorIndex}; -use polkadot_node_subsystem::{ - messages::RuntimeApiMessage, - OverseerSignal, - SubsystemContext, -}; +use polkadot_node_subsystem::SubsystemContext; use crate::{ diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs index ccca60eed373..10eb7436716d 100644 --- a/node/subsystem-util/src/tests.rs +++ b/node/subsystem-util/src/tests.rs @@ -18,9 +18,8 @@ use super::*; use executor::block_on; use thiserror::Error; use polkadot_node_jaeger as jaeger; -use polkadot_overseer::AllMessages; use polkadot_node_subsystem::{ - messages::{CollatorProtocolMessage}, ActiveLeavesUpdate, FromOverseer, OverseerSignal, + messages::{AllMessages, CollatorProtocolMessage}, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, ActivatedLeaf, LeafStatus, }; use assert_matches::assert_matches; diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 04bb503b7cdd..303a749faa4c 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -23,26 +23,16 @@ pub use polkadot_node_jaeger as jaeger; pub use jaeger::*; -use std::sync::Arc; -use std::fmt; -use smallvec::SmallVec; -use futures::prelude::*; -use futures::channel::{oneshot, mpsc}; use futures::future::BoxFuture; use polkadot_node_subsystem_types::errors::*; pub use polkadot_overseer::{OverseerSignal, ActiveLeavesUpdate, self as overseer}; -use polkadot_primitives::v1::{Hash, BlockNumber}; -/// How many slots are stack-reserved for active leaves updates -/// -/// If there are fewer than this number of slots, then we've wasted some stack space. -/// If there are greater than this number of slots, then we fall back to a heap vector. -const ACTIVE_LEAVES_SMALLVEC_CAPACITY: usize = 8; pub use polkadot_node_subsystem_types::{ errors::{self, *}, ActivatedLeaf, LeafStatus, }; + pub mod messages { pub use super::overseer::AllMessages; pub use polkadot_node_subsystem_types::messages::*; From cff421e99a28d3140bd82bdaaac07380381dd267 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 5 Jul 2021 15:54:48 +0200 Subject: [PATCH 104/161] avoid instantiating dead subsystems --- .../overseer-gen/proc-macro/src/impl_builder.rs | 14 +++++++------- .../proc-macro/src/impl_channels_out.rs | 15 ++++++++++++--- .../overseer-gen/proc-macro/src/parse_struct.rs | 8 -------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index bb8fb5432f02..c1fb671fdab7 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -28,17 +28,17 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Result { let message_wrapper = info.message_wrapper.clone(); - let channel_name = &info.channel_names(""); - let channel_name_unbounded = &info.channel_names("_unbounded"); + let channel_name = &info.channel_names_without_wip(""); + let channel_name_unbounded = &info.channel_names_without_wip("_unbounded"); - let consumes = &info.consumes(); + let consumes = &info.consumes_without_wip(); + let unconsumes = &info.consumes_only_wip(); let ts = quote! { /// Collection of channels to the individual subsystems. @@ -66,6 +67,10 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Ok(()), + )* }; if res.is_err() { @@ -93,6 +98,10 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Ok(()), + )* }; if res.is_err() { diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 552079995dbf..3f78e7ab65b2 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -159,14 +159,6 @@ impl OverseerInfo { self.subsystems.as_slice() } - pub(crate) fn subsystem_names(&self) -> Vec { - self.subsystems - .iter() - .filter(|ssf| !ssf.wip) - .map(|ssf| ssf.name.clone()) - .collect::>() - } - pub(crate) fn subsystem_names_without_wip(&self) -> Vec { self.subsystems .iter() From 117497a5e23941409ff12290b8796e03d99e7ce2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 5 Jul 2021 17:16:04 +0200 Subject: [PATCH 105/161] minro --- node/overseer/overseer-gen/proc-macro/src/parse_struct.rs | 8 +------- node/subsystem/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 3f78e7ab65b2..641d255fb8fc 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -212,13 +212,6 @@ impl OverseerInfo { .collect::>() } - pub(crate) fn channel_names(&self, suffix: &'static str) -> Vec { - self.subsystems - .iter() - .map(|ssf| Ident::new(&(ssf.name.to_string() + suffix), ssf.name.span())) - .collect::>() - } - pub(crate) fn consumes(&self) -> Vec { self.subsystems.iter().map(|ssf| ssf.consumes.clone()).collect::>() } @@ -238,6 +231,7 @@ impl OverseerInfo { .map(|ssf| ssf.consumes.clone()) .collect::>() } + pub(crate) fn consumes_only_wip(&self) -> Vec { self.subsystems .iter() diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 303a749faa4c..8cd80c2a1192 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -20,10 +20,10 @@ #![warn(missing_docs)] +use overseer::Overseer; pub use polkadot_node_jaeger as jaeger; pub use jaeger::*; -use futures::future::BoxFuture; use polkadot_node_subsystem_types::errors::*; pub use polkadot_overseer::{OverseerSignal, ActiveLeavesUpdate, self as overseer}; From 54c87e678916b581e4a82d55e2343d3416efaede Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 5 Jul 2021 18:19:51 +0200 Subject: [PATCH 106/161] silence warnings --- node/core/approval-voting/src/import.rs | 1 - node/overseer/src/lib.rs | 63 ++++++++++++------------- node/subsystem/src/lib.rs | 15 +++--- 3 files changed, 40 insertions(+), 39 deletions(-) diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index 17f60678b289..64ce368fb717 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -33,7 +33,6 @@ use polkadot_node_subsystem::{ messages::{ RuntimeApiMessage, RuntimeApiRequest, ChainApiMessage, ApprovalDistributionMessage, ApprovalVotingMessage, }, - SubsystemSender, SubsystemContext, SubsystemError, SubsystemResult, }; use polkadot_node_subsystem_util::determine_new_blocks; diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index f0253cb1c185..24a59e7910ba 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -420,38 +420,6 @@ pub struct Overseer { pub metrics: Metrics, } - -// Additional `From` implementations, in order to deal with incoming network messages. -// Kept out of the proc macro, for sake of simplicity reduce the need to make even -// more types to the proc macro logic. - -use polkadot_node_network_protocol::{ - request_response::{request::IncomingRequest, v1 as req_res_v1, Requests}, - PeerId, UnifiedReputationChange, -}; - -impl From> for AllMessages { - fn from(req: IncomingRequest) -> Self { - From::::from(From::from(req)) - } -} -impl From> for AllMessages { - fn from(req: IncomingRequest) -> Self { - From::::from(From::from(req)) - } -} -impl From> for AllMessages { - fn from(req: IncomingRequest) -> Self { - From::::from(From::from(req)) - } -} -impl From> for AllMessages { - fn from(req: IncomingRequest) -> Self { - From::::from(From::from(req)) - } -} - - impl Overseer where SupportsParachains: HeadSupportsParachains, @@ -881,5 +849,36 @@ where + +// Additional `From` implementations, in order to deal with incoming network messages. +// Kept out of the proc macro, for sake of simplicity reduce the need to make even +// more types to the proc macro logic. + +use polkadot_node_network_protocol::{ + request_response::{request::IncomingRequest, v1 as req_res_v1}, +}; + +impl From> for AllMessages { + fn from(req: IncomingRequest) -> Self { + From::::from(From::from(req)) + } +} +impl From> for AllMessages { + fn from(req: IncomingRequest) -> Self { + From::::from(From::from(req)) + } +} +impl From> for AllMessages { + fn from(req: IncomingRequest) -> Self { + From::::from(From::from(req)) + } +} +impl From> for AllMessages { + fn from(req: IncomingRequest) -> Self { + From::::from(From::from(req)) + } +} + + #[cfg(test)] mod tests; diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index 8cd80c2a1192..f4e76dc1a276 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -20,11 +20,9 @@ #![warn(missing_docs)] -use overseer::Overseer; pub use polkadot_node_jaeger as jaeger; pub use jaeger::*; -use polkadot_node_subsystem_types::errors::*; pub use polkadot_overseer::{OverseerSignal, ActiveLeavesUpdate, self as overseer}; pub use polkadot_node_subsystem_types::{ @@ -33,6 +31,7 @@ pub use polkadot_node_subsystem_types::{ LeafStatus, }; +/// Re-export of all messages type, including the wrapper type. pub mod messages { pub use super::overseer::AllMessages; pub use polkadot_node_subsystem_types::messages::*; @@ -47,30 +46,34 @@ pub type SubsystemResult = Result; // Simplify usage without having to do large scale modifications of all // subsystems at once. -pub type FromOverseer = polkadot_overseer::gen::FromOverseer; +/// Specialized message type originating from the overseer. +pub type FromOverseer = polkadot_overseer::gen::FromOverseer; +/// Specialized subsystem instance type of subsystems consuming a particular message type. pub type SubsystemInstance = polkadot_overseer::gen::SubsystemInstance; -// Same for traits +/// Sender trait for the `AllMessages` wrapper. pub trait SubsystemSender: polkadot_overseer::gen::SubsystemSender { } impl SubsystemSender for T where T: polkadot_overseer::gen::SubsystemSender { } -// pub use polkadot_overseer::gen::SubsystemSender; - +/// Spawned subsystem. pub type SpawnedSubsystem = polkadot_overseer::gen::SpawnedSubsystem; +/// Convenience trait specialization. pub trait SubsystemContext: polkadot_overseer::gen::SubsystemContext< Signal=OverseerSignal, AllMessages=messages::AllMessages, Error=SubsystemError, > { + /// The message type the subsystem consumes. type Message: std::fmt::Debug + Send + 'static; + /// Sender type to communicate with other subsystems. type Sender: SubsystemSender + Send + Clone + 'static; } From 417daffb24f6e1b25606be171645cb2199c8e449 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jul 2021 13:25:09 +0200 Subject: [PATCH 107/161] avoid duplicate *Message suffix --- node/collation-generation/src/lib.rs | 2 +- .../src/requester/fetch_task/mod.rs | 4 +- node/network/bridge/src/lib.rs | 2 +- .../network/statement-distribution/src/lib.rs | 10 ++--- .../proc-macro/src/impl_channels_out.rs | 12 +++--- .../proc-macro/src/impl_dispatch.rs | 19 +++++--- .../proc-macro/src/impl_message_wrapper.rs | 16 ++----- .../proc-macro/src/impl_overseer.rs | 7 +-- .../proc-macro/src/parse_struct.rs | 43 +++++++++++++------ 9 files changed, 68 insertions(+), 47 deletions(-) diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index 735a4697a184..5b020a18e83d 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -404,7 +404,7 @@ async fn handle_new_activations( ); metrics.on_collation_generated(); - if let Err(err) = task_sender.send(AllMessages::CollatorProtocolMessage( + if let Err(err) = task_sender.send(AllMessages::CollatorProtocol( CollatorProtocolMessage::DistributeCollation(ccr, pov, result_sender) )).await { tracing::warn!( diff --git a/node/network/availability-distribution/src/requester/fetch_task/mod.rs b/node/network/availability-distribution/src/requester/fetch_task/mod.rs index 450abcc98e6e..51835fcd167d 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/mod.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/mod.rs @@ -336,7 +336,7 @@ impl RunningTask { let requests = Requests::ChunkFetching(full_request); self.sender - .send(FromFetchTask::Message(AllMessages::NetworkBridgeMessage( + .send(FromFetchTask::Message(AllMessages::NetworkBridge( NetworkBridgeMessage::SendRequests(vec![requests], IfDisconnected::TryConnect) ))) .await @@ -399,7 +399,7 @@ impl RunningTask { let (tx, rx) = oneshot::channel(); let r = self .sender - .send(FromFetchTask::Message(AllMessages::AvailabilityStoreMessage( + .send(FromFetchTask::Message(AllMessages::AvailabilityStore( AvailabilityStoreMessage::StoreChunk { candidate_hash: self.request.candidate_hash, chunk, diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 0eecc86375cd..f4616cb382b8 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -1158,7 +1158,7 @@ async fn dispatch_collation_events_to_all( I::IntoIter: Send, { let messages_for = |event: NetworkBridgeEvent| { - event.focus().ok().map(|m| AllMessages::CollatorProtocolMessage( + event.focus().ok().map(|m| AllMessages::CollatorProtocol( CollatorProtocolMessage::NetworkBridgeUpdateV1(m) )) }; diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index 56fafc477bd9..0060f8fa4ebc 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -1020,7 +1020,7 @@ async fn circulate_statement<'a>( statement = ?stored.statement, "Sending statement", ); - ctx.send_message(AllMessages::NetworkBridgeMessage(NetworkBridgeMessage::SendValidationMessage( + ctx.send_message(AllMessages::NetworkBridge(NetworkBridgeMessage::SendValidationMessage( peers_to_send.iter().map(|(p, _)| p.clone()).collect(), payload, ))).await; @@ -1062,7 +1062,7 @@ async fn send_statements_about( statement = ?statement.statement, "Sending statement", ); - ctx.send_message(AllMessages::NetworkBridgeMessage( + ctx.send_message(AllMessages::NetworkBridge( NetworkBridgeMessage::SendValidationMessage(vec![peer.clone()], payload) )).await; @@ -1097,7 +1097,7 @@ async fn send_statements( statement = ?statement.statement, "Sending statement" ); - ctx.send_message(AllMessages::NetworkBridgeMessage( + ctx.send_message(AllMessages::NetworkBridge( NetworkBridgeMessage::SendValidationMessage(vec![peer.clone()], payload) )).await; @@ -1110,7 +1110,7 @@ async fn report_peer( peer: PeerId, rep: Rep, ) { - ctx.send_message(AllMessages::NetworkBridgeMessage( + ctx.send_message(AllMessages::NetworkBridge( NetworkBridgeMessage::ReportPeer(peer, rep) )).await } @@ -1774,7 +1774,7 @@ impl StatementDistribution { } RequesterMessage::SendRequest(req) => { ctx.send_message( - AllMessages::NetworkBridgeMessage( + AllMessages::NetworkBridge( NetworkBridgeMessage::SendRequests( vec![req], IfDisconnected::ImmediateError, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index f7e65692fe1b..060d266fd1eb 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -27,7 +27,9 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Result = match message { #( - #message_wrapper :: #consumes ( inner ) => { + #message_wrapper :: #consumes_variant ( inner ) => { self. #channel_name .send( ::polkadot_overseer_gen::make_packet(signals_received, inner) ).await @@ -69,7 +71,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Ok(()), + #message_wrapper :: #unconsumes_variant ( _ ) => Ok(()), )* }; @@ -91,7 +93,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result = match message { #( - #message_wrapper :: #consumes (inner) => { + #message_wrapper :: #consumes_variant (inner) => { self. #channel_name_unbounded .unbounded_send( ::polkadot_overseer_gen::make_packet(signals_received, inner) ) @@ -100,7 +102,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Ok(()), + #message_wrapper :: #unconsumes_variant ( _ ) => Ok(()), )* }; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index fbb215a4656d..480c5b3bd0c0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -15,17 +15,26 @@ // along with Polkadot. If not, see . use super::*; -use proc_macro2::TokenStream; +use proc_macro2::{TokenStream, Ident}; use quote::quote; use syn::{Path, Result}; pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { let message_wrapper = &info.message_wrapper; - let dispatchable = info + let dispatchable_variant = info .subsystems() .into_iter() .filter(|ssf| !ssf.no_dispatch) + .filter(|ssf| !ssf.wip) + .map(|ssf| ssf.generic.clone()) + .collect::>(); + + let dispatchable_message = info + .subsystems() + .into_iter() + .filter(|ssf| !ssf.no_dispatch) + .filter(|ssf| !ssf.wip) .map(|ssf| ssf.consumes.clone()) .collect::>(); @@ -46,10 +55,10 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { // TODO do not require this to be hardcoded, either externalize or ... .focus() .ok() - .map(|event: NetworkBridgeEvent<_>| { - #message_wrapper :: #dispatchable ( + .map(|event| { + #message_wrapper :: #dispatchable_variant ( // the inner type of the enum variant - #dispatchable :: from( event ) + #dispatchable_message :: from( event ) ) }) ) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 88f6d6ccc472..56d2acdb8615 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -1,6 +1,3 @@ -use quote::quote; -use syn::Result; -use syn::Path; // Copyright 2021 Parity Technologies (UK) Ltd. // This file is part of Polkadot. @@ -17,23 +14,16 @@ use syn::Path; // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use quote::quote; +use syn::Result; use syn::spanned::Spanned; -use proc_macro2::Ident; use super::*; /// Generates the wrapper type enum. pub(crate) fn impl_message_wrapper_enum(info: &OverseerInfo) -> Result { let consumes = info.consumes(); - let consumes_variant = info.consumes() - .into_iter() - .try_fold(Vec::new(), |mut acc: Vec, path: Path| { - let ident = path.get_ident().ok_or_else(||{ - syn::Error::new(path.span(), "Missing identifier to use as enum variant.") - })?; - acc.push(ident.clone()); - Ok::<_, syn::Error>(acc) - })?; + let consumes_variant = info.variant_names(); let outgoing = &info.outgoing_ty; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 2db6e97b9783..a0d6c204080a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -39,7 +39,8 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result Result ::std::result::Result<(), #error_ty > { match message { #( - #message_wrapper :: #consumes ( inner ) => + #message_wrapper :: #consumes_variant ( inner ) => OverseenSubsystem::< #consumes >::send_message2(&mut self. #subsystem_name, inner, origin ).await?, )* // subsystems that are still work in progress #( - #message_wrapper :: #unconsumes ( inner ) => {} + #message_wrapper :: #unconsumes_variant ( _ ) => {} )* } Ok(()) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 641d255fb8fc..889f319febf3 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -37,7 +37,9 @@ use syn::{Error, GenericParam, ItemStruct, Result}; pub(crate) struct SubSysField { /// Name of the field. pub(crate) name: Ident, - /// Generate generic type name for the `AllSubsystems` type. + /// Generate generic type name for the `AllSubsystems` type + /// which is also used `#wrapper_message :: #variant` variant + /// part. pub(crate) generic: Ident, /// Type of the subsystem. pub(crate) ty: Path, @@ -155,6 +157,29 @@ pub(crate) struct OverseerInfo { } impl OverseerInfo { + pub(crate) fn variant_names(&self) -> Vec { + self.subsystems + .iter() + .map(|ssf| ssf.generic.clone()) + .collect::>() + } + + pub(crate) fn variant_names_without_wip(&self) -> Vec { + self.subsystems + .iter() + .filter(|ssf| !ssf.wip) + .map(|ssf| ssf.generic.clone()) + .collect::>() + } + + pub(crate) fn variant_names_only_wip(&self) -> Vec { + self.subsystems + .iter() + .filter(|ssf| ssf.wip) + .map(|ssf| ssf.generic.clone()) + .collect::>() + } + pub(crate) fn subsystems(&self) -> &[SubSysField] { self.subsystems.as_slice() } @@ -231,14 +256,6 @@ impl OverseerInfo { .map(|ssf| ssf.consumes.clone()) .collect::>() } - - pub(crate) fn consumes_only_wip(&self) -> Vec { - self.subsystems - .iter() - .filter(|ssf| ssf.wip) - .map(|ssf| ssf.consumes.clone()) - .collect::>() - } } /// Internals of the overseer. @@ -254,7 +271,7 @@ impl OverseerGuts { let n = fields.named.len(); let mut subsystems = Vec::with_capacity(n); let mut baggage = Vec::with_capacity(n); - for (idx, Field { attrs, vis, ident, ty, .. }) in fields.named.into_iter().enumerate() { + for Field { attrs, vis, ident, ty, .. } in fields.named.into_iter() { let mut consumes = attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| { let span = attr.path.span(); attr.path.get_ident().filter(|ident| *ident == "subsystem").map(move |_ident| { @@ -284,10 +301,12 @@ impl OverseerGuts { return Err(Error::new(span, "Subsystem must consume at least one message")); } + let ty = try_type_to_path(ty, span)?; + let generic = ty.get_ident().ok_or_else(|| Error::new(ty.span(), "Must be a identifier, not a path."))?.clone(); subsystems.push(SubSysField { name: ident, - generic: Ident::new(format!("Sub{}", idx).as_str(), span), - ty: try_type_to_path(ty, span)?, + generic, + ty, consumes: consumes_paths[0].clone(), no_dispatch: variant.no_dispatch, wip: variant.wip, From 6a89fd048971a17042f57e75bfb663821e77751b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jul 2021 13:38:50 +0200 Subject: [PATCH 108/161] fix candidate validation tests --- node/core/candidate-validation/Cargo.toml | 2 +- node/core/candidate-validation/src/lib.rs | 2 +- node/core/candidate-validation/src/tests.rs | 1 + node/overseer/overseer-gen/src/lib.rs | 2 +- node/subsystem-test-helpers/src/lib.rs | 31 +++++++++++++++------ 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/node/core/candidate-validation/Cargo.toml b/node/core/candidate-validation/Cargo.toml index a109cb590510..d9ccae79bd85 100644 --- a/node/core/candidate-validation/Cargo.toml +++ b/node/core/candidate-validation/Cargo.toml @@ -15,7 +15,7 @@ parity-scale-codec = { version = "2.0.0", default-features = false, features = [ polkadot-primitives = { path = "../../../primitives" } polkadot-parachain = { path = "../../../parachain" } polkadot-node-primitives = { path = "../../primitives" } -polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } +polkadot-node-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } [target.'cfg(not(any(target_os = "android", target_os = "unknown")))'.dependencies] diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index d986fcc876b2..4c3df0c4fbc9 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -23,7 +23,7 @@ #![deny(unused_crate_dependencies, unused_results)] #![warn(missing_docs)] -use polkadot_subsystem::{ +use polkadot_node_subsystem::{ overseer, SubsystemContext, SpawnedSubsystem, SubsystemResult, SubsystemError, FromOverseer, OverseerSignal, diff --git a/node/core/candidate-validation/src/tests.rs b/node/core/candidate-validation/src/tests.rs index 26ccdbb7cc9e..c718122a63e7 100644 --- a/node/core/candidate-validation/src/tests.rs +++ b/node/core/candidate-validation/src/tests.rs @@ -15,6 +15,7 @@ // along with Polkadot. If not, see . use super::*; +use polkadot_node_subsystem::messages::AllMessages; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_primitives::v1::{HeadData, UpwardMessage}; use sp_core::testing::TaskExecutor; diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 3b62bd889252..8d4b8398c70c 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -368,7 +368,7 @@ pub trait SubsystemContext: Send + 'static { /// In some cases can be identical to `Self::Message`. type AllMessages: From + Send + 'static; /// The sender type as provided by `sender()` and underlying. - type Sender: SubsystemSender + std::fmt::Debug + Send; + type Sender: SubsystemSender + Send + 'static; /// The error type. type Error: ::std::error::Error + ::std::convert::From< OverseerError > + Sync + Send + 'static; diff --git a/node/subsystem-test-helpers/src/lib.rs b/node/subsystem-test-helpers/src/lib.rs index d2c7c6bda7d3..66bfa4cefa80 100644 --- a/node/subsystem-test-helpers/src/lib.rs +++ b/node/subsystem-test-helpers/src/lib.rs @@ -19,8 +19,10 @@ #![warn(missing_docs)] use polkadot_node_subsystem::{ - FromOverseer, SubsystemContext, SubsystemError, SubsystemResult, Subsystem, - SpawnedSubsystem, OverseerSignal, SubsystemSender, + messages::AllMessages, + overseer, + FromOverseer, SubsystemContext, SubsystemError, SubsystemResult, + SpawnedSubsystem, OverseerSignal, }; use polkadot_node_subsystem_util::TimeoutExt; @@ -171,7 +173,7 @@ pub fn sender_receiver() -> (TestSubsystemSender, mpsc::UnboundedReceiver for TestSubsystemSender { async fn send_message(&mut self, msg: AllMessages) { self.tx .send(msg) @@ -204,11 +206,18 @@ pub struct TestSubsystemContext { } #[async_trait::async_trait] -impl SubsystemContext +impl overseer::SubsystemContext for TestSubsystemContext +where + M: std::fmt::Debug + Send + 'static, + AllMessages: From, + S: SpawnNamed + Send + 'static, { type Message = M; type Sender = TestSubsystemSender; + type Signal = OverseerSignal; + type AllMessages = AllMessages; + type Error = SubsystemError; async fn try_recv(&mut self) -> Result>, ()> { match poll!(self.rx.next()) { @@ -223,7 +232,7 @@ impl SubsystemContext .ok_or_else(|| SubsystemError::Context("Receiving end closed".to_owned())) } - fn spawn( + async fn spawn( &mut self, name: &'static str, s: Pin + Send>>, @@ -232,7 +241,7 @@ impl SubsystemContext Ok(()) } - fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) + async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) -> SubsystemResult<()> { self.spawn.spawn_blocking(name, s); @@ -332,10 +341,14 @@ pub fn subsystem_test_harness( /// channel. /// /// This subsystem is useful for testing functionality that interacts with the overseer. -pub struct ForwardSubsystem(pub mpsc::Sender); +pub struct ForwardSubsystem(pub mpsc::Sender); -impl, Msg: Send + 'static> Subsystem for ForwardSubsystem { - fn start(mut self, mut ctx: C) -> SpawnedSubsystem { +impl overseer::Subsystem for ForwardSubsystem +where + M: std::fmt::Debug + Send + 'static, + Context: SubsystemContext + overseer::SubsystemContext, +{ + fn start(mut self, mut ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { loop { match ctx.recv().await { From db267b63a90439db5cf49e46c459f6c5399b2e62 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jul 2021 13:44:06 +0200 Subject: [PATCH 109/161] fix dispute participation --- node/core/dispute-participation/src/lib.rs | 13 ++++++------- node/core/dispute-participation/src/tests.rs | 3 ++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/node/core/dispute-participation/src/lib.rs b/node/core/dispute-participation/src/lib.rs index 373b2b4ea806..19827cab4120 100644 --- a/node/core/dispute-participation/src/lib.rs +++ b/node/core/dispute-participation/src/lib.rs @@ -26,12 +26,13 @@ use futures::prelude::*; use polkadot_node_primitives::ValidationResult; use polkadot_node_subsystem::{ errors::{RecoveryError, RuntimeApiError}, + overseer, messages::{ - AllMessages, AvailabilityRecoveryMessage, AvailabilityStoreMessage, + AvailabilityRecoveryMessage, AvailabilityStoreMessage, CandidateValidationMessage, DisputeCoordinatorMessage, DisputeParticipationMessage, RuntimeApiMessage, RuntimeApiRequest, }, - ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, Subsystem, + ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError, }; use polkadot_primitives::v1::{BlockNumber, CandidateHash, CandidateReceipt, Hash, SessionIndex}; @@ -55,9 +56,10 @@ impl DisputeParticipationSubsystem { } } -impl Subsystem for DisputeParticipationSubsystem +impl overseer::Subsystem for DisputeParticipationSubsystem where Context: SubsystemContext, + Context: overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run(ctx).map(|_| Ok(())).boxed(); @@ -111,6 +113,7 @@ impl Error { async fn run(mut ctx: Context) where Context: SubsystemContext, + Context: overseer::SubsystemContext, { let mut state = State { recent_block: None }; @@ -196,7 +199,6 @@ async fn participate( None, recover_available_data_tx, ) - .into(), ) .await; @@ -223,7 +225,6 @@ async fn participate( code_tx, ), ) - .into(), ) .await; @@ -252,7 +253,6 @@ async fn participate( available_data.clone(), store_available_data_tx, ) - .into(), ) .await; @@ -277,7 +277,6 @@ async fn participate( available_data.pov, validation_tx, ) - .into(), ) .await; diff --git a/node/core/dispute-participation/src/tests.rs b/node/core/dispute-participation/src/tests.rs index 2b086c43d179..a56e204b0551 100644 --- a/node/core/dispute-participation/src/tests.rs +++ b/node/core/dispute-participation/src/tests.rs @@ -24,7 +24,8 @@ use super::*; use parity_scale_codec::Encode; use polkadot_node_primitives::{AvailableData, BlockData, InvalidCandidate, PoV}; use polkadot_node_subsystem::{ - jaeger, messages::ValidationFailed, ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, + overseer::Subsystem, + jaeger, messages::{AllMessages, ValidationFailed}, ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, }; use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle}; use polkadot_primitives::v1::{BlakeTwo256, CandidateCommitments, HashT, Header, ValidationCode}; From d3b6ab4c0ba1a914619fdfe4fbce5ac33f016d6f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jul 2021 13:48:55 +0200 Subject: [PATCH 110/161] fix dispute coord test --- node/core/dispute-coordinator/src/lib.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index 9d4daeeba7e4..05b51925622a 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -30,10 +30,11 @@ use std::sync::Arc; use polkadot_node_primitives::{CandidateVotes, SignedDisputeStatement}; use polkadot_node_subsystem::{ + overseer, messages::{ DisputeCoordinatorMessage, ChainApiMessage, DisputeParticipationMessage, }, - Subsystem, SubsystemContext, FromOverseer, OverseerSignal, SpawnedSubsystem, + SubsystemContext, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemError, errors::{ChainApiError, RuntimeApiError}, }; @@ -101,8 +102,10 @@ impl DisputeCoordinatorSubsystem { } } -impl Subsystem for DisputeCoordinatorSubsystem - where Context: SubsystemContext +impl overseer::Subsystem for DisputeCoordinatorSubsystem +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run(self, ctx) @@ -160,7 +163,9 @@ impl Error { } async fn run(subsystem: DisputeCoordinatorSubsystem, mut ctx: Context) - where Context: SubsystemContext +where + Context: overseer::SubsystemContext, + Context: SubsystemContext { loop { let res = run_iteration(&mut ctx, &subsystem).await; @@ -187,7 +192,9 @@ async fn run(subsystem: DisputeCoordinatorSubsystem, mut ctx: Context) // lead to another call to this function. async fn run_iteration(ctx: &mut Context, subsystem: &DisputeCoordinatorSubsystem) -> Result<(), Error> - where Context: SubsystemContext +where + Context: overseer::SubsystemContext, + Context: SubsystemContext { let DisputeCoordinatorSubsystem { ref store, ref keystore, ref config } = *subsystem; let mut state = State { @@ -236,7 +243,7 @@ async fn handle_new_activations( let (tx, rx) = oneshot::channel(); ctx.send_message( - ChainApiMessage::BlockHeader(new_leaf, tx).into() + ChainApiMessage::BlockHeader(new_leaf, tx) ).await; match rx.await?? { @@ -488,7 +495,7 @@ async fn handle_import_statements( candidate_receipt, session, n_validators: n_validators as u32, - }.into()).await; + }).await; } if concluded_valid && already_disputed { From 7a31f9316756b706ac56c54c6c1ac9211602604d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jul 2021 14:02:55 +0200 Subject: [PATCH 111/161] fix chain selection test --- node/core/chain-selection/Cargo.toml | 2 +- node/core/chain-selection/src/lib.rs | 20 ++++++++++++-------- node/core/chain-selection/src/tests.rs | 7 +++++-- node/overseer/overseer-gen/src/lib.rs | 6 ++++++ 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/node/core/chain-selection/Cargo.toml b/node/core/chain-selection/Cargo.toml index ee498427ea0d..968700ad859c 100644 --- a/node/core/chain-selection/Cargo.toml +++ b/node/core/chain-selection/Cargo.toml @@ -10,7 +10,7 @@ futures = "0.3.15" tracing = "0.1.26" polkadot-primitives = { path = "../../../primitives" } polkadot-node-primitives = { path = "../../primitives" } -polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" } +polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } kvdb = "0.9.0" thiserror = "1.0.23" diff --git a/node/core/chain-selection/src/lib.rs b/node/core/chain-selection/src/lib.rs index dddfc2590d33..9bab52eab242 100644 --- a/node/core/chain-selection/src/lib.rs +++ b/node/core/chain-selection/src/lib.rs @@ -18,8 +18,8 @@ use polkadot_primitives::v1::{BlockNumber, Hash, Header, ConsensusLog}; use polkadot_node_primitives::BlockWeight; -use polkadot_subsystem::{ - Subsystem, SubsystemContext, SubsystemError, SpawnedSubsystem, +use polkadot_node_subsystem::{ + overseer, SubsystemContext, SubsystemError, SpawnedSubsystem, OverseerSignal, FromOverseer, messages::{ChainSelectionMessage, ChainApiMessage}, errors::ChainApiError, @@ -273,8 +273,10 @@ impl Backend for VoidBackend { /// The chain selection subsystem. pub struct ChainSelectionSubsystem; -impl Subsystem for ChainSelectionSubsystem - where Context: SubsystemContext +impl overseer::Subsystem for ChainSelectionSubsystem +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { let backend = VoidBackend; @@ -288,6 +290,7 @@ impl Subsystem for ChainSelectionSubsystem async fn run(mut ctx: Context, mut backend: B) where Context: SubsystemContext, + Context: overseer::SubsystemContext, B: Backend, { loop { @@ -317,6 +320,7 @@ async fn run_iteration(ctx: &mut Context, backend: &mut B) -> Result<(), Error> where Context: SubsystemContext, + Context: overseer::SubsystemContext, B: Backend, { // TODO https://github.com/paritytech/polkadot/issues/3293: Add stagnant checking timer loop. @@ -371,11 +375,11 @@ async fn fetch_finalized( let (number_tx, number_rx) = oneshot::channel(); let (hash_tx, hash_rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::FinalizedBlockNumber(number_tx).into()).await; + ctx.send_message(ChainApiMessage::FinalizedBlockNumber(number_tx)).await; let number = number_rx.await??; - ctx.send_message(ChainApiMessage::FinalizedBlockHash(number, hash_tx).into()).await; + ctx.send_message(ChainApiMessage::FinalizedBlockHash(number, hash_tx)).await; match hash_rx.await?? { None => { @@ -396,7 +400,7 @@ async fn fetch_header( hash: Hash, ) -> Result, Error> { let (h_tx, h_rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::BlockHeader(hash, h_tx).into()).await; + ctx.send_message(ChainApiMessage::BlockHeader(hash, h_tx)).await; h_rx.await?.map_err(Into::into) } @@ -406,7 +410,7 @@ async fn fetch_block_weight( hash: Hash, ) -> Result, Error> { let (tx, rx) = oneshot::channel(); - ctx.send_message(ChainApiMessage::BlockWeight(hash, tx).into()).await; + ctx.send_message(ChainApiMessage::BlockWeight(hash, tx)).await; rx.await?.map_err(Into::into) } diff --git a/node/core/chain-selection/src/tests.rs b/node/core/chain-selection/src/tests.rs index 945578a47e6e..efe7d970dd74 100644 --- a/node/core/chain-selection/src/tests.rs +++ b/node/core/chain-selection/src/tests.rs @@ -31,8 +31,11 @@ use sp_core::testing::TaskExecutor; use assert_matches::assert_matches; use polkadot_primitives::v1::{BlakeTwo256, HashT, ConsensusLog}; -use polkadot_subsystem::{jaeger, ActiveLeavesUpdate, ActivatedLeaf, LeafStatus}; -use polkadot_subsystem::messages::AllMessages; +use polkadot_node_subsystem::{ + messages::AllMessages, + jaeger, + ActiveLeavesUpdate, ActivatedLeaf, LeafStatus, +}; use polkadot_node_subsystem_test_helpers as test_helpers; #[derive(Default)] diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 8d4b8398c70c..31caca821118 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -350,6 +350,12 @@ pub enum FromOverseer { }, } +impl From for FromOverseer { + fn from(signal: Signal) -> Self { + Self::Signal(signal) + } +} + /// A context type that is given to the [`Subsystem`] upon spawning. /// It can be used by [`Subsystem`] to communicate with other [`Subsystem`]s /// or spawn jobs. From 93786cb02506fbc431cf9f2012f99571b963aac7 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jul 2021 14:07:44 +0200 Subject: [PATCH 112/161] fix availability dist --- node/network/availability-distribution/src/pov_requester/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/network/availability-distribution/src/pov_requester/mod.rs b/node/network/availability-distribution/src/pov_requester/mod.rs index e6aa9cfa55e2..e605ae577db0 100644 --- a/node/network/availability-distribution/src/pov_requester/mod.rs +++ b/node/network/availability-distribution/src/pov_requester/mod.rs @@ -129,7 +129,7 @@ mod tests { use polkadot_primitives::v1::{CandidateHash, Hash, ValidatorIndex}; use polkadot_node_primitives::BlockData; use polkadot_subsystem_testhelpers as test_helpers; - use polkadot_subsystem::messages::{AvailabilityDistributionMessage, RuntimeApiMessage, RuntimeApiRequest}; + use polkadot_subsystem::messages::{AllMessages, AvailabilityDistributionMessage, RuntimeApiMessage, RuntimeApiRequest}; use super::*; use crate::LOG_TARGET; From f0adaf2850c72c018e0286cd538c362507690cc1 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jul 2021 14:08:37 +0200 Subject: [PATCH 113/161] fix approval dist tests --- node/network/approval-distribution/src/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/network/approval-distribution/src/tests.rs b/node/network/approval-distribution/src/tests.rs index 675761ef14b9..a0d55c3371d3 100644 --- a/node/network/approval-distribution/src/tests.rs +++ b/node/network/approval-distribution/src/tests.rs @@ -17,7 +17,7 @@ use std::time::Duration; use futures::{future, Future, executor}; use assert_matches::assert_matches; -use polkadot_node_subsystem::messages::ApprovalCheckError; +use polkadot_node_subsystem::messages::{AllMessages, ApprovalCheckError}; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt as _; use polkadot_node_network_protocol::{view, ObservedRole}; From c18f85c80fe10c58b33008d1f4775ef2edc7e2e2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jul 2021 20:01:44 +0200 Subject: [PATCH 114/161] fixup malus --- Cargo.lock | 16 ---------------- Cargo.toml | 6 ------ node/malus/src/lib.rs | 42 +++++++++++++++++++++++------------------- 3 files changed, 23 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a12f4c6d13d..bbad1103e15a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -540,22 +540,6 @@ dependencies = [ "sp-std", ] -[[package]] -name = "behavior-tests" -version = "0.1.0" -dependencies = [ - "env_logger 0.8.4", - "futures 0.3.15", - "log", - "polkadot-service", - "sc-chain-spec", - "sc-service", - "sp-core", - "test-runner", - "thiserror", - "tokio 0.2.21", -] - [[package]] name = "bincode" version = "1.3.1" diff --git a/Cargo.toml b/Cargo.toml index 410b1140dd52..6b987e621617 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,12 +23,6 @@ tempfile = "3.2.0" [workspace] members = [ - "bridges/primitives/chain-kusama", - "bridges/primitives/chain-polkadot", - "bridges/primitives/chain-rococo", - "bridges/primitives/chain-westend", - "bridges/primitives/runtime", - "behavior-tests", "cli", "core-primitives", "erasure-coding", diff --git a/node/malus/src/lib.rs b/node/malus/src/lib.rs index 1161ced5a570..38982b8964eb 100644 --- a/node/malus/src/lib.rs +++ b/node/malus/src/lib.rs @@ -21,7 +21,7 @@ //! messages on the overseer level. use polkadot_node_subsystem::*; -pub use polkadot_node_subsystem::{messages::AllMessages, FromOverseer}; +pub use polkadot_node_subsystem::{overseer, messages::AllMessages, FromOverseer}; use std::future::Future; use std::pin::Pin; @@ -50,9 +50,9 @@ pub struct FilteredSender { } #[async_trait::async_trait] -impl SubsystemSender for FilteredSender +impl overseer::SubsystemSender for FilteredSender where - Sender: SubsystemSender, + Sender: overseer::SubsystemSender, Fil: MsgFilter, { async fn send_message(&mut self, msg: AllMessages) { @@ -79,19 +79,19 @@ where } /// A subsystem context, that filters the outgoing messages. -pub struct FilteredContext { +pub struct FilteredContext { inner: Context, message_filter: Fil, - sender: FilteredSender<::Sender, Fil>, + sender: FilteredSender<::Sender, Fil>, } impl FilteredContext where - Context: SubsystemContext, - Fil: MsgFilter::Message>, + Context: overseer::SubsystemContext + SubsystemContext, + Fil: MsgFilter::Message>, { pub fn new(mut inner: Context, message_filter: Fil) -> Self { - let sender = FilteredSender::<::Sender, Fil> { + let sender = FilteredSender::<::Sender, Fil> { inner: inner.sender().clone(), message_filter: message_filter.clone(), }; @@ -104,13 +104,17 @@ where } #[async_trait::async_trait] -impl SubsystemContext for FilteredContext +impl overseer::SubsystemContext for FilteredContext where - Context: SubsystemContext, - Fil: MsgFilter::Message>, + Context: overseer::SubsystemContext + SubsystemContext, + Fil: MsgFilter::Message>, + ::AllMessages: From<::Message>, { - type Message = ::Message; - type Sender = FilteredSender<::Sender, Fil>; + type Message = ::Message; + type Sender = FilteredSender<::Sender, Fil>; + type Error = ::Error; + type AllMessages = ::AllMessages; + type Signal = ::Signal; async fn try_recv(&mut self) -> Result>, ()> { loop { @@ -170,15 +174,15 @@ impl FilteredSubsystem { } } -impl Subsystem for FilteredSubsystem +impl overseer::Subsystem for FilteredSubsystem where - Context: SubsystemContext + Sync + Send, - Sub: Subsystem>, - FilteredContext: SubsystemContext, - Fil: MsgFilter::Message>, + Context: overseer::SubsystemContext + SubsystemContext + Sync + Send, + Sub: overseer::Subsystem, SubsystemError>, + FilteredContext: overseer::SubsystemContext + SubsystemContext, + Fil: MsgFilter::Message>, { fn start(self, ctx: Context) -> SpawnedSubsystem { let ctx = FilteredContext::new(ctx, self.message_filter); - Subsystem::>::start(self.subsystem, ctx) + overseer::Subsystem::, SubsystemError>::start(self.subsystem, ctx) } } From 3435ca7343cd6b9d282d4a26995bfbd5c59d4107 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jul 2021 20:02:15 +0200 Subject: [PATCH 115/161] re-add all-subsystems-gen Required for the current instatiation code. --- Cargo.toml | 1 + node/overseer/all-subsystems-gen/Cargo.toml | 18 ++ node/overseer/all-subsystems-gen/src/lib.rs | 179 ++++++++++++++++++ .../tests/ui/err-01-enum.rs | 13 ++ .../tests/ui/err-01-enum.stderr | 5 + .../tests/ui/err-01-generic-used-twice.rs | 17 ++ .../tests/ui/err-01-generic-used-twice.stderr | 14 ++ .../tests/ui/err-01-no-generic.rs | 17 ++ .../tests/ui/err-01-no-generics.stderr | 14 ++ .../tests/ui/ok-01-w-generics.rs | 17 ++ 10 files changed, 295 insertions(+) create mode 100644 node/overseer/all-subsystems-gen/Cargo.toml create mode 100644 node/overseer/all-subsystems-gen/src/lib.rs create mode 100644 node/overseer/all-subsystems-gen/tests/ui/err-01-enum.rs create mode 100644 node/overseer/all-subsystems-gen/tests/ui/err-01-enum.stderr create mode 100644 node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.rs create mode 100644 node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.stderr create mode 100644 node/overseer/all-subsystems-gen/tests/ui/err-01-no-generic.rs create mode 100644 node/overseer/all-subsystems-gen/tests/ui/err-01-no-generics.stderr create mode 100644 node/overseer/all-subsystems-gen/tests/ui/ok-01-w-generics.rs diff --git a/Cargo.toml b/Cargo.toml index 6b987e621617..f03e91485d2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ members = [ "node/overseer", "node/overseer/overseer-gen", "node/overseer/overseer-gen/proc-macro", + "node/overseer/all-subsystems-gen", "node/malus", "node/primitives", "node/service", diff --git a/node/overseer/all-subsystems-gen/Cargo.toml b/node/overseer/all-subsystems-gen/Cargo.toml new file mode 100644 index 000000000000..957a086cf2f3 --- /dev/null +++ b/node/overseer/all-subsystems-gen/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "polkadot-overseer-all-subsystems-gen" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2018" +description = "Small proc macro to create mocking level iface type helpers" + +[lib] +proc-macro = true + +[dependencies] +syn = { version = "1.0.60", features = ["full", "extra-traits"] } +quote = "1.0.9" +proc-macro2 = "1.0.24" +assert_matches = "1.5.0" + +[dev-dependencies] +trybuild = "1.0.42" diff --git a/node/overseer/all-subsystems-gen/src/lib.rs b/node/overseer/all-subsystems-gen/src/lib.rs new file mode 100644 index 000000000000..c15d08bb04f4 --- /dev/null +++ b/node/overseer/all-subsystems-gen/src/lib.rs @@ -0,0 +1,179 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use std::collections::HashSet; + +use proc_macro2::TokenStream; +use quote::quote; + +use syn::{Error, GenericParam, Ident, Result, Type, parse2}; + +#[proc_macro_derive(AllSubsystemsGen)] +pub fn subsystems_gen(item: proc_macro::TokenStream) -> proc_macro::TokenStream { + let item: TokenStream = item.into(); + impl_subsystems_gen(item).unwrap_or_else(|err| err.to_compile_error()).into() +} + +fn impl_subsystems_gen(item: TokenStream) -> Result { + let span = proc_macro2::Span::call_site(); + let ds = parse2::(item.clone())?; + + match ds.fields { + syn::Fields::Named(named) => { + #[derive(Clone)] + struct NameTyTup { + field: Ident, + ty: Type, + } + let mut orig_generics = ds.generics; + // remove default types + orig_generics.params = orig_generics.params.into_iter().map(|mut generic| { + match generic { + GenericParam::Type(ref mut param) => { + param.eq_token = None; + param.default = None; + } + _ => {} + } + generic + }).collect(); + + // prepare a hashmap of generic type to member that uses it + let generic_types = orig_generics.params.iter().filter_map(|generic| { + if let GenericParam::Type(param) = generic { + Some(param.ident.clone()) + } else { + None + } + }).collect::>(); + + let strukt_ty = ds.ident; + + if generic_types.is_empty() { + return Err(Error::new(strukt_ty.span(), "struct must have at least one generic parameter.")) + } + + // collect all fields that exist, and all fields that are replaceable + let mut replacable_items = Vec::::with_capacity(64); + let mut all_fields = replacable_items.clone(); + + + let mut duplicate_generic_detection = HashSet::::with_capacity(64); + + for field in named.named { + let field_ident = field.ident.clone().ok_or_else(|| Error::new(span, "Member field must have a name."))?; + let ty = field.ty.clone(); + let ntt = NameTyTup { field: field_ident, ty }; + + replacable_items.push(ntt.clone()); + + + // assure every generic is used exactly once + let ty_ident = match field.ty { + Type::Path(path) => path.path.get_ident().cloned().ok_or_else(|| { + Error::new(proc_macro2::Span::call_site(), "Expected an identifier, but got a path.") + }), + _ => return Err(Error::new(proc_macro2::Span::call_site(), "Must be path.")) + }?; + + if generic_types.contains(&ty_ident) { + if let Some(previous) = duplicate_generic_detection.replace(ty_ident) { + return Err(Error::new(previous.span(), "Generic type parameters may only be used for exactly one field, but is used more than once.")) + } + } + + all_fields.push(ntt); + } + + + let msg = "Generated by #[derive(AllSubsystemsGen)] derive proc-macro."; + let mut additive = TokenStream::new(); + + // generate an impl of `fn replace_#name` + for NameTyTup { field: replacable_item, ty: replacable_item_ty } in replacable_items { + let keeper = all_fields.iter().filter(|ntt| ntt.field != replacable_item).map(|ntt| ntt.field.clone()); + let strukt_ty = strukt_ty.clone(); + let fname = Ident::new(&format!("replace_{}", replacable_item), span); + // adjust the generics such that the appropriate member type is replaced + let mut modified_generics = orig_generics.clone(); + modified_generics.params = modified_generics.params.into_iter().map(|mut generic| { + match generic { + GenericParam::Type(ref mut param) => { + param.eq_token = None; + param.default = None; + if match &replacable_item_ty { + Type::Path(path) => + path.path.get_ident().filter(|&ident| ident == ¶m.ident).is_some(), + _ => false + } { + param.ident = Ident::new("NEW", span); + } + } + _ => {} + } + generic + }).collect(); + + additive.extend(quote! { + impl #orig_generics #strukt_ty #orig_generics { + #[doc = #msg] + pub fn #fname < NEW > (self, replacement: NEW) -> #strukt_ty #modified_generics { + #strukt_ty :: #modified_generics { + #replacable_item: replacement, + #( + #keeper: self.#keeper, + )* + } + } + } + }); + } + + Ok(additive) + } + syn::Fields::Unit => Err(Error::new(span, "Must be a struct with named fields. Not an unit struct.")), + syn::Fields::Unnamed(_) => { + Err(Error::new(span, "Must be a struct with named fields. Not an unnamed fields struct.")) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn basic() { + let item = quote! { + pub struct AllSubsystems { + pub a: A, + pub beee: B, + pub dj: CD, + } + }; + + let output = impl_subsystems_gen(item).expect("Simple example always works. qed"); + println!("//generated:"); + println!("{}", output); + } + + #[test] + fn ui() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/err-*.rs"); + t.pass("tests/ui/ok-*.rs"); + } +} diff --git a/node/overseer/all-subsystems-gen/tests/ui/err-01-enum.rs b/node/overseer/all-subsystems-gen/tests/ui/err-01-enum.rs new file mode 100644 index 000000000000..318636279ea5 --- /dev/null +++ b/node/overseer/all-subsystems-gen/tests/ui/err-01-enum.rs @@ -0,0 +1,13 @@ +#![allow(dead_code)] + +use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; + +#[derive(Clone, AllSubsystemsGen)] +enum AllSubsystems { + A(A), + B(B), +} + +fn main() { + let all = AllSubsystems::::A(0u8); +} diff --git a/node/overseer/all-subsystems-gen/tests/ui/err-01-enum.stderr b/node/overseer/all-subsystems-gen/tests/ui/err-01-enum.stderr new file mode 100644 index 000000000000..5f61df1057cb --- /dev/null +++ b/node/overseer/all-subsystems-gen/tests/ui/err-01-enum.stderr @@ -0,0 +1,5 @@ +error: expected `struct` + --> $DIR/err-01-enum.rs:6:1 + | +6 | enum AllSubsystems { + | ^^^^ diff --git a/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.rs b/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.rs new file mode 100644 index 000000000000..29aa76a38852 --- /dev/null +++ b/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.rs @@ -0,0 +1,17 @@ +#![allow(dead_code)] + +use polkadot_overseer_all_subsystems_gen::AllSubsystemsGen; + +#[derive(Clone, AllSubsystemsGen)] +struct AllSubsystems { + a: X, + b: X, +} + +fn main() { + let all = AllSubsystems:: { + a: 0_u16, + b: 1_u16, + }; + let _all = all.replace_a(77u8); +} diff --git a/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.stderr b/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.stderr new file mode 100644 index 000000000000..23e1404ff822 --- /dev/null +++ b/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.stderr @@ -0,0 +1,14 @@ +error: Generic type parameters may only be used for exactly one field, but is used more than once. + --> $DIR/err-01-generic-used-twice.rs:7:5 + | +7 | a: X, + | ^ + +error[E0599]: no method named `replace_a` found for struct `AllSubsystems` in the current scope + --> $DIR/err-01-generic-used-twice.rs:16:17 + | +6 | struct AllSubsystems { + | ----------------------- method `replace_a` not found for this +... +16 | let _all = all.replace_a(77u8); + | ^^^^^^^^^ method not found in `AllSubsystems` diff --git a/node/overseer/all-subsystems-gen/tests/ui/err-01-no-generic.rs b/node/overseer/all-subsystems-gen/tests/ui/err-01-no-generic.rs new file mode 100644 index 000000000000..2a2312954890 --- /dev/null +++ b/node/overseer/all-subsystems-gen/tests/ui/err-01-no-generic.rs @@ -0,0 +1,17 @@ +#![allow(dead_code)] + +use polkadot_overseer_all_subsystems_gen::AllSubsystemsGen; + +#[derive(Clone, AllSubsystemsGen)] +struct AllSubsystems { + a: f32, + b: u16, +} + +fn main() { + let all = AllSubsystems { + a: 0_f32, + b: 1_u16, + }; + let _all = all.replace_a(77u8); +} diff --git a/node/overseer/all-subsystems-gen/tests/ui/err-01-no-generics.stderr b/node/overseer/all-subsystems-gen/tests/ui/err-01-no-generics.stderr new file mode 100644 index 000000000000..1de880ae433c --- /dev/null +++ b/node/overseer/all-subsystems-gen/tests/ui/err-01-no-generics.stderr @@ -0,0 +1,14 @@ +error: Generic type parameters may only be used once have at least one generic parameter. + --> $DIR/err-01-no-generics.rs:7:5 + | +7 | a: X, + | ^ + +error[E0599]: no method named `replace_a` found for struct `AllSubsystems` in the current scope + --> $DIR/err-01-no-generics.rs:16:17 + | +6 | struct AllSubsystems { + | ----------------------- method `replace_a` not found for this +... +16 | let _all = all.replace_a(77u8); + | ^^^^^^^^^ method not found in `AllSubsystems` diff --git a/node/overseer/all-subsystems-gen/tests/ui/ok-01-w-generics.rs b/node/overseer/all-subsystems-gen/tests/ui/ok-01-w-generics.rs new file mode 100644 index 000000000000..b049d40ff486 --- /dev/null +++ b/node/overseer/all-subsystems-gen/tests/ui/ok-01-w-generics.rs @@ -0,0 +1,17 @@ +#![allow(dead_code)] + +use polkadot_overseer_all_subsystems_gen::AllSubsystemsGen; + +#[derive(Clone, AllSubsystemsGen)] +struct AllSubsystems { + a: A, + b: B, +} + +fn main() { + let all = AllSubsystems:: { + a: 0u8, + b: 1u16, + }; + let _all: AllSubsystems<_,_> = all.replace_a::(777_777u32); +} From f5f91f77746ead9c4d184f2f2cc86c4f9f1f37c9 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jul 2021 20:03:14 +0200 Subject: [PATCH 116/161] gen foo --- node/overseer/examples/minimal-example.rs | 37 ++++++++++++++------ node/overseer/overseer-gen/examples/dummy.rs | 20 +++++++++-- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index ae8ee8cea985..a6f5cc34225f 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -28,10 +28,20 @@ use futures_timer::Delay; use polkadot_node_primitives::{PoV, BlockData}; use polkadot_primitives::v1::Hash; -use polkadot_overseer::{Overseer, HeadSupportsParachains, AllSubsystems}; - -use polkadot_subsystem::{Subsystem, SubsystemContext, SpawnedSubsystem, FromOverseer}; -use polkadot_subsystem::messages::{ +use polkadot_overseer::{ + self as overseer, + AllMessages, + AllSubsystems, + HeadSupportsParachains, + Overseer, + SubsystemError, + gen::{ + SubsystemContext, + FromOverseer, + SpawnedSubsystem, + }, +}; +use polkadot_node_subsystem_types::messages::{ CandidateValidationMessage, CandidateBackingMessage, NetworkBridgeMessage, }; @@ -76,10 +86,13 @@ impl Subsystem1 { } } -impl Subsystem for Subsystem1 - where C: SubsystemContext + +impl overseer::Subsystem for Subsystem1 +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, { - fn start(self, ctx: C) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { Self::run(ctx).await; Ok(()) @@ -104,7 +117,7 @@ impl Subsystem2 { Delay::new(Duration::from_secs(1)).await; } }), - ).unwrap(); + ).await.unwrap(); loop { match ctx.try_recv().await { @@ -122,10 +135,12 @@ impl Subsystem2 { } } -impl Subsystem for Subsystem2 - where C: SubsystemContext +impl overseer::Subsystem for Subsystem2 +where + Context: SubsystemContext, + Context: overseer::SubsystemContext, { - fn start(self, ctx: C) -> SpawnedSubsystem { + fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { Self::run(ctx).await; Ok(()) diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index 21d35d2fb135..e078319ea5a2 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -3,7 +3,8 @@ use std::convert::Infallible; use polkadot_overseer_gen::*; -use polkadot_subsystem::messages::NetworkBridgeEvent; +use polkadot_node_subsystem::messages::NetworkBridgeEvent; +use polkadot_node_network_protocol::WrongVariant; /// Concrete subsystem implementation for `MsgStrukt` msg type. @@ -59,6 +60,12 @@ impl From for Yikes { } } +impl From for Yikes { + fn from(_: polkadot_overseer_gen::mpsc::SendError) -> Yikes { + Yikes + } +} + #[derive(Debug, Clone)] pub struct MsgStrukt(u8); @@ -73,13 +80,22 @@ impl From for MsgStrukt { #[derive(Debug, Clone, Copy)] -enum NetworkMsg { +pub enum NetworkMsg { A, B, C, } +impl NetworkMsg { + fn focus(&self) -> Result { + Ok(match self { + Self::B => return Err(WrongVariant), + Self::A | Self::C => self.clone() + }) + } +} + #[overlord(signal=SigSigSig, event=EvX, error=Yikes, network=NetworkMsg, gen=AllMessages)] From 2269761333409fc9f4e6ed0d877d129406d8abbe Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jul 2021 20:03:46 +0200 Subject: [PATCH 117/161] criss cross --- node/core/approval-voting/src/import.rs | 2 +- node/core/dispute-coordinator/src/lib.rs | 2 +- .../availability-recovery/src/tests.rs | 2 +- node/network/bitfield-distribution/src/lib.rs | 3 - node/network/bridge/src/lib.rs | 3 - node/network/gossip-support/src/tests.rs | 2 +- node/overseer/Cargo.toml | 9 +- .../proc-macro/src/impl_message_wrapper.rs | 1 - node/overseer/src/lib.rs | 14 +-- node/overseer/src/subsystems.rs | 13 ++- node/overseer/src/tests.rs | 85 +++++++++++++------ node/subsystem-util/src/lib.rs | 3 - .../src/rolling_session_window.rs | 5 +- 13 files changed, 89 insertions(+), 55 deletions(-) diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index 64ce368fb717..1ce56f3933dc 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -84,7 +84,7 @@ struct ImportedBlockInfoEnv<'a> { // Computes information about the imported block. Returns `None` if the info couldn't be extracted - // failure to communicate with overseer, async fn imported_block_info( - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), env: ImportedBlockInfoEnv<'_>, block_hash: Hash, block_header: &Header, diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index 05b51925622a..60e2d9d31c16 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -232,7 +232,7 @@ where } async fn handle_new_activations( - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), store: &dyn KeyValueDB, state: &mut State, config: &Config, diff --git a/node/network/availability-recovery/src/tests.rs b/node/network/availability-recovery/src/tests.rs index 81d9a53c9a50..d98de4f2bb06 100644 --- a/node/network/availability-recovery/src/tests.rs +++ b/node/network/availability-recovery/src/tests.rs @@ -34,7 +34,7 @@ use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks}; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_subsystem_testhelpers as test_helpers; use polkadot_subsystem::{ - messages::{RuntimeApiMessage, RuntimeApiRequest}, jaeger, ActivatedLeaf, LeafStatus, + messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest}, jaeger, ActivatedLeaf, LeafStatus, }; type VirtualOverseer = test_helpers::TestSubsystemContextHandle; diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index a0c66b18fec1..d2a5ebe78b36 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -39,9 +39,6 @@ use polkadot_primitives::v1::{Hash, SignedAvailabilityBitfield, SigningContext, use polkadot_node_network_protocol::{v1 as protocol_v1, PeerId, View, UnifiedReputationChange as Rep, OurView}; use std::collections::{HashMap, HashSet}; -#[cfg(test)] -mod tests; - const COST_SIGNATURE_INVALID: Rep = Rep::CostMajor("Bitfield signature invalid"); const COST_VALIDATOR_INDEX_INVALID: Rep = Rep::CostMajor("Bitfield validator index invalid"); const COST_MISSING_PEER_SESSION_KEY: Rep = Rep::CostMinor("Missing peer session key"); diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index f4616cb382b8..34cdd02aa72d 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -72,9 +72,6 @@ use network::{Network, send_message, get_peer_id_by_authority_id}; mod multiplexer; pub use multiplexer::RequestMultiplexer; -#[cfg(test)] -mod tests; - /// The maximum amount of heads a peer is allowed to have in their view at any time. /// /// We use the same limit to compute the view sent to peers locally. diff --git a/node/network/gossip-support/src/tests.rs b/node/network/gossip-support/src/tests.rs index 7f96f4caf7ac..8d80d84d9b0b 100644 --- a/node/network/gossip-support/src/tests.rs +++ b/node/network/gossip-support/src/tests.rs @@ -19,7 +19,7 @@ use super::*; use polkadot_node_subsystem::{ jaeger, ActivatedLeaf, LeafStatus, - messages::{RuntimeApiMessage, RuntimeApiRequest}, + messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest}, }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt as _; diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml index d563200b7d9f..f39e84f64b67 100644 --- a/node/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -11,17 +11,20 @@ sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" } futures = "0.3.15" futures-timer = "3.0.2" polkadot-node-network-protocol = { path = "../network/protocol" } -polkadot-node-primitives = { package = "polkadot-node-primitives", path = "../primitives" } -polkadot-node-subsystem = { package = "polkadot-node-subsystem-types", path = "../subsystem-types" } +polkadot-node-primitives = { path = "../primitives" } +polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-metrics = { path = "../metrics" } polkadot-primitives = { path = "../../primitives" } -polkadot-overseer-gen = { package = "polkadot-overseer-gen", path = "./overseer-gen" } +polkadot-overseer-gen = { path = "./overseer-gen" } +polkadot-overseer-all-subsystems-gen = { path = "./all-subsystems-gen" } tracing = "0.1.26" lru = "0.6" [dev-dependencies] sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } polkadot-node-network-protocol = { path = "../network/protocol" } +polkadot-node-subsystem = { path = "../subsystem" } +polkadot-node-subsystem-util = { path = "../subsystem-util" } futures = { version = "0.3.15", features = ["thread-pool"] } femme = "2.1.1" kv-log-macro = "1.0.7" diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 56d2acdb8615..89ba264fea26 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -29,7 +29,6 @@ pub(crate) fn impl_message_wrapper_enum(info: &OverseerInfo) -> Result Subsystem for ValidationSubsystem - /// where C: SubsystemContext + /// impl overseer::Subsystem for ValidationSubsystem + /// where + /// Ctx: overseer::SubsystemContext, + /// Ctx: SubsystemContext, /// { /// fn start( /// self, - /// mut ctx: C, + /// mut ctx: Ctx, /// ) -> SpawnedSubsystem { /// SpawnedSubsystem { /// name: "validation-subsystem", diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index 462340dd5181..7bc736cab680 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -14,14 +14,19 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use polkadot_node_subsystem::errors::SubsystemError; -use ::polkadot_overseer_gen::{ +//! Legacy way of defining subsystems. +//! +//! In the future, everything should be set up using the generated +//! overeseer builder pattern instead. + +use polkadot_node_subsystem_types::errors::SubsystemError; +use polkadot_overseer_gen::{ MapSubsystem, SubsystemContext, Subsystem, SpawnedSubsystem, FromOverseer, }; - +use polkadot_overseer_all_subsystems_gen::AllSubsystemsGen; use crate::OverseerSignal; /// A dummy subsystem that implements [`Subsystem`] for all @@ -67,7 +72,7 @@ where /// Each [`Subsystem`] is supposed to implement some interface that is generic over /// message type that is specific to this [`Subsystem`]. At the moment not all /// subsystems are implemented and the rest can be mocked with the [`DummySubsystem`]. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, AllSubsystemsGen)] pub struct AllSubsystems< CV = (), CB = (), SD = (), AD = (), AR = (), BS = (), BD = (), P = (), RA = (), AS = (), NB = (), CA = (), CG = (), CP = (), ApD = (), ApV = (), diff --git a/node/overseer/src/tests.rs b/node/overseer/src/tests.rs index 8d4d5b9e4320..adf8729a0123 100644 --- a/node/overseer/src/tests.rs +++ b/node/overseer/src/tests.rs @@ -16,12 +16,29 @@ use std::sync::atomic; use std::collections::HashMap; -use futures::{executor, pin_mut, select, FutureExt, pending}; +use std::task::{Poll}; +use futures::{executor, pin_mut, select, FutureExt, pending, poll, stream}; use polkadot_primitives::v1::{CollatorPair, CandidateHash}; -use polkadot_subsystem::{messages::RuntimeApiRequest, messages::NetworkBridgeEvent, jaeger}; use polkadot_node_primitives::{CollationResult, CollationGenerationConfig, PoV, BlockData}; use polkadot_node_network_protocol::{PeerId, UnifiedReputationChange}; +use polkadot_node_subsystem_types::{ + ActivatedLeaf, LeafStatus, + messages::RuntimeApiRequest, + messages::NetworkBridgeEvent, + jaeger, +}; +use polkadot_node_subsystem::{ + overseer::{ + Overseer, + HeadSupportsParachains, + gen::Delay, + SubsystemContext as OSC, + }, + SpawnedSubsystem, + SubsystemContext, + overseer, +}; use polkadot_node_subsystem_util::metered; use sp_core::crypto::Pair as _; @@ -31,8 +48,11 @@ use super::*; struct TestSubsystem1(metered::MeteredSender); -impl Subsystem for TestSubsystem1 - where C: SubsystemContext +impl overseer::Subsystem for TestSubsystem1 +where + C: overseer::SubsystemContext, + C: SubsystemContext, + { fn start(self, mut ctx: C) -> SpawnedSubsystem { let mut sender = self.0; @@ -59,8 +79,10 @@ impl Subsystem for TestSubsystem1 struct TestSubsystem2(metered::MeteredSender); -impl Subsystem for TestSubsystem2 - where C: SubsystemContext +impl overseer::Subsystem for TestSubsystem2 +where + C: overseer::SubsystemContext, + C: SubsystemContext, { fn start(self, mut ctx: C) -> SpawnedSubsystem { let sender = self.0.clone(); @@ -73,14 +95,12 @@ impl Subsystem for TestSubsystem2 if c < 10 { let (tx, _) = oneshot::channel(); ctx.send_message( - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromChainState( - Default::default(), - PoV { - block_data: BlockData(Vec::new()), - }.into(), - tx, - ) + CandidateValidationMessage::ValidateFromChainState( + Default::default(), + PoV { + block_data: BlockData(Vec::new()), + }.into(), + tx, ) ).await; c += 1; @@ -107,8 +127,10 @@ impl Subsystem for TestSubsystem2 struct ReturnOnStart; -impl Subsystem for ReturnOnStart - where C: SubsystemContext +impl overseer::Subsystem for ReturnOnStart +where + C: overseer::SubsystemContext, + C: SubsystemContext, { fn start(self, mut _ctx: C) -> SpawnedSubsystem { SpawnedSubsystem { @@ -283,8 +305,10 @@ fn overseer_ends_on_subsystem_exit() { struct TestSubsystem5(metered::MeteredSender); -impl Subsystem for TestSubsystem5 - where C: SubsystemContext +impl overseer::Subsystem for TestSubsystem5 +where + C: overseer::SubsystemContext, + C: SubsystemContext, { fn start(self, mut ctx: C) -> SpawnedSubsystem { let mut sender = self.0.clone(); @@ -314,8 +338,10 @@ impl Subsystem for TestSubsystem5 struct TestSubsystem6(metered::MeteredSender); -impl Subsystem for TestSubsystem6 - where C: SubsystemContext +impl Subsystem for TestSubsystem6 +where + C: overseer::SubsystemContext, + C: SubsystemContext, { fn start(self, mut ctx: C) -> SpawnedSubsystem { let mut sender = self.0.clone(); @@ -397,12 +423,14 @@ fn overseer_start_stop_works() { hash: first_block_hash, number: 1, span: Arc::new(jaeger::Span::Disabled), + status: LeafStatus::Fresh, })), OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { activated: [ActivatedLeaf { hash: second_block_hash, number: 2, span: Arc::new(jaeger::Span::Disabled), + status: LeafStatus::Fresh, }].as_ref().into(), deactivated: [first_block_hash].as_ref().into(), }), @@ -411,6 +439,7 @@ fn overseer_start_stop_works() { hash: third_block_hash, number: 3, span: Arc::new(jaeger::Span::Disabled), + status: LeafStatus::Fresh, }].as_ref().into(), deactivated: [second_block_hash].as_ref().into(), }), @@ -505,11 +534,13 @@ fn overseer_finalize_works() { hash: first_block_hash, number: 1, span: Arc::new(jaeger::Span::Disabled), + status: LeafStatus::Fresh, }, ActivatedLeaf { hash: second_block_hash, number: 2, span: Arc::new(jaeger::Span::Disabled), + status: LeafStatus::Fresh, }, ].as_ref().into(), ..Default::default() @@ -601,7 +632,8 @@ fn do_not_send_empty_leaves_update_on_block_finalization() { ActivatedLeaf { hash: imported_block.hash, number: imported_block.number, - span: Arc::new(jaeger::Span::Disabled) + span: Arc::new(jaeger::Span::Disabled), + status: LeafStatus::Fresh, } ].as_ref().into(), ..Default::default() @@ -656,10 +688,11 @@ impl CounterSubsystem { } } -impl Subsystem for CounterSubsystem - where - C: SubsystemContext, - M: Send, +impl Subsystem for CounterSubsystem +where + C: overseer::SubsystemContext, + C: SubsystemContext, + M: Send, { fn start(self, mut ctx: C) -> SpawnedSubsystem { SpawnedSubsystem { @@ -970,7 +1003,7 @@ fn context_holds_onto_message_until_enough_signals_received() { let (unbounded_tx, unbounded_rx) = metered::unbounded(); let (to_overseer_tx, _to_overseer_rx) = metered::unbounded(); - let mut ctx = OverseerSubsystemContext::<()>::new_unmetered( + let mut ctx = OverseerSubsystemContext::<()>::new( signal_rx, stream::select(bounded_rx, unbounded_rx), channels_out, diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 1e3943c65755..1c52592151dc 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -100,9 +100,6 @@ pub mod rolling_session_window; mod determine_new_blocks; mod error_handling; -#[cfg(test)] -mod tests; - /// Duration a job will wait after sending a stop signal before hard-aborting. pub const JOB_GRACEFUL_STOP_DURATION: Duration = Duration::from_secs(1); /// Capacity of channels to and from individual jobs diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index 68bfdedd7ffa..5c14f5d98df4 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -22,6 +22,7 @@ use polkadot_primitives::v1::{Hash, Header, SessionInfo, SessionIndex}; use polkadot_node_subsystem::{ + overseer, messages::{RuntimeApiMessage, RuntimeApiRequest}, errors::RuntimeApiError, SubsystemContext, @@ -148,7 +149,7 @@ impl RollingSessionWindow { /// some backwards drift in session index is acceptable. pub async fn cache_session_info_for_head( &mut self, - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), block_hash: Hash, block_header: &Header, ) -> Result { @@ -264,7 +265,7 @@ impl RollingSessionWindow { } async fn load_all_sessions( - ctx: &mut impl SubsystemContext, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), block_hash: Hash, start: SessionIndex, end_inclusive: SessionIndex, From ba9f2f1484bd706c239b15aedbceec8165ce09b3 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 00:03:52 +0200 Subject: [PATCH 118/161] fix overseer except for doctests --- node/core/approval-voting/src/import.rs | 6 +-- node/core/dispute-coordinator/src/lib.rs | 2 +- node/network/bridge/src/tests.rs | 1 + node/overseer/Cargo.toml | 4 +- node/overseer/examples/minimal-example.rs | 51 +++++++++++------- .../proc-macro/src/impl_channels_out.rs | 8 ++- .../proc-macro/src/impl_message_wrapper.rs | 8 +++ .../overseer-gen/proc-macro/src/impl_misc.rs | 3 +- .../proc-macro/src/impl_overseer.rs | 1 + node/overseer/src/lib.rs | 10 ++-- node/overseer/src/subsystems.rs | 3 +- node/overseer/src/tests.rs | 52 ++++++++----------- .../src/determine_new_blocks.rs | 5 +- .../src/rolling_session_window.rs | 4 +- node/subsystem-util/src/tests.rs | 2 +- 15 files changed, 91 insertions(+), 69 deletions(-) diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index 1ce56f3933dc..80ecdae311cf 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -31,7 +31,7 @@ use polkadot_node_subsystem::{ overseer, messages::{ - RuntimeApiMessage, RuntimeApiRequest, ChainApiMessage, ApprovalDistributionMessage, ApprovalVotingMessage, + RuntimeApiMessage, RuntimeApiRequest, ChainApiMessage, ApprovalDistributionMessage, }, SubsystemContext, SubsystemError, SubsystemResult, }; @@ -84,7 +84,7 @@ struct ImportedBlockInfoEnv<'a> { // Computes information about the imported block. Returns `None` if the info couldn't be extracted - // failure to communicate with overseer, async fn imported_block_info( - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), env: ImportedBlockInfoEnv<'_>, block_hash: Hash, block_header: &Header, @@ -285,7 +285,7 @@ pub struct BlockImportedCandidates { /// /// It is the responsibility of the caller to schedule wakeups for each block. pub(crate) async fn handle_new_head( - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), state: &mut State, db_writer: &dyn KeyValueDB, db_config: DatabaseConfig, diff --git a/node/core/dispute-coordinator/src/lib.rs b/node/core/dispute-coordinator/src/lib.rs index 60e2d9d31c16..0bd94103ca8b 100644 --- a/node/core/dispute-coordinator/src/lib.rs +++ b/node/core/dispute-coordinator/src/lib.rs @@ -232,7 +232,7 @@ where } async fn handle_new_activations( - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), store: &dyn KeyValueDB, state: &mut State, config: &Config, diff --git a/node/network/bridge/src/tests.rs b/node/network/bridge/src/tests.rs index 2ecc6b0b9d31..be74fb8f8129 100644 --- a/node/network/bridge/src/tests.rs +++ b/node/network/bridge/src/tests.rs @@ -1261,6 +1261,7 @@ fn spread_event_to_subsystems_is_up_to_date() { let mut cnt = 0_usize; for msg in AllMessages::dispatch_iter(NetworkBridgeEvent::PeerDisconnected(PeerId::random())) { match msg { + AllMessages::Empty => unreachable!("Nobody cares about the dummy"), AllMessages::CandidateValidation(_) => unreachable!("Not interested in network events"), AllMessages::CandidateBacking(_) => unreachable!("Not interested in network events"), AllMessages::ChainApi(_) => unreachable!("Not interested in network events"), diff --git a/node/overseer/Cargo.toml b/node/overseer/Cargo.toml index f39e84f64b67..76f96e0a4182 100644 --- a/node/overseer/Cargo.toml +++ b/node/overseer/Cargo.toml @@ -23,8 +23,8 @@ lru = "0.6" [dev-dependencies] sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } polkadot-node-network-protocol = { path = "../network/protocol" } -polkadot-node-subsystem = { path = "../subsystem" } -polkadot-node-subsystem-util = { path = "../subsystem-util" } +polkadot-node-metrics = { path = "../metrics" } +metered-channel = { path = "../metered-channel" } futures = { version = "0.3.15", features = ["thread-pool"] } femme = "2.1.1" kv-log-macro = "1.0.7" diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index a6f5cc34225f..ac87f7c7a0d2 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -26,9 +26,11 @@ use futures::{ }; use futures_timer::Delay; +use overseer::OverseerSignal; use polkadot_node_primitives::{PoV, BlockData}; use polkadot_primitives::v1::Hash; use polkadot_overseer::{ + SubsystemSender as _, self as overseer, AllMessages, AllSubsystems, @@ -51,46 +53,51 @@ impl HeadSupportsParachains for AlwaysSupportsParachains { fn head_supports_parachains(&self, _head: &Hash) -> bool { true } } + +//////// + struct Subsystem1; impl Subsystem1 { - async fn run(mut ctx: impl SubsystemContext) { - loop { + async fn run(mut ctx: Ctx) -> () + where + Ctx: overseer::SubsystemContext, + { + 'louy: loop { match ctx.try_recv().await { Ok(Some(msg)) => { if let FromOverseer::Communication { msg } = msg { tracing::info!("msg {:?}", msg); } - continue; + continue 'louy; } Ok(None) => (), Err(_) => { tracing::info!("exiting"); - return; + break 'louy; } } Delay::new(Duration::from_secs(1)).await; let (tx, _) = oneshot::channel(); - ctx.send_message( - CandidateValidationMessage::ValidateFromChainState( - Default::default(), - PoV { - block_data: BlockData(Vec::new()), - }.into(), - tx, - ) - ).await; + let msg = CandidateValidationMessage::ValidateFromChainState( + Default::default(), + PoV { + block_data: BlockData(Vec::new()), + }.into(), + tx, + ); + ctx.send_message(::AllMessages::from(msg)).await; } + () } } impl overseer::Subsystem for Subsystem1 where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { @@ -105,10 +112,15 @@ where } } +////////////////// + struct Subsystem2; impl Subsystem2 { - async fn run(mut ctx: impl SubsystemContext) { + async fn run(mut ctx: Ctx) + where + Ctx: overseer::SubsystemContext, + { ctx.spawn( "subsystem-2-job", Box::pin(async { @@ -137,8 +149,7 @@ impl Subsystem2 { impl overseer::Subsystem for Subsystem2 where - Context: SubsystemContext, - Context: overseer::SubsystemContext, + Context: overseer::SubsystemContext, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { @@ -163,7 +174,9 @@ fn main() { let all_subsystems = AllSubsystems::<()>::dummy() .replace_candidate_validation(Subsystem2) - .replace_candidate_backing(Subsystem1); + .replace_candidate_backing(Subsystem1) + ; + let (overseer, _handler) = Overseer::new( vec![], all_subsystems, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 060d266fd1eb..a6d7ff747bc0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -69,10 +69,12 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Ok(()), )* + // dummy message type + #message_wrapper :: Empty => Ok(()), }; if res.is_err() { @@ -100,10 +102,12 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Ok(()), )* + // dummy message type + #message_wrapper :: Empty => Ok(()) }; if res.is_err() { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs index 89ba264fea26..b2cb7ce11f91 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_message_wrapper.rs @@ -58,6 +58,14 @@ pub(crate) fn impl_message_wrapper_enum(info: &OverseerInfo) -> Result for #message_wrapper { + fn from(_: ()) -> Self { + #message_wrapper :: Empty + } } #( diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index c2aeeaad9fae..a89ca128126f 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -138,8 +138,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result impl SubsystemContext for #subsystem_ctx_name where #subsystem_sender_name: ::polkadot_overseer_gen::SubsystemSender< #wrapper_message >, - #subsystem_sender_name: ::polkadot_overseer_gen::SubsystemSender< M >, - AllMessages: From, + #wrapper_message: From, { type Message = M; type Signal = #signal; diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index a0d6c204080a..e3ce2fbdac04 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -148,6 +148,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result {} )* + #message_wrapper :: Empty => {} } Ok(()) } diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 694394f5f7a8..dfe28edee331 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -472,17 +472,19 @@ where /// # use futures_timer::Delay; /// # use polkadot_overseer::{Overseer, HeadSupportsParachains, AllSubsystems}; /// # use polkadot_primitives::v1::Hash; - /// # use polkadot_node_subsystem::{ + /// # use crate::{ + /// # self as overseer, OverseerSignal, AllMessages, /// # Subsystem, DummySubsystem, SpawnedSubsystem, SubsystemContext, - /// # messages::CandidateValidationMessage, SubsystemError, + /// # }; + /// # use polkadot_node_subsystem_types::{ + /// # messages::CandidateValidationMessage}, SubsystemError, /// # }; /// /// struct ValidationSubsystem; /// /// impl overseer::Subsystem for ValidationSubsystem /// where - /// Ctx: overseer::SubsystemContext, - /// Ctx: SubsystemContext, + /// Ctx: overseer::SubsystemContext, /// { /// fn start( /// self, diff --git a/node/overseer/src/subsystems.rs b/node/overseer/src/subsystems.rs index 7bc736cab680..b75e7e50e6b8 100644 --- a/node/overseer/src/subsystems.rs +++ b/node/overseer/src/subsystems.rs @@ -28,6 +28,7 @@ use polkadot_overseer_gen::{ }; use polkadot_overseer_all_subsystems_gen::AllSubsystemsGen; use crate::OverseerSignal; +use crate::AllMessages; /// A dummy subsystem that implements [`Subsystem`] for all /// types of messages. Used for tests or as a placeholder. @@ -36,7 +37,7 @@ pub struct DummySubsystem; impl Subsystem for DummySubsystem where - Context: SubsystemContext, + Context: SubsystemContext, { fn start(self, mut ctx: Context) -> SpawnedSubsystem { let future = Box::pin(async move { diff --git a/node/overseer/src/tests.rs b/node/overseer/src/tests.rs index adf8729a0123..4d769b8072e8 100644 --- a/node/overseer/src/tests.rs +++ b/node/overseer/src/tests.rs @@ -24,34 +24,35 @@ use polkadot_node_primitives::{CollationResult, CollationGenerationConfig, PoV, use polkadot_node_network_protocol::{PeerId, UnifiedReputationChange}; use polkadot_node_subsystem_types::{ ActivatedLeaf, LeafStatus, - messages::RuntimeApiRequest, - messages::NetworkBridgeEvent, + messages::{ + RuntimeApiRequest, + NetworkBridgeEvent, + }, jaeger, }; -use polkadot_node_subsystem::{ - overseer::{ - Overseer, - HeadSupportsParachains, - gen::Delay, - SubsystemContext as OSC, - }, - SpawnedSubsystem, - SubsystemContext, - overseer, + +use crate::{ + self as overseer, + Overseer, + HeadSupportsParachains, + gen::Delay, + }; -use polkadot_node_subsystem_util::metered; +use metered_channel as metered; use sp_core::crypto::Pair as _; use assert_matches::assert_matches; use super::*; + +type SpawnedSubsystem = crate::gen::SpawnedSubsystem; + struct TestSubsystem1(metered::MeteredSender); impl overseer::Subsystem for TestSubsystem1 where - C: overseer::SubsystemContext, - C: SubsystemContext, + C: overseer::SubsystemContext, { fn start(self, mut ctx: C) -> SpawnedSubsystem { @@ -81,8 +82,7 @@ struct TestSubsystem2(metered::MeteredSender); impl overseer::Subsystem for TestSubsystem2 where - C: overseer::SubsystemContext, - C: SubsystemContext, + C: overseer::SubsystemContext, { fn start(self, mut ctx: C) -> SpawnedSubsystem { let sender = self.0.clone(); @@ -129,8 +129,7 @@ struct ReturnOnStart; impl overseer::Subsystem for ReturnOnStart where - C: overseer::SubsystemContext, - C: SubsystemContext, + C: overseer::SubsystemContext, { fn start(self, mut _ctx: C) -> SpawnedSubsystem { SpawnedSubsystem { @@ -307,8 +306,7 @@ struct TestSubsystem5(metered::MeteredSender); impl overseer::Subsystem for TestSubsystem5 where - C: overseer::SubsystemContext, - C: SubsystemContext, + C: overseer::SubsystemContext, { fn start(self, mut ctx: C) -> SpawnedSubsystem { let mut sender = self.0.clone(); @@ -340,8 +338,7 @@ struct TestSubsystem6(metered::MeteredSender); impl Subsystem for TestSubsystem6 where - C: overseer::SubsystemContext, - C: SubsystemContext, + C: overseer::SubsystemContext, { fn start(self, mut ctx: C) -> SpawnedSubsystem { let mut sender = self.0.clone(); @@ -690,8 +687,7 @@ impl CounterSubsystem { impl Subsystem for CounterSubsystem where - C: overseer::SubsystemContext, - C: SubsystemContext, + C: overseer::SubsystemContext, M: Send, { fn start(self, mut ctx: C) -> SpawnedSubsystem { @@ -922,7 +918,6 @@ fn overseer_all_subsystems_receive_signals_and_messages() { fn context_holds_onto_message_until_enough_signals_received() { let (candidate_validation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); let (candidate_backing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (candidate_selection_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); let (statement_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); let (availability_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); let (availability_recovery_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); @@ -941,7 +936,6 @@ fn context_holds_onto_message_until_enough_signals_received() { let (candidate_validation_unbounded_tx, _) = metered::unbounded(); let (candidate_backing_unbounded_tx, _) = metered::unbounded(); - let (candidate_selection_unbounded_tx, _) = metered::unbounded(); let (statement_distribution_unbounded_tx, _) = metered::unbounded(); let (availability_distribution_unbounded_tx, _) = metered::unbounded(); let (availability_recovery_unbounded_tx, _) = metered::unbounded(); @@ -961,7 +955,6 @@ fn context_holds_onto_message_until_enough_signals_received() { let channels_out = ChannelsOut { candidate_validation: candidate_validation_bounded_tx.clone(), candidate_backing: candidate_backing_bounded_tx.clone(), - candidate_selection: candidate_selection_bounded_tx.clone(), statement_distribution: statement_distribution_bounded_tx.clone(), availability_distribution: availability_distribution_bounded_tx.clone(), availability_recovery: availability_recovery_bounded_tx.clone(), @@ -980,7 +973,6 @@ fn context_holds_onto_message_until_enough_signals_received() { candidate_validation_unbounded: candidate_validation_unbounded_tx.clone(), candidate_backing_unbounded: candidate_backing_unbounded_tx.clone(), - candidate_selection_unbounded: candidate_selection_unbounded_tx.clone(), statement_distribution_unbounded: statement_distribution_unbounded_tx.clone(), availability_distribution_unbounded: availability_distribution_unbounded_tx.clone(), availability_recovery_unbounded: availability_recovery_unbounded_tx.clone(), @@ -1003,7 +995,7 @@ fn context_holds_onto_message_until_enough_signals_received() { let (unbounded_tx, unbounded_rx) = metered::unbounded(); let (to_overseer_tx, _to_overseer_rx) = metered::unbounded(); - let mut ctx = OverseerSubsystemContext::<()>::new( + let mut ctx = OverseerSubsystemContext::new( signal_rx, stream::select(bounded_rx, unbounded_rx), channels_out, diff --git a/node/subsystem-util/src/determine_new_blocks.rs b/node/subsystem-util/src/determine_new_blocks.rs index 6379d01c3570..e439a1692150 100644 --- a/node/subsystem-util/src/determine_new_blocks.rs +++ b/node/subsystem-util/src/determine_new_blocks.rs @@ -160,7 +160,8 @@ pub async fn determine_new_blocks( mod tests { use super::*; use std::collections::{HashSet, HashMap}; - use sp_core::testing::TaskExecutor; + use polkadot_node_subsystem::messages::AvailabilityStoreMessage; +use sp_core::testing::TaskExecutor; use polkadot_overseer::{AllMessages, SubsystemContext}; use polkadot_node_subsystem_test_helpers::make_subsystem_context; use assert_matches::assert_matches; @@ -611,7 +612,7 @@ mod tests { } ); - for _ in 0..2 { + for _ in 0_u8..2 { assert_matches!( handle.recv().await, AllMessages::ChainApi(ChainApiMessage::BlockHeader(h, tx)) => { diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index 5c14f5d98df4..ecbe5cd36850 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -297,7 +297,7 @@ async fn load_all_sessions( mod tests { use super::*; use polkadot_node_subsystem_test_helpers::make_subsystem_context; - use polkadot_node_subsystem::messages::AllMessages; + use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage, AvailabilityStoreMessage}; use sp_core::testing::TaskExecutor; use assert_matches::assert_matches; @@ -333,7 +333,7 @@ mod tests { }; let pool = TaskExecutor::new(); - let (mut ctx, mut handle) = make_subsystem_context::<(), _>(pool.clone()); + let (mut ctx, mut handle) = make_subsystem_context::(pool.clone()); let hash = header.hash(); diff --git a/node/subsystem-util/src/tests.rs b/node/subsystem-util/src/tests.rs index 10eb7436716d..828d47baed13 100644 --- a/node/subsystem-util/src/tests.rs +++ b/node/subsystem-util/src/tests.rs @@ -75,7 +75,7 @@ impl JobTrait for FakeCollatorProtocolJob { sender.send_message(CollatorProtocolMessage::Invalid( Default::default(), Default::default(), - ).into()).await; + )).await; } // it isn't necessary to break run_loop into its own function, From 441619bab6d508697739e7aa0d5ab31b274df9b4 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 07:56:51 +0200 Subject: [PATCH 119/161] fix overseer doc test --- node/overseer/examples/minimal-example.rs | 2 +- node/overseer/src/lib.rs | 38 +++++++++++++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index ac87f7c7a0d2..defbce5112c6 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -26,7 +26,6 @@ use futures::{ }; use futures_timer::Delay; -use overseer::OverseerSignal; use polkadot_node_primitives::{PoV, BlockData}; use polkadot_primitives::v1::Hash; use polkadot_overseer::{ @@ -36,6 +35,7 @@ use polkadot_overseer::{ AllSubsystems, HeadSupportsParachains, Overseer, + OverseerSignal, SubsystemError, gen::{ SubsystemContext, diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index dfe28edee331..efbd0fe4cd45 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -470,26 +470,42 @@ where /// # use std::time::Duration; /// # use futures::{executor, pin_mut, select, FutureExt}; /// # use futures_timer::Delay; - /// # use polkadot_overseer::{Overseer, HeadSupportsParachains, AllSubsystems}; /// # use polkadot_primitives::v1::Hash; - /// # use crate::{ - /// # self as overseer, OverseerSignal, AllMessages, - /// # Subsystem, DummySubsystem, SpawnedSubsystem, SubsystemContext, + /// # use polkadot_overseer::{ + /// # self as overseer, + /// # OverseerSignal, + /// # SubsystemSender as _, + /// # AllMessages, + /// # AllSubsystems, + /// # HeadSupportsParachains, + /// # Overseer, + /// # SubsystemError, + /// # gen::{ + /// # SubsystemContext, + /// # FromOverseer, + /// # SpawnedSubsystem, + /// # }, /// # }; - /// # use polkadot_node_subsystem_types::{ - /// # messages::CandidateValidationMessage}, SubsystemError, + /// # use polkadot_node_subsystem_types::messages::{ + /// # CandidateValidationMessage, CandidateBackingMessage, + /// # NetworkBridgeMessage, /// # }; /// /// struct ValidationSubsystem; /// /// impl overseer::Subsystem for ValidationSubsystem /// where - /// Ctx: overseer::SubsystemContext, + /// Ctx: overseer::SubsystemContext< + /// Message=CandidateValidationMessage, + /// AllMessages=AllMessages, + /// Signal=OverseerSignal, + /// Error=SubsystemError, + /// >, /// { /// fn start( /// self, /// mut ctx: Ctx, - /// ) -> SpawnedSubsystem { + /// ) -> SpawnedSubsystem { /// SpawnedSubsystem { /// name: "validation-subsystem", /// future: Box::pin(async move { @@ -508,7 +524,8 @@ where /// fn head_supports_parachains(&self, _head: &Hash) -> bool { true } /// } /// let spawner = sp_core::testing::TaskExecutor::new(); - /// let all_subsystems = AllSubsystems::<()>::dummy().replace_candidate_validation(ValidationSubsystem); + /// let all_subsystems = AllSubsystems::<()>::dummy() + /// .replace_candidate_validation(ValidationSubsystem); /// let (overseer, _handler) = Overseer::new( /// vec![], /// all_subsystems, @@ -528,7 +545,8 @@ where /// _ = timer => (), /// } /// # - /// # }); } + /// # }); + /// # } /// ``` pub fn new( leaves: impl IntoIterator, From 782802737f5e049f05db77539f039049474a126c Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 10:12:22 +0200 Subject: [PATCH 120/161] remainder warnings --- node/core/runtime-api/src/tests.rs | 1 - node/network/bridge/src/tests.rs | 1 - .../tests/ui/err-01-enum.rs | 2 +- .../tests/ui/err-01-generic-used-twice.rs | 1 - node/overseer/examples/minimal-example.rs | 3 --- node/overseer/overseer-gen/examples/dummy.rs | 21 +++++++++---------- node/overseer/overseer-gen/src/tests.rs | 2 -- .../src/determine_new_blocks.rs | 3 +-- .../src/rolling_session_window.rs | 2 +- 9 files changed, 13 insertions(+), 23 deletions(-) diff --git a/node/core/runtime-api/src/tests.rs b/node/core/runtime-api/src/tests.rs index 1f6078f73e4f..787dc8884b0b 100644 --- a/node/core/runtime-api/src/tests.rs +++ b/node/core/runtime-api/src/tests.rs @@ -29,7 +29,6 @@ use futures::channel::oneshot; use polkadot_node_primitives::{ BabeEpoch, BabeEpochConfiguration, BabeAllowedSlots, }; -use polkadot_subsystem::messages::AllMessages; #[derive(Default, Clone)] struct MockRuntimeApi { diff --git a/node/network/bridge/src/tests.rs b/node/network/bridge/src/tests.rs index be74fb8f8129..4fcb57d7554e 100644 --- a/node/network/bridge/src/tests.rs +++ b/node/network/bridge/src/tests.rs @@ -21,7 +21,6 @@ use futures::channel::oneshot; use std::borrow::Cow; use std::collections::HashSet; -use std::pin::Pin; use std::sync::atomic::{AtomicBool, Ordering}; use async_trait::async_trait; use parking_lot::Mutex; diff --git a/node/overseer/all-subsystems-gen/tests/ui/err-01-enum.rs b/node/overseer/all-subsystems-gen/tests/ui/err-01-enum.rs index 318636279ea5..ffcbecd0b3f4 100644 --- a/node/overseer/all-subsystems-gen/tests/ui/err-01-enum.rs +++ b/node/overseer/all-subsystems-gen/tests/ui/err-01-enum.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use polkadot_procmacro_overseer_subsystems_gen::AllSubsystemsGen; +use polkadot_overseer_all_subsystems_gen::AllSubsystemsGen; #[derive(Clone, AllSubsystemsGen)] enum AllSubsystems { diff --git a/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.rs b/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.rs index 29aa76a38852..5c80dca787ea 100644 --- a/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.rs +++ b/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.rs @@ -1,7 +1,6 @@ #![allow(dead_code)] use polkadot_overseer_all_subsystems_gen::AllSubsystemsGen; - #[derive(Clone, AllSubsystemsGen)] struct AllSubsystems { a: X, diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index defbce5112c6..1ee1b661079f 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -29,7 +29,6 @@ use futures_timer::Delay; use polkadot_node_primitives::{PoV, BlockData}; use polkadot_primitives::v1::Hash; use polkadot_overseer::{ - SubsystemSender as _, self as overseer, AllMessages, AllSubsystems, @@ -38,14 +37,12 @@ use polkadot_overseer::{ OverseerSignal, SubsystemError, gen::{ - SubsystemContext, FromOverseer, SpawnedSubsystem, }, }; use polkadot_node_subsystem_types::messages::{ CandidateValidationMessage, CandidateBackingMessage, - NetworkBridgeMessage, }; struct AlwaysSupportsParachains; diff --git a/node/overseer/overseer-gen/examples/dummy.rs b/node/overseer/overseer-gen/examples/dummy.rs index e078319ea5a2..401e70e89f2d 100644 --- a/node/overseer/overseer-gen/examples/dummy.rs +++ b/node/overseer/overseer-gen/examples/dummy.rs @@ -1,9 +1,6 @@ //! A dummy to be used with cargo expand -use std::convert::Infallible; - use polkadot_overseer_gen::*; -use polkadot_node_subsystem::messages::NetworkBridgeEvent; use polkadot_node_network_protocol::WrongVariant; @@ -12,7 +9,7 @@ use polkadot_node_network_protocol::WrongVariant; pub struct AwesomeSubSys; impl ::polkadot_overseer_gen::Subsystem, Yikes> for AwesomeSubSys { - fn start(self, ctx: XxxSubsystemContext) -> SpawnedSubsystem < Yikes > { + fn start(self, _ctx: XxxSubsystemContext) -> SpawnedSubsystem < Yikes > { unimplemented!("starting yay!") } } @@ -21,7 +18,7 @@ impl ::polkadot_overseer_gen::Subsystem, Yikes> f pub struct GoblinTower; impl ::polkadot_overseer_gen::Subsystem, Yikes> for GoblinTower { - fn start(self, ctx: XxxSubsystemContext) -> SpawnedSubsystem < Yikes > { + fn start(self, _ctx: XxxSubsystemContext) -> SpawnedSubsystem < Yikes > { unimplemented!("welcum") } } @@ -113,12 +110,12 @@ struct Xxx { struct DummySpawner; impl SpawnNamed for DummySpawner{ - fn spawn_blocking(&self, name: &'static str, future: futures::future::BoxFuture<'static, ()>) { - unimplemented!("spawn blocking") + fn spawn_blocking(&self, name: &'static str, _future: futures::future::BoxFuture<'static, ()>) { + unimplemented!("spawn blocking {}", name) } - fn spawn(&self, name: &'static str, future: futures::future::BoxFuture<'static, ()>) { - unimplemented!("spawn") + fn spawn(&self, name: &'static str, _future: futures::future::BoxFuture<'static, ()>) { + unimplemented!("spawn {}", name) } } @@ -126,10 +123,12 @@ impl SpawnNamed for DummySpawner{ struct DummyCtx; fn main() { - let overseer = Xxx::builder() + let (overseer, _handler): (Xxx<_>, _) = Xxx::builder() .sub0(AwesomeSubSys::default()) .plinkos(GoblinTower::default()) .i_like_pi(::std::f64::consts::PI) .spawner(DummySpawner) - .build(); + .build() + .unwrap(); + assert_eq!(overseer.i_like_pi.floor() as i8, 3); } diff --git a/node/overseer/overseer-gen/src/tests.rs b/node/overseer/overseer-gen/src/tests.rs index a5e77d1e7077..5553e1f86601 100644 --- a/node/overseer/overseer-gen/src/tests.rs +++ b/node/overseer/overseer-gen/src/tests.rs @@ -1,5 +1,3 @@ -use super::*; - #[test] fn ui_pass() { let t = trybuild::TestCases::new(); diff --git a/node/subsystem-util/src/determine_new_blocks.rs b/node/subsystem-util/src/determine_new_blocks.rs index e439a1692150..9d47592d9741 100644 --- a/node/subsystem-util/src/determine_new_blocks.rs +++ b/node/subsystem-util/src/determine_new_blocks.rs @@ -160,8 +160,7 @@ pub async fn determine_new_blocks( mod tests { use super::*; use std::collections::{HashSet, HashMap}; - use polkadot_node_subsystem::messages::AvailabilityStoreMessage; -use sp_core::testing::TaskExecutor; + use sp_core::testing::TaskExecutor; use polkadot_overseer::{AllMessages, SubsystemContext}; use polkadot_node_subsystem_test_helpers::make_subsystem_context; use assert_matches::assert_matches; diff --git a/node/subsystem-util/src/rolling_session_window.rs b/node/subsystem-util/src/rolling_session_window.rs index ecbe5cd36850..944fe0c05489 100644 --- a/node/subsystem-util/src/rolling_session_window.rs +++ b/node/subsystem-util/src/rolling_session_window.rs @@ -297,7 +297,7 @@ async fn load_all_sessions( mod tests { use super::*; use polkadot_node_subsystem_test_helpers::make_subsystem_context; - use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage, AvailabilityStoreMessage}; + use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage}; use sp_core::testing::TaskExecutor; use assert_matches::assert_matches; From f59078c6da84840285d4b6d6fe85e1b499962943 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 11:07:52 +0200 Subject: [PATCH 121/161] do not rename the proc-macro --- node/overseer/overseer-gen/Cargo.toml | 2 +- node/overseer/overseer-gen/src/lib.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index 23dc35c84684..c4e2cc5f2d37 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -10,8 +10,8 @@ tracing = "0.1" futures = "0.3" async-trait = "0.1" thiserror = "1" -overseer-gen-proc-macro = { package = "polkadot-overseer-gen-proc-macro", path = "./proc-macro" } metered = { package = "metered-channel", path = "../../metered-channel" } +polkadot-overseer-gen-proc-macro = { path = "./proc-macro" } polkadot-node-network-protocol = { path = "../../network/protocol"} # trait SpawnNamed diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 31caca821118..68604a918e2a 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -59,7 +59,8 @@ // yielding false positives #![deny(missing_docs)] -pub use overseer_gen_proc_macro::*; +pub use polkadot_overseer_gen_proc_macro::overlord; + #[doc(hidden)] pub use tracing; #[doc(hidden)] From 8f777fe85b6491ee516b542a8d9f1fe7a8464240 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 13:37:02 +0200 Subject: [PATCH 122/161] better error --- .../tests/ui/err-01-generic-used-twice.stderr | 10 +++--- .../tests/ui/err-01-no-generic.stderr | 14 ++++++++ .../overseer-gen/proc-macro/src/parse_attr.rs | 2 +- .../proc-macro/src/parse_struct.rs | 26 +++++++------- node/overseer/overseer-gen/src/tests.rs | 10 +++--- .../tests/ui/err-01-duplicate-consumer.rs | 5 +-- .../tests/ui/err-01-duplicate-consumer.stderr | 21 +++++++++++ .../overseer-gen/tests/ui/err-02-enum.rs | 6 ++-- .../overseer-gen/tests/ui/err-02-enum.stderr | 11 ++++++ .../tests/ui/err-03-subsys-twice.rs | 9 ++--- .../tests/ui/err-03-subsys-twice.stderr | 17 +++++++++ ...no-dispatch.rs => err-04-missing-error.rs} | 8 +++-- .../tests/ui/err-04-missing-error.stderr | 11 ++++++ .../overseer-gen/tests/ui/ok-01-boring.rs | 32 ----------------- .../overseer-gen/tests/ui/ok-02-w-generic.rs | 35 ------------------- .../overseer-gen/tests/ui/ok-04-blocking.rs | 34 ------------------ 16 files changed, 115 insertions(+), 136 deletions(-) create mode 100644 node/overseer/all-subsystems-gen/tests/ui/err-01-no-generic.stderr create mode 100644 node/overseer/overseer-gen/tests/ui/err-01-duplicate-consumer.stderr create mode 100644 node/overseer/overseer-gen/tests/ui/err-02-enum.stderr create mode 100644 node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.stderr rename node/overseer/overseer-gen/tests/ui/{ok-03-no-dispatch.rs => err-04-missing-error.rs} (79%) create mode 100644 node/overseer/overseer-gen/tests/ui/err-04-missing-error.stderr delete mode 100644 node/overseer/overseer-gen/tests/ui/ok-01-boring.rs delete mode 100644 node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs delete mode 100644 node/overseer/overseer-gen/tests/ui/ok-04-blocking.rs diff --git a/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.stderr b/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.stderr index 23e1404ff822..019dc42aae00 100644 --- a/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.stderr +++ b/node/overseer/all-subsystems-gen/tests/ui/err-01-generic-used-twice.stderr @@ -1,14 +1,14 @@ error: Generic type parameters may only be used for exactly one field, but is used more than once. - --> $DIR/err-01-generic-used-twice.rs:7:5 + --> $DIR/err-01-generic-used-twice.rs:6:5 | -7 | a: X, +6 | a: X, | ^ error[E0599]: no method named `replace_a` found for struct `AllSubsystems` in the current scope - --> $DIR/err-01-generic-used-twice.rs:16:17 + --> $DIR/err-01-generic-used-twice.rs:15:17 | -6 | struct AllSubsystems { +5 | struct AllSubsystems { | ----------------------- method `replace_a` not found for this ... -16 | let _all = all.replace_a(77u8); +15 | let _all = all.replace_a(77u8); | ^^^^^^^^^ method not found in `AllSubsystems` diff --git a/node/overseer/all-subsystems-gen/tests/ui/err-01-no-generic.stderr b/node/overseer/all-subsystems-gen/tests/ui/err-01-no-generic.stderr new file mode 100644 index 000000000000..f8eb8e56c3ac --- /dev/null +++ b/node/overseer/all-subsystems-gen/tests/ui/err-01-no-generic.stderr @@ -0,0 +1,14 @@ +error: struct must have at least one generic parameter. + --> $DIR/err-01-no-generic.rs:6:8 + | +6 | struct AllSubsystems { + | ^^^^^^^^^^^^^ + +error[E0599]: no method named `replace_a` found for struct `AllSubsystems` in the current scope + --> $DIR/err-01-no-generic.rs:16:17 + | +6 | struct AllSubsystems { + | -------------------- method `replace_a` not found for this +... +16 | let _all = all.replace_a(77u8); + | ^^^^^^^^^ method not found in `AllSubsystems` diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs index a4a3450a0610..d7dc05c86d55 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs @@ -166,7 +166,7 @@ impl Parse for AttrArgs { .remove(TAG_EXT_ERROR_TY) .map(|x| if let AttrItem::ExternErrorType(x) = x { x.clone() } else { unreachable!() }) .ok_or_else(|| { - Error::new(span, format!("Must declare the overseer signals type via `{}=..`.", TAG_EXT_ERROR_TY)) + Error::new(span, format!("Must declare the overseer error type via `{}=..`.", TAG_EXT_ERROR_TY)) })?; let extern_signal_ty = unique diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 889f319febf3..c820a904d8fb 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -41,8 +41,6 @@ pub(crate) struct SubSysField { /// which is also used `#wrapper_message :: #variant` variant /// part. pub(crate) generic: Ident, - /// Type of the subsystem. - pub(crate) ty: Path, /// Type to be consumed by the subsystem. pub(crate) consumes: Path, /// If `no_dispatch` is present, if the message is incoming via @@ -192,13 +190,6 @@ impl OverseerInfo { .collect::>() } - #[allow(dead_code)] - // TODO use as the defaults, if no subsystem is specified - // TODO or drop the type argument. - pub(crate) fn subsystem_types(&self) -> Vec { - self.subsystems.iter().map(|ssf| ssf.ty.clone()).collect::>() - } - pub(crate) fn baggage_names(&self) -> Vec { self.baggage.iter().map(|bag| bag.field_name.clone()).collect::>() } @@ -271,6 +262,11 @@ impl OverseerGuts { let n = fields.named.len(); let mut subsystems = Vec::with_capacity(n); let mut baggage = Vec::with_capacity(n); + + // The types of `#[subsystem(..)]` annotated fields + // have to be unique, since they are used as generics + // for the builder pattern besides other places. + let mut unique_subsystem_idents = HashSet::::new(); for Field { attrs, vis, ident, ty, .. } in fields.named.into_iter() { let mut consumes = attrs.iter().filter(|attr| attr.style == AttrStyle::Outer).filter_map(|attr| { let span = attr.path.span(); @@ -301,12 +297,18 @@ impl OverseerGuts { return Err(Error::new(span, "Subsystem must consume at least one message")); } - let ty = try_type_to_path(ty, span)?; - let generic = ty.get_ident().ok_or_else(|| Error::new(ty.span(), "Must be a identifier, not a path."))?.clone(); + let field_ty = try_type_to_path(ty, span)?; + let generic = field_ty.get_ident().ok_or_else(|| Error::new(field_ty.span(), "Must be an identifier, not a path."))?.clone(); + if let Some(previous) = unique_subsystem_idents.get(&generic) { + let mut e = Error::new(generic.span(), format!("Duplicate subsystem names `{}`", generic)); + e.combine(Error::new(previous.span(), "previously defined here.")); + return Err(e) + } + unique_subsystem_idents.insert(generic.clone()); + subsystems.push(SubSysField { name: ident, generic, - ty, consumes: consumes_paths[0].clone(), no_dispatch: variant.no_dispatch, wip: variant.wip, diff --git a/node/overseer/overseer-gen/src/tests.rs b/node/overseer/overseer-gen/src/tests.rs index 5553e1f86601..b14e2695a939 100644 --- a/node/overseer/overseer-gen/src/tests.rs +++ b/node/overseer/overseer-gen/src/tests.rs @@ -1,8 +1,8 @@ -#[test] -fn ui_pass() { - let t = trybuild::TestCases::new(); - t.pass("tests/ui/ok-01*.rs"); -} + +// The generated code requires quite a bit of surrounding code to work. +// Please refer to [the examples](examples/dummy.rs) and +// [the minimal usage example](../examples/minimal-example.rs). + #[test] fn ui_compile_fail() { let t = trybuild::TestCases::new(); diff --git a/node/overseer/overseer-gen/tests/ui/err-01-duplicate-consumer.rs b/node/overseer/overseer-gen/tests/ui/err-01-duplicate-consumer.rs index 48560852464b..b81f10a7f0fa 100644 --- a/node/overseer/overseer-gen/tests/ui/err-01-duplicate-consumer.rs +++ b/node/overseer/overseer-gen/tests/ui/err-01-duplicate-consumer.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use polkadot_overseer_gen::overlord; +use polkadot_overseer_gen::*; #[derive(Default)] struct AwesomeSubSys; @@ -8,6 +8,7 @@ struct AwesomeSubSys; #[derive(Default)] struct AwesomeSubSys2; +#[derive(Clone, Debug)] struct SigSigSig; struct Event; @@ -15,7 +16,7 @@ struct Event; #[derive(Clone)] struct MsgStrukt(u8); -#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] +#[overlord(signal=SigSigSig, event=Event, gen=AllMessages, error=OverseerError)] struct Overseer { #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, diff --git a/node/overseer/overseer-gen/tests/ui/err-01-duplicate-consumer.stderr b/node/overseer/overseer-gen/tests/ui/err-01-duplicate-consumer.stderr new file mode 100644 index 000000000000..edb0ef436951 --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/err-01-duplicate-consumer.stderr @@ -0,0 +1,21 @@ +error[E0119]: conflicting implementations of trait `std::convert::From` for type `AllMessages` + --> $DIR/err-01-duplicate-consumer.rs:19:1 + | +19 | #[overlord(signal=SigSigSig, event=Event, gen=AllMessages, error=OverseerError)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | | + | first implementation here + | conflicting implementation for `AllMessages` + | + = note: this error originates in an attribute macro (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0119]: conflicting implementations of trait `polkadot_overseer_gen::SubsystemSender` for type `OverseerSubsystemSender` + --> $DIR/err-01-duplicate-consumer.rs:19:1 + | +19 | #[overlord(signal=SigSigSig, event=Event, gen=AllMessages, error=OverseerError)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | | + | first implementation here + | conflicting implementation for `OverseerSubsystemSender` + | + = note: this error originates in an attribute macro (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/node/overseer/overseer-gen/tests/ui/err-02-enum.rs b/node/overseer/overseer-gen/tests/ui/err-02-enum.rs index 7f159217252c..c7e491bfba9a 100644 --- a/node/overseer/overseer-gen/tests/ui/err-02-enum.rs +++ b/node/overseer/overseer-gen/tests/ui/err-02-enum.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use polkadot_overseer_gen::overlord; +use polkadot_overseer_gen::*; #[derive(Default)] struct AwesomeSubSys; @@ -9,10 +9,10 @@ struct SigSigSig; struct Event; -#[derive(Clone)] +#[derive(Clone, Debug)] struct MsgStrukt(u8); -#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] +#[overlord(signal=SigSigSig, event=Event, gen=AllMessages, error=OverseerError)] enum Overseer { #[subsystem(MsgStrukt)] Sub0(AwesomeSubSys), diff --git a/node/overseer/overseer-gen/tests/ui/err-02-enum.stderr b/node/overseer/overseer-gen/tests/ui/err-02-enum.stderr new file mode 100644 index 000000000000..7ed414a6ecb3 --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/err-02-enum.stderr @@ -0,0 +1,11 @@ +error: expected `struct` + --> $DIR/err-02-enum.rs:16:1 + | +16 | enum Overseer { + | ^^^^ + +error[E0433]: failed to resolve: use of undeclared type `Overseer` + --> $DIR/err-02-enum.rs:27:17 + | +27 | let overseer = Overseer::<_,_>::builder() + | ^^^^^^^^ use of undeclared type `Overseer` diff --git a/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs b/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs index c8979d7a3adf..9a7ad951c8b7 100644 --- a/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs +++ b/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.rs @@ -1,21 +1,22 @@ #![allow(dead_code)] -use polkadot_overseer_gen::overlord; +use polkadot_overseer_gen::*; #[derive(Default)] struct AwesomeSubSys; +#[derive(Clone, Debug)] struct SigSigSig; struct Event; -#[derive(Clone)] +#[derive(Clone, Debug)] struct MsgStrukt(u8); -#[derive(Clone)] +#[derive(Clone, Debug)] struct MsgStrukt2(f64); -#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] +#[overlord(signal=SigSigSig, event=Event, gen=AllMessages, error=OverseerError)] struct Overseer { #[subsystem(MsgStrukt)] sub0: AwesomeSubSys, diff --git a/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.stderr b/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.stderr new file mode 100644 index 000000000000..cba46366daed --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/err-03-subsys-twice.stderr @@ -0,0 +1,17 @@ +error: Duplicate subsystem names `AwesomeSubSys` + --> $DIR/err-03-subsys-twice.rs:25:8 + | +25 | sub1: AwesomeSubSys, + | ^^^^^^^^^^^^^ + +error: previously defined here. + --> $DIR/err-03-subsys-twice.rs:22:8 + | +22 | sub0: AwesomeSubSys, + | ^^^^^^^^^^^^^ + +error[E0433]: failed to resolve: use of undeclared type `Overseer` + --> $DIR/err-03-subsys-twice.rs:34:17 + | +34 | let overseer = Overseer::<_,_>::builder() + | ^^^^^^^^ use of undeclared type `Overseer` diff --git a/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs b/node/overseer/overseer-gen/tests/ui/err-04-missing-error.rs similarity index 79% rename from node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs rename to node/overseer/overseer-gen/tests/ui/err-04-missing-error.rs index 3fe6d0fa75ba..3b6966f1da79 100644 --- a/node/overseer/overseer-gen/tests/ui/ok-03-no-dispatch.rs +++ b/node/overseer/overseer-gen/tests/ui/err-04-missing-error.rs @@ -1,10 +1,11 @@ #![allow(dead_code)] -use polkadot_overseer_gen::overlord; +use polkadot_overseer_gen::*; #[derive(Default)] struct AwesomeSubSys; +#[derive(Clone, Debug)] struct SigSigSig; struct Event; @@ -26,9 +27,10 @@ struct DummySpawner; struct DummyCtx; fn main() { - let overseer = Overseer::<_,_>::builder() + let _ = Overseer::builder() .sub0(AwesomeSubSys::default()) .i_like_pie(std::f64::consts::PI) .spawner(DummySpawner) - .build(|| -> DummyCtx { DummyCtx } ); + .build() + .unwrap(); } diff --git a/node/overseer/overseer-gen/tests/ui/err-04-missing-error.stderr b/node/overseer/overseer-gen/tests/ui/err-04-missing-error.stderr new file mode 100644 index 000000000000..92ddb2e26af0 --- /dev/null +++ b/node/overseer/overseer-gen/tests/ui/err-04-missing-error.stderr @@ -0,0 +1,11 @@ +error: Must declare the overseer error type via `error=..`. + --> $DIR/err-04-missing-error.rs:16:12 + | +16 | #[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] + | ^^^^^^ + +error[E0433]: failed to resolve: use of undeclared type `Overseer` + --> $DIR/err-04-missing-error.rs:30:10 + | +30 | let _ = Overseer::builder() + | ^^^^^^^^ use of undeclared type `Overseer` diff --git a/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs b/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs deleted file mode 100644 index 8f6a436c5743..000000000000 --- a/node/overseer/overseer-gen/tests/ui/ok-01-boring.rs +++ /dev/null @@ -1,32 +0,0 @@ -#![allow(dead_code)] - -use polkadot_overseer_gen::overlord; - -#[derive(Default)] -struct AwesomeSubSys; - -struct SigSigSig; - -struct Event; - -#[derive(Clone)] -struct MsgStrukt(u8); - -#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] -struct Overseer { - #[subsystem(MsgStrukt)] - sub0: AwesomeSubSys, -} - -#[derive(Debug, Clone)] -struct DummySpawner; - -struct DummyCtx; - -fn main() { - let overseer = Overseer::<_,_>::builder() - .sub0(AwesomeSubSys::default()) - .i_like_pie(std::f64::consts::PI) - .spawner(DummySpawner) - .build(|| -> DummyCtx { DummyCtx } ); -} diff --git a/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs b/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs deleted file mode 100644 index f92df3abcb06..000000000000 --- a/node/overseer/overseer-gen/tests/ui/ok-02-w-generic.rs +++ /dev/null @@ -1,35 +0,0 @@ -#![allow(dead_code)] - -use polkadot_overseer_gen::overlord; - -#[derive(Default)] -struct AwesomeSubSys; - -struct SigSigSig; - -struct Event; - -#[derive(Clone)] -struct MsgStrukt(u8); - -#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] -struct Overseer { - #[subsystem(MsgStrukt)] - sub0: AwesomeSubSys, - - something_else: T, -} - -#[derive(Debug, Clone)] -struct DummySpawner; - -struct DummyCtx; - - -fn main() { - let overseer = Overseer::<_,_,_>::builder() - .sub0(AwesomeSubSys::default()) - .something_else(7777u32) - .spawner(DummySpawner) - .build(|| -> DummyCtx { DummyCtx } ); -} diff --git a/node/overseer/overseer-gen/tests/ui/ok-04-blocking.rs b/node/overseer/overseer-gen/tests/ui/ok-04-blocking.rs deleted file mode 100644 index 921ee5c3e19d..000000000000 --- a/node/overseer/overseer-gen/tests/ui/ok-04-blocking.rs +++ /dev/null @@ -1,34 +0,0 @@ -#![allow(dead_code)] - -use polkadot_overseer_gen::overlord; - -#[derive(Default)] -struct BlockingSubSys; - -struct SigSigSig; - -struct Event; - -#[derive(Clone)] -struct MsgStrukt(u8); - -#[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] -struct Overseer { - #[subsystem(blocking, MsgStrukt)] - sub0: BlockingSubSys, - - i_like_pie: f64, -} - -#[derive(Debug, Clone)] -struct DummySpawner; - -struct DummyCtx; - -fn main() { - let overseer = Overseer::<_,_>::builder() - .sub0(BlockingSubSys::default()) - .i_like_pie(std::f64::consts::PI) - .spawner(DummySpawner) - .build(|| -> DummyCtx { DummyCtx } ); -} From 1855c54b0c4cb352714b97fdb8f3d812e9e9624d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 13:43:07 +0200 Subject: [PATCH 123/161] chore: remove dead code --- node/overseer/src/lib.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index efbd0fe4cd45..f16835a26afb 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -132,18 +132,6 @@ pub use polkadot_overseer_gen::{ }; pub use polkadot_overseer_gen as gen; -/// Boiler plate reduction, `Signal` and `AllMessages` are fixed for one overseer. -/// So the only variation for external entities is `M`. -pub trait SubsystemCtx : SubsystemContext { -} - -impl SubsystemCtx for T -where - T: SubsystemContext -{ -} - - /// Whether a header supports parachain consensus or not. pub trait HeadSupportsParachains { /// Return true if the given header supports parachain consensus. Otherwise, false. From 212a85817931e5d0f0e3c7f07ffc705466e7ce94 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 13:48:11 +0200 Subject: [PATCH 124/161] behavior tests are legacy, see malus --- behavior-tests/Cargo.toml | 19 -------- behavior-tests/src/lib.rs | 92 --------------------------------------- 2 files changed, 111 deletions(-) delete mode 100644 behavior-tests/Cargo.toml delete mode 100644 behavior-tests/src/lib.rs diff --git a/behavior-tests/Cargo.toml b/behavior-tests/Cargo.toml deleted file mode 100644 index 0d79bafa2a34..000000000000 --- a/behavior-tests/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "behavior-tests" -version = "0.1.0" -authors = ["Parity Technologies "] -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -thiserror = "1" -polkadot-service = { path = "../node/service", default-features = false } -sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } -sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } -sc-chain-spec = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } -test-runner = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } -log = "0.4" -env_logger = "0.8" -tokio = { version = "0.2.21", features = [ "signal", "rt-core", "rt-threaded", "blocking" ] } -futures = "0.3" diff --git a/behavior-tests/src/lib.rs b/behavior-tests/src/lib.rs deleted file mode 100644 index 8d35997adb91..000000000000 --- a/behavior-tests/src/lib.rs +++ /dev/null @@ -1,92 +0,0 @@ - -use polkadot_service as service; -use sc_service::{error::Error as ServiceError, Configuration, TaskManager, TaskExecutor}; -use sp_core::crypto::set_default_ss58_version; -use test_runner::default_config; -use log::info; -use sc_chain_spec::ChainSpec; -use futures::future::FutureExt; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error(transparent)] - SubstrateService(#[from] sc_service::Error), - - - #[error(transparent)] - PolkadotService(#[from] polkadot_service::Error), - - #[error(transparent)] - Io(#[from]std::io::Error) -} - -pub type Result = std::result::Result; - - -macro_rules! behavior_testcase { - ($name:ident => (&name:literal : $x:ty)* ) => { - let _ = env_logger::Builder::new().is_test(true).try_init(); - - - for - }; -} - -/// ```rust -bahvior_test!(alpha_omega => { - "Alice" -} -/// ``` - - -/// Launch a modified node with a given all subsystems instance -pub fn modded_node(subsystemselection: Sel, chain_spec: Box) -> Result<()> { - - // set_default_ss58_version(&chain_spec); - - let grandpa_pause = None; - - let jaeger_agent = None; - - let telemetry_worker = None; - - let mut tokio_rt = tokio::runtime::Builder::new() - .threaded_scheduler() - .enable_all() - .build()?; - - - let handle = tokio_rt.handle().clone(); - let task_executor: TaskExecutor = (move |future, _task_type| { - handle.spawn(future).map(|_| ()) - }).into(); - - - let config = default_config(task_executor, chain_spec); - let initialize = move |config: Configuration| async move { - let task_manager = - service::build_full( - config, - service::IsCollator::No, - grandpa_pause, - jaeger_agent, - telemetry_worker, - ) - .map(|full| full.task_manager)?; - Ok::<_, Error>(task_manager) - }; - - let mut task_manager = tokio_rt.block_on(initialize(config))?; - // futures::pin_mut!(task_manager); - let res = tokio_rt.block_on(task_manager.future().fuse()); - tokio_rt.block_on(task_manager.clean_shutdown()); - Ok(()) -} - -#[cfg(test)] -mod tests { - #[test] - fn it_works() { - assert_eq!(2 + 2, 4); - } -} From f2d08c4b918732512b10186e331c2edff79b9873 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 13:51:27 +0200 Subject: [PATCH 125/161] Update node/core/approval-voting/src/lib.rs --- node/core/approval-voting/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 1f666e6696e7..79fe56a1a25c 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -854,7 +854,8 @@ where // // returns `true` if any of the actions was a `Conclude` command. async fn handle_actions( - ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), state: &mut State, + ctx: &mut (impl SubsystemContext + overseer::SubsystemContext), + state: &mut State, metrics: &Metrics, wakeups: &mut Wakeups, currently_checking_set: &mut CurrentlyCheckingSet, From 33942ea07fcaa716e6e4084ca965afc45bdb8710 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 14:31:19 +0200 Subject: [PATCH 126/161] attr parsing fixin --- node/overseer/overseer-gen/proc-macro/src/tests.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/node/overseer/overseer-gen/proc-macro/src/tests.rs b/node/overseer/overseer-gen/proc-macro/src/tests.rs index 10adb2fd17ab..79c901518947 100644 --- a/node/overseer/overseer-gen/proc-macro/src/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/tests.rs @@ -23,6 +23,7 @@ use syn::parse_quote; fn print() { let attr = quote! { gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, + error=OverseerError, }; let item = quote! { @@ -85,6 +86,7 @@ fn struct_parse_basic() { fn attr_full() { let attr: AttrArgs = parse_quote! { gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, + error=OverseerError, }; assert_matches!(attr, AttrArgs { message_channel_capacity, @@ -100,6 +102,7 @@ fn attr_full() { fn attr_partial() { let attr: AttrArgs = parse_quote! { gen=AllMessage, event=::some::why::ExternEvent, signal=::foo::SigSigSig, + error=OverseerError, }; assert_matches!(attr, AttrArgs { message_channel_capacity: _, From 2cd459382f3a5219cc9e6bd0f0871a8e89bd181a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 16:02:47 +0200 Subject: [PATCH 127/161] remove unused dependencies --- Cargo.lock | 23 ++----------------- node/overseer/overseer-gen/Cargo.toml | 3 --- .../overseer-gen/proc-macro/Cargo.toml | 3 --- .../overseer-gen/proc-macro/src/lib.rs | 2 ++ node/overseer/overseer-gen/src/lib.rs | 1 + 5 files changed, 5 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd89775cf672..5deddc6c6b16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2568,7 +2568,7 @@ dependencies = [ "log", "slab", "tokio 0.2.21", - "tokio-util 0.3.1", + "tokio-util", ] [[package]] @@ -3289,7 +3289,7 @@ dependencies = [ "thiserror", "tokio 0.2.21", "tokio-rustls 0.15.0", - "tokio-util 0.3.1", + "tokio-util", "url 2.2.0", ] @@ -6535,11 +6535,9 @@ dependencies = [ "metered-channel", "pin-project 1.0.7", "polkadot-node-network-protocol", - "polkadot-node-subsystem", "polkadot-overseer-gen-proc-macro", "sp-core", "thiserror", - "tokio-util 0.6.7", "tracing", "trybuild", ] @@ -6549,11 +6547,9 @@ name = "polkadot-overseer-gen-proc-macro" version = "0.1.0" dependencies = [ "assert_matches", - "proc-macro-crate 1.0.0", "proc-macro2", "quote", "syn", - "trybuild", ] [[package]] @@ -10949,21 +10945,6 @@ dependencies = [ "tokio 0.2.21", ] -[[package]] -name = "tokio-util" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" -dependencies = [ - "bytes 1.0.1", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.2.4", - "slab", - "tokio 1.6.1", -] - [[package]] name = "toml" version = "0.5.6" diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index c4e2cc5f2d37..62e020b1e640 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -13,14 +13,11 @@ thiserror = "1" metered = { package = "metered-channel", path = "../../metered-channel" } polkadot-overseer-gen-proc-macro = { path = "./proc-macro" } polkadot-node-network-protocol = { path = "../../network/protocol"} - # trait SpawnNamed sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } futures-timer = "3.0.2" -tokio-util = { version = "0.6.7", features = [ "time" ] } pin-project = "1.0" futures-util = "0.3" [dev-dependencies] trybuild = "1.0.41" -polkadot-node-subsystem = { path = "../../subsystem" } diff --git a/node/overseer/overseer-gen/proc-macro/Cargo.toml b/node/overseer/overseer-gen/proc-macro/Cargo.toml index 3b02a5d8bc82..87d6d471140e 100644 --- a/node/overseer/overseer-gen/proc-macro/Cargo.toml +++ b/node/overseer/overseer-gen/proc-macro/Cargo.toml @@ -15,9 +15,6 @@ proc-macro = true syn = { version = "1.0.60", features = ["full", "extra-traits"] } quote = "1.0.9" proc-macro2 = "1.0.26" -proc-macro-crate = "1.0.0" -assert_matches = "1.5.0" [dev-dependencies] -trybuild = "1.0.41" assert_matches = "1.5.0" diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index d95393e52ae6..133f22d2ea8d 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +#![deny(unused_crate_dependencies)] + use proc_macro2::TokenStream; use syn::{parse2, Result}; diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 68604a918e2a..094a27c5d224 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -58,6 +58,7 @@ // unused dependencies can not work for test and examples at the same time // yielding false positives #![deny(missing_docs)] +#![deny(unused_crate_dependencies)] pub use polkadot_overseer_gen_proc_macro::overlord; From 6a31014b20779e9ee0cd6f450c94c53ae9929ae7 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 16:02:56 +0200 Subject: [PATCH 128/161] reintroduce a constant --- node/overseer/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index f16835a26afb..5703eac48a90 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -132,6 +132,10 @@ pub use polkadot_overseer_gen::{ }; pub use polkadot_overseer_gen as gen; +/// Store 2 days worth of blocks, not accounting for forks, +/// in the LRU cache. Assumes a 6-second block time. +const KNOWN_LEAVES_CACHE_SIZE: usize = 2 * 24 * 3600 / 6; + /// Whether a header supports parachain consensus or not. pub trait HeadSupportsParachains { /// Return true if the given header supports parachain consensus. Otherwise, false. @@ -586,7 +590,7 @@ where .leaves(Vec::from_iter( leaves.into_iter().map(|BlockInfo { hash, parent_hash: _, number }| (hash, number)) )) - .known_leaves(LruCache::new(128))// FIXME verify this number + .known_leaves(LruCache::new(KNOWN_LEAVES_CACHE_SIZE)) .active_leaves(Default::default()) .span_per_active_leaf(Default::default()) .activation_external_listeners(Default::default()) From cb2564d161abc7dd5b43199f4ae3fb4c18dfbbf1 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 16:04:27 +0200 Subject: [PATCH 129/161] get rid of ::std::mem::drop and use let _ = .. --- node/overseer/src/lib.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 5703eac48a90..f939bfd913ed 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -643,12 +643,10 @@ where /// Stop the overseer. async fn stop(mut self) { - ::std::mem::drop( - self.wait_terminate( + let _ = self.wait_terminate( OverseerSignal::Conclude, ::std::time::Duration::from_secs(1_u64) - ).await - ); + ).await; } /// Run the `Overseer`. From b38c643ed87eab0bbb9583f75eca4edf1ae567fe Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 16:05:03 +0200 Subject: [PATCH 130/161] fix comment --- node/overseer/overseer-gen/proc-macro/src/parse_struct.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index c820a904d8fb..010235987ea3 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -211,7 +211,7 @@ impl OverseerInfo { .collect::>() } - /// Generic types per subsystem, in the form `Sub#N`. + /// Generic types per subsystem, as defined by the user. pub(crate) fn builder_generic_types(&self) -> Vec { self.subsystems .iter() From cd4803b6493d141ff7cc9ecc85d8da88236de92d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 16:07:58 +0200 Subject: [PATCH 131/161] reference accumulator issue for further improvements --- node/overseer/overseer-gen/proc-macro/src/impl_builder.rs | 1 + node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs | 1 + node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs | 1 + node/overseer/overseer-gen/src/lib.rs | 1 - node/overseer/src/lib.rs | 3 ++- 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index c1fb671fdab7..035f50ea9331 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -204,6 +204,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Result { extern_msg // focuses on a `NetworkBridgeEvent< protocol_v1::* >` // TODO do not require this to be hardcoded, either externalize or ... + // https://github.com/paritytech/polkadot/issues/3427 .focus() .ok() .map(|event| { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index e3ce2fbdac04..5968386bab55 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -37,6 +37,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result /// * Subsystems dying when they are not expected to /// * Subsystems not dying when they are told to die /// * etc. -// FIXME XXX make generic over the source error of FromOrigin #[derive(thiserror::Error, Debug)] #[allow(missing_docs)] pub enum OverseerError { diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index f939bfd913ed..3ff7c7ca9ba3 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -99,7 +99,8 @@ pub use polkadot_node_subsystem_types::{ jaeger, }; -/// TODO legacy, to be deleted, left for easier integration +// TODO legacy, to be deleted, left for easier integration +// TODO https://github.com/paritytech/polkadot/issues/3427 mod subsystems; pub use self::subsystems::AllSubsystems; From 7c8b89433c6577433b6bd85acc19e27fae8e12aa Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 7 Jul 2021 16:12:07 +0200 Subject: [PATCH 132/161] proc-macro remove impl_replace for now --- .../proc-macro/src/impl_replace.rs | 92 ------------------- .../overseer-gen/proc-macro/src/lib.rs | 1 - 2 files changed, 93 deletions(-) delete mode 100644 node/overseer/overseer-gen/proc-macro/src/impl_replace.rs diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs b/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs deleted file mode 100644 index ae60743a519f..000000000000 --- a/node/overseer/overseer-gen/proc-macro/src/impl_replace.rs +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2021 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Replace a subsystem with a different implementation. - -use proc_macro2::TokenStream; -use quote::quote; -use syn::{Ident, Result}; - -use super::*; - -pub(crate) fn impl_replacable_subsystem( - info: &OverseerInfo, -) -> Result { - let msg = "Generated by #[overlord] derive proc-macro."; - - let overseer_name = &info.overseer_name; - - let error_ty = &info.error_ty; - - let field_ty = &info.builder_generic_types(); - let baggage_generic_ty = &info.baggage_generic_types(); - - let baggage_name = &info.baggage_names(); - - let generics = quote! { - < Ctx, S, #( #baggage_generic_ty, )* #( #field_ty, )* > - }; - - let where_clause = quote! { - where - Ctx: SubsystemContext, - S: ::polkadot_overseer_gen::SpawnNamed, - #( #field_ty : Subsystem, )* - }; - - let mut additive = TokenStream::new(); - - // generate an impl of `fn replace_#name` - for SubSysField { name: replacable_item, ty: _replacable_item_ty, generic, .. } in info.subsystems.iter() { - let keeper = info.subsystems.iter().filter(|&ssf| ssf.generic != *generic).map(|ssf| ssf.name.clone()); - - let fn_name = Ident::new(format!("replace_{}", replacable_item).as_str(), replacable_item.span()); - // adjust the generics such that the appropriate member type is replaced - - let new = Ident::new("NEW", replacable_item.span()); - let modified_generics = &info.subsystems - .iter() - .map(|ssf| if ssf.generic != *generic { ssf.generic.clone() } else { new.clone() }) - .collect::>(); - - let modified_generics = quote! { - < Ctx, #( #baggage_generic_ty, )* #( #modified_generics, )* > - }; - - let x: TokenStream = quote! { - impl #generics #overseer_name #generics #where_clause { - #[doc = #msg] - pub fn #fn_name < #new > (self, replacement: #new) -> #overseer_name #modified_generics - where - #new: Subsystem, - { - #overseer_name :: #modified_generics { - #replacable_item: replacement, - #( - #keeper: self. #keeper, - )* - #( - #baggage_name: self. #baggage_name, - )* - } - } - } - }; - additive.extend(x); - } - - Ok(additive) -} diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 133f22d2ea8d..021bd55dba48 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -24,7 +24,6 @@ mod impl_misc; mod impl_overseer; mod parse_attr; mod parse_struct; -// mod impl_replace; mod impl_channels_out; mod impl_dispatch; mod impl_message_wrapper; From 3a56d547e7abddf2ecc640ae9e3b6e8c9b78a159 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 10:17:01 +0200 Subject: [PATCH 133/161] remove unused crate deps --- Cargo.lock | 7 +------ node/subsystem/Cargo.toml | 7 +------ node/subsystem/src/lib.rs | 3 ++- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5deddc6c6b16..3226ef7a1cc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6389,16 +6389,11 @@ dependencies = [ name = "polkadot-node-subsystem" version = "0.1.0" dependencies = [ + "async-trait", "futures 0.3.15", "polkadot-node-jaeger", - "polkadot-node-metrics", - "polkadot-node-primitives", "polkadot-node-subsystem-types", "polkadot-overseer", - "polkadot-overseer-gen", - "polkadot-primitives", - "smallvec 1.6.1", - "thiserror", ] [[package]] diff --git a/node/subsystem/Cargo.toml b/node/subsystem/Cargo.toml index d83e41994a19..815d164f30c0 100644 --- a/node/subsystem/Cargo.toml +++ b/node/subsystem/Cargo.toml @@ -7,12 +7,7 @@ description = "Subsystem traits and message definitions and the generated overse [dependencies] polkadot-overseer = { path = "../overseer" } -polkadot-overseer-gen = { path = "../overseer/overseer-gen" } polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-jaeger = { path = "../jaeger" } -thiserror = "1.0.23" -smallvec = "1.6.1" futures = "0.3" -polkadot-primitives = { path = "../../primitives" } -polkadot-node-primitives = { path = "../primitives" } -polkadot-node-metrics = { path = "../metrics" } +async-trait = "0.1.50" diff --git a/node/subsystem/src/lib.rs b/node/subsystem/src/lib.rs index f4e76dc1a276..53666c6f3fa5 100644 --- a/node/subsystem/src/lib.rs +++ b/node/subsystem/src/lib.rs @@ -18,7 +18,8 @@ //! //! Node-side types and generated overseer. -#![warn(missing_docs)] +#![deny(missing_docs)] +#![deny(unused_crate_dependencies)] pub use polkadot_node_jaeger as jaeger; pub use jaeger::*; From f607bd8dff068ce48e05047571390142f7131693 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 10:24:45 +0200 Subject: [PATCH 134/161] remove wrong deprecation warnigns --- node/overseer/overseer-gen/proc-macro/src/impl_misc.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index a89ca128126f..b50e1f42c062 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -221,7 +221,6 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result &mut self.to_subsystems } - #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn(ctx, name, fut)")] async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) -> ::std::result::Result<(), #error_ty> { @@ -231,7 +230,6 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result }).await.map_err(Into::into) } - #[deprecated(note="Avoid the message roundtrip and use `<_ as SubsystemContext>::spawn_blocking(ctx, name, fut)")] async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) -> ::std::result::Result<(), #error_ty> { From a9f2118406a27ac3a4eaa5221f5f08f90a8191bd Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 10:39:45 +0200 Subject: [PATCH 135/161] undo accidental undo Applies changes of https://github.com/paritytech/polkadot/pull/3337/files --- Cargo.lock | 2 -- node/collation-generation/src/lib.rs | 2 +- node/core/approval-voting/src/lib.rs | 2 +- node/core/candidate-validation/src/lib.rs | 2 +- .../src/pov_requester/mod.rs | 2 +- .../src/requester/fetch_task/mod.rs | 2 +- node/network/availability-recovery/src/lib.rs | 2 +- node/network/bridge/src/lib.rs | 2 +- node/network/statement-distribution/src/lib.rs | 4 ++-- node/overseer/examples/minimal-example.rs | 2 +- .../overseer-gen/proc-macro/src/impl_misc.rs | 12 ++++++------ node/overseer/overseer-gen/src/lib.rs | 4 ++-- node/subsystem-types/src/errors.rs | 4 ++-- node/subsystem-util/src/lib.rs | 5 ++--- node/subsystem/Cargo.toml | 2 -- 15 files changed, 22 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3226ef7a1cc6..ac8a930d2771 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6389,8 +6389,6 @@ dependencies = [ name = "polkadot-node-subsystem" version = "0.1.0" dependencies = [ - "async-trait", - "futures 0.3.15", "polkadot-node-jaeger", "polkadot-node-subsystem-types", "polkadot-overseer", diff --git a/node/collation-generation/src/lib.rs b/node/collation-generation/src/lib.rs index 5b020a18e83d..72a1beaeafe8 100644 --- a/node/collation-generation/src/lib.rs +++ b/node/collation-generation/src/lib.rs @@ -414,7 +414,7 @@ async fn handle_new_activations( "failed to send collation result", ); } - })).await?; + }))?; } } diff --git a/node/core/approval-voting/src/lib.rs b/node/core/approval-voting/src/lib.rs index 79fe56a1a25c..ac99b95c4d48 100644 --- a/node/core/approval-voting/src/lib.rs +++ b/node/core/approval-voting/src/lib.rs @@ -2270,7 +2270,7 @@ async fn launch_approval( }; let (background, remote_handle) = background.remote_handle(); - ctx.spawn("approval-checks", Box::pin(background)).await + ctx.spawn("approval-checks", Box::pin(background)) .map(move |()| remote_handle) } diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index 4c3df0c4fbc9..9b22d82a3704 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -114,7 +114,7 @@ where let (mut validation_host, task) = polkadot_node_core_pvf::start( polkadot_node_core_pvf::Config::new(cache_path, program_path), ); - ctx.spawn_blocking("pvf-validation-host", task.boxed()).await?; + ctx.spawn_blocking("pvf-validation-host", task.boxed())?; loop { match ctx.recv().await? { diff --git a/node/network/availability-distribution/src/pov_requester/mod.rs b/node/network/availability-distribution/src/pov_requester/mod.rs index e605ae577db0..650072f0ea7e 100644 --- a/node/network/availability-distribution/src/pov_requester/mod.rs +++ b/node/network/availability-distribution/src/pov_requester/mod.rs @@ -74,7 +74,7 @@ where let span = jaeger::Span::new(candidate_hash, "fetch-pov") .with_validator_index(from_validator) .with_relay_parent(parent); - ctx.spawn("pov-fetcher", fetch_pov_job(pov_hash, pending_response.boxed(), span, tx).boxed()).await + ctx.spawn("pov-fetcher", fetch_pov_job(pov_hash, pending_response.boxed(), span, tx).boxed()) .map_err(|e| Fatal::SpawnTask(e))?; Ok(()) } diff --git a/node/network/availability-distribution/src/requester/fetch_task/mod.rs b/node/network/availability-distribution/src/requester/fetch_task/mod.rs index 51835fcd167d..c936d443fc6b 100644 --- a/node/network/availability-distribution/src/requester/fetch_task/mod.rs +++ b/node/network/availability-distribution/src/requester/fetch_task/mod.rs @@ -188,7 +188,7 @@ impl FetchTask { if let Some(running) = prepared_running { let (handle, kill) = oneshot::channel(); - ctx.spawn("chunk-fetcher", running.run(kill).boxed()).await + ctx.spawn("chunk-fetcher", running.run(kill).boxed()) .map_err(|e| Fatal::SpawnTask(e))?; Ok(FetchTask { diff --git a/node/network/availability-recovery/src/lib.rs b/node/network/availability-recovery/src/lib.rs index 5fba19211e65..b164675083d1 100644 --- a/node/network/availability-recovery/src/lib.rs +++ b/node/network/availability-recovery/src/lib.rs @@ -657,7 +657,7 @@ where awaiting: vec![response_sender], }); - if let Err(e) = ctx.spawn("recovery interaction", Box::pin(remote)).await { + if let Err(e) = ctx.spawn("recovery interaction", Box::pin(remote)) { tracing::warn!( target: LOG_TARGET, err = ?e, diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 34cdd02aa72d..11574bca5d6f 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -901,7 +901,7 @@ where shared.clone(), ).remote_handle(); - ctx.spawn("network-bridge-network-worker", Box::pin(remote)).await?; + ctx.spawn("network-bridge-network-worker", Box::pin(remote))?; ctx.send_message( StatementDistributionMessage::StatementFetchingReceiver(statement_receiver) diff --git a/node/network/statement-distribution/src/lib.rs b/node/network/statement-distribution/src/lib.rs index 0060f8fa4ebc..6588d996ce22 100644 --- a/node/network/statement-distribution/src/lib.rs +++ b/node/network/statement-distribution/src/lib.rs @@ -1241,7 +1241,7 @@ async fn launch_request( ) .remote_handle(); - let result = ctx.spawn("large-statement-fetcher", task.boxed()).await; + let result = ctx.spawn("large-statement-fetcher", task.boxed()); if let Err(err) = result { tracing::error!(target: LOG_TARGET, ?err, "Spawning task failed."); return None @@ -1951,7 +1951,7 @@ impl StatementDistribution { ctx.spawn( "large-statement-responder", respond(receiver, res_sender.clone()).boxed() - ).await.map_err(Fatal::SpawnTask)?; + ).map_err(Fatal::SpawnTask)?; } } } diff --git a/node/overseer/examples/minimal-example.rs b/node/overseer/examples/minimal-example.rs index 1ee1b661079f..71a703f99336 100644 --- a/node/overseer/examples/minimal-example.rs +++ b/node/overseer/examples/minimal-example.rs @@ -126,7 +126,7 @@ impl Subsystem2 { Delay::new(Duration::from_secs(1)).await; } }), - ).await.unwrap(); + ).unwrap(); loop { match ctx.try_recv().await { diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index b50e1f42c062..de652d28a77a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -221,22 +221,22 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result &mut self.to_subsystems } - async fn spawn(&mut self, name: &'static str, s: Pin + Send>>) + fn spawn(&mut self, name: &'static str, s: Pin + Send>>) -> ::std::result::Result<(), #error_ty> { - self.to_overseer.send(::polkadot_overseer_gen::ToOverseer::SpawnJob { + self.to_overseer.unbounded_send(::polkadot_overseer_gen::ToOverseer::SpawnJob { name, s, - }).await.map_err(Into::into) + }).map_err(|_| SubsystemError::TaskSpawn(name)) } - async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) + fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) -> ::std::result::Result<(), #error_ty> { - self.to_overseer.send(::polkadot_overseer_gen::ToOverseer::SpawnBlockingJob { + self.to_overseer.unbounded_send(::polkadot_overseer_gen::ToOverseer::SpawnBlockingJob { name, s, - }).await.map_err(Into::into) + }).map_err(|_| SubsystemError::TaskSpawn(name)) } } }; diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index a98ec4fd4392..c50dc6535c03 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -389,14 +389,14 @@ pub trait SubsystemContext: Send + 'static { async fn recv(&mut self) -> Result, Self::Error>; /// Spawn a child task on the executor. - async fn spawn( + fn spawn( &mut self, name: &'static str, s: ::std::pin::Pin + Send>> ) -> Result<(), Self::Error>; /// Spawn a blocking child task on the executor's dedicated thread pool. - async fn spawn_blocking( + fn spawn_blocking( &mut self, name: &'static str, s: ::std::pin::Pin + Send>>, diff --git a/node/subsystem-types/src/errors.rs b/node/subsystem-types/src/errors.rs index de2ca01b658a..6c02d609f570 100644 --- a/node/subsystem-types/src/errors.rs +++ b/node/subsystem-types/src/errors.rs @@ -97,8 +97,8 @@ pub enum SubsystemError { #[error(transparent)] QueueError(#[from] futures::channel::mpsc::SendError), - #[error(transparent)] - TaskSpawn(#[from] futures::task::SpawnError), + #[error("Failed to spawn a task: {0}")] + TaskSpawn(&'static str), #[error(transparent)] Infallible(#[from] std::convert::Infallible), diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 1c52592151dc..98c93467a250 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -748,9 +748,8 @@ impl JobSubsystem { // TODO it should only wait for the spawn to complete // TODO but not for anything beyond that let res = match outgoing.expect("the Jobs stream never ends; qed") { - FromJobCommand::Spawn(name, task) => ctx.spawn(name, task).await, - FromJobCommand::SpawnBlocking(name, task) - => ctx.spawn_blocking(name, task).await, + FromJobCommand::Spawn(name, task) => ctx.spawn(name, task), + FromJobCommand::SpawnBlocking(name, task) => ctx.spawn_blocking(name, task), }; if let Err(e) = res { diff --git a/node/subsystem/Cargo.toml b/node/subsystem/Cargo.toml index 815d164f30c0..e4b3e0cad504 100644 --- a/node/subsystem/Cargo.toml +++ b/node/subsystem/Cargo.toml @@ -9,5 +9,3 @@ description = "Subsystem traits and message definitions and the generated overse polkadot-overseer = { path = "../overseer" } polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-jaeger = { path = "../jaeger" } -futures = "0.3" -async-trait = "0.1.50" From 256fb389518d97d2c5bd6e2b560f593bfa5ce352 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 11:31:29 +0200 Subject: [PATCH 136/161] Update node/metrics/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Köcher --- node/metrics/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/node/metrics/src/lib.rs b/node/metrics/src/lib.rs index 806970b9ab70..6555e304f12e 100644 --- a/node/metrics/src/lib.rs +++ b/node/metrics/src/lib.rs @@ -82,8 +82,7 @@ pub struct Metronome { state: MetronomeState, } -impl Metronome -{ +impl Metronome { /// Create a new metronome source with a defined cycle duration. pub fn new(cycle: Duration) -> Self { let period = cycle.into(); From 6e8dfad147de8ad99d47c8086bc07a802e04458d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 11:31:37 +0200 Subject: [PATCH 137/161] Update node/metrics/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Köcher --- node/metrics/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/node/metrics/src/lib.rs b/node/metrics/src/lib.rs index 6555e304f12e..7a149a553b7c 100644 --- a/node/metrics/src/lib.rs +++ b/node/metrics/src/lib.rs @@ -94,8 +94,7 @@ impl Metronome { } } -impl futures::Stream for Metronome -{ +impl futures::Stream for Metronome { type Item = (); fn poll_next( mut self: Pin<&mut Self>, From 1e774efe9e0add22676577ad02505642ea8d5c59 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 12:02:59 +0200 Subject: [PATCH 138/161] Update node/overseer/overseer-gen/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Köcher --- node/overseer/overseer-gen/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index c50dc6535c03..a4b6b817cc96 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -92,8 +92,7 @@ pub use async_trait::async_trait; #[doc(hidden)] pub use std::time::Duration; -use std::sync::atomic::{self, AtomicUsize}; -use std::sync::Arc; +use std::sync{Arc, atomic::{self, AtomicUsize}}; #[doc(hidden)] pub use futures_timer::Delay; From 6700a2d7fd611ce3cc8d53b62b301a3772a0ffde Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 12:05:18 +0200 Subject: [PATCH 139/161] use proc-macro-crate --- Cargo.lock | 1 + .../overseer-gen/proc-macro/Cargo.toml | 1 + .../proc-macro/src/impl_builder.rs | 57 ++++++++++--------- .../proc-macro/src/impl_channels_out.rs | 14 +++-- .../overseer-gen/proc-macro/src/impl_misc.rs | 47 +++++++-------- .../proc-macro/src/impl_overseer.rs | 24 ++++---- .../overseer-gen/proc-macro/src/lib.rs | 12 +++- .../proc-macro/src/parse_struct.rs | 7 +++ 8 files changed, 95 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac8a930d2771..a24c9e65cc8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6540,6 +6540,7 @@ name = "polkadot-overseer-gen-proc-macro" version = "0.1.0" dependencies = [ "assert_matches", + "proc-macro-crate 1.0.0", "proc-macro2", "quote", "syn", diff --git a/node/overseer/overseer-gen/proc-macro/Cargo.toml b/node/overseer/overseer-gen/proc-macro/Cargo.toml index 87d6d471140e..ba8b32f96825 100644 --- a/node/overseer/overseer-gen/proc-macro/Cargo.toml +++ b/node/overseer/overseer-gen/proc-macro/Cargo.toml @@ -15,6 +15,7 @@ proc-macro = true syn = { version = "1.0.60", features = ["full", "extra-traits"] } quote = "1.0.9" proc-macro2 = "1.0.26" +proc-macro-crate = "1.0.0" [dev-dependencies] assert_matches = "1.5.0" diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 035f50ea9331..878b6e8e1744 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -46,6 +46,8 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Result Result, #error_ty>, )* @@ -101,7 +103,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result; + pub type #handler = #support_crate ::metered::MeteredSender< #event >; #[allow(missing_docs)] pub struct #builder #builder_generics { @@ -118,7 +120,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Self { // explicitly assure the required traits are implemented fn trait_from_must_be_implemented() - where E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::OverseerError> + where E: std::error::Error + Send + Sync + 'static + From<#support_crate ::OverseerError> {} trait_from_must_be_implemented::< #error_ty >(); @@ -137,7 +139,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Self where S: ::polkadot_overseer_gen::SpawnNamed + Send { + pub fn spawner(mut self, spawner: S) -> Self where S: #support_crate ::SpawnNamed + Send { self.spawner = Some(spawner); self } @@ -161,27 +163,27 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result ::std::result::Result<(#overseer_name #generics, #handler), #error_ty> { - let (events_tx, events_rx) = ::polkadot_overseer_gen::metered::channel::< + let (events_tx, events_rx) = #support_crate ::metered::channel::< #event >(SIGNAL_CHANNEL_CAPACITY); let handler: #handler = events_tx.clone(); - let (to_overseer_tx, to_overseer_rx) = ::polkadot_overseer_gen::metered::unbounded::< + let (to_overseer_tx, to_overseer_rx) = #support_crate ::metered::unbounded::< ToOverseer >(); #( let (#channel_name_tx, #channel_name_rx) = - ::polkadot_overseer_gen::metered::channel::< + #support_crate ::metered::channel::< MessagePacket< #consumes > >(CHANNEL_CAPACITY); )* #( let (#channel_name_unbounded_tx, #channel_name_unbounded_rx) = - ::polkadot_overseer_gen::metered::unbounded::< + #support_crate ::metered::unbounded::< MessagePacket< #consumes > >(); )* @@ -198,7 +200,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result > >::new(); @@ -209,10 +211,10 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result = ::polkadot_overseer_gen::select( + let message_rx: SubsystemIncomingMessages< #consumes > = #support_crate ::select( #channel_name_rx, #channel_name_unbounded_rx ); - let (signal_tx, signal_rx) = ::polkadot_overseer_gen::metered::channel(SIGNAL_CHANNEL_CAPACITY); + let (signal_tx, signal_rx) = #support_crate ::metered::channel(SIGNAL_CHANNEL_CAPACITY); let ctx = #subsyste_ctx_name::< #consumes >::new( signal_rx, message_rx, @@ -237,7 +239,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Result Result { let signal = &info.extern_signal_ty; let error_ty = &info.extern_error_ty; + let support_crate = info.support_crate_name(); let ts = quote! { - use ::polkadot_overseer_gen::FutureExt as _; + use #support_crate ::FutureExt as _; /// Task kind to launch. pub trait TaskKind { @@ -296,33 +299,33 @@ pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result( spawner: &mut S, - message_tx: ::polkadot_overseer_gen::metered::MeteredSender>, - signal_tx: ::polkadot_overseer_gen::metered::MeteredSender< #signal >, + message_tx: #support_crate ::metered::MeteredSender>, + signal_tx: #support_crate ::metered::MeteredSender< #signal >, // meter for the unbounded channel - unbounded_meter: ::polkadot_overseer_gen::metered::Meter, + unbounded_meter: #support_crate ::metered::Meter, // connection to the subsystems channels_out: ChannelsOut, ctx: Ctx, s: SubSys, - futures: &mut ::polkadot_overseer_gen::FuturesUnordered >>, + futures: &mut #support_crate ::FuturesUnordered >>, ) -> ::std::result::Result, #error_ty > where - S: ::polkadot_overseer_gen::SpawnNamed, + S: #support_crate ::SpawnNamed, M: std::fmt::Debug + Send + 'static, TK: TaskKind, - Ctx: ::polkadot_overseer_gen::SubsystemContext, - E: std::error::Error + Send + Sync + 'static + From<::polkadot_overseer_gen::OverseerError>, - SubSys: ::polkadot_overseer_gen::Subsystem, + Ctx: #support_crate ::SubsystemContext, + E: std::error::Error + Send + Sync + 'static + From<#support_crate ::OverseerError>, + SubSys: #support_crate ::Subsystem, { - let ::polkadot_overseer_gen::SpawnedSubsystem:: { future, name } = s.start(ctx); + let #support_crate ::SpawnedSubsystem:: { future, name } = s.start(ctx); - let (tx, rx) = ::polkadot_overseer_gen::oneshot::channel(); + let (tx, rx) = #support_crate ::oneshot::channel(); let fut = Box::pin(async move { if let Err(e) = future.await { - ::polkadot_overseer_gen::tracing::error!(subsystem=name, err = ?e, "subsystem exited with error"); + #support_crate ::tracing::error!(subsystem=name, err = ?e, "subsystem exited with error"); } else { - ::polkadot_overseer_gen::tracing::debug!(subsystem=name, "subsystem exited without an error"); + #support_crate ::tracing::debug!(subsystem=name, "subsystem exited without an error"); } let _ = tx.send(()); }); @@ -337,7 +340,7 @@ pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result Result Result >, )* @@ -48,7 +50,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result >, )* @@ -65,7 +67,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result { self. #channel_name .send( - ::polkadot_overseer_gen::make_packet(signals_received, inner) + #support_crate ::make_packet(signals_received, inner) ).await } )* @@ -78,7 +80,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Result { self. #channel_name_unbounded .unbounded_send( - ::polkadot_overseer_gen::make_packet(signals_received, inner) + #support_crate ::make_packet(signals_received, inner) ) .map_err(|e| e.into_send_error()) }, @@ -111,7 +113,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Result let signal = &info.extern_signal_ty; let wrapper_message = &info.message_wrapper; let error_ty = &info.extern_error_ty; + let support_crate = info.support_crate_name(); let ts = quote! { /// Connector to send messages towards all subsystems, @@ -42,7 +43,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result } /// impl for wrapping message type... - #[::polkadot_overseer_gen::async_trait] + #[#support_crate ::async_trait] impl SubsystemSender< #wrapper_message > for #subsystem_sender_name { async fn send_message(&mut self, msg: #wrapper_message) { self.channels.send_and_log_error(self.signals_received.load(), msg).await; @@ -68,7 +69,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result // the necessity for manual wrapping, and do the conversion // based on the generated `From::from` impl for the individual variants. #( - #[::polkadot_overseer_gen::async_trait] + #[#support_crate ::async_trait] impl SubsystemSender< #consumes > for #subsystem_sender_name { async fn send_message(&mut self, msg: #consumes) { self.channels.send_and_log_error(self.signals_received.load(), #wrapper_message ::from ( msg )).await; @@ -101,11 +102,11 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result #[derive(Debug)] #[allow(missing_docs)] pub struct #subsystem_ctx_name{ - signals: ::polkadot_overseer_gen::metered::MeteredReceiver< #signal >, + signals: #support_crate ::metered::MeteredReceiver< #signal >, messages: SubsystemIncomingMessages, to_subsystems: #subsystem_sender_name, - to_overseer: ::polkadot_overseer_gen::metered::UnboundedMeteredSender< - ::polkadot_overseer_gen::ToOverseer + to_overseer: #support_crate ::metered::UnboundedMeteredSender< + #support_crate ::ToOverseer >, signals_received: SignalsReceived, pending_incoming: Option<(usize, M)>, @@ -114,10 +115,10 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result impl #subsystem_ctx_name { /// Create a new context. fn new( - signals: ::polkadot_overseer_gen::metered::MeteredReceiver< #signal >, + signals: #support_crate ::metered::MeteredReceiver< #signal >, messages: SubsystemIncomingMessages, to_subsystems: ChannelsOut, - to_overseer: ::polkadot_overseer_gen::metered::UnboundedMeteredSender, + to_overseer: #support_crate ::metered::UnboundedMeteredSender, ) -> Self { let signals_received = SignalsReceived::default(); #subsystem_ctx_name { @@ -134,10 +135,10 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result } } - #[::polkadot_overseer_gen::async_trait] + #[#support_crate ::async_trait] impl SubsystemContext for #subsystem_ctx_name where - #subsystem_sender_name: ::polkadot_overseer_gen::SubsystemSender< #wrapper_message >, + #subsystem_sender_name: #support_crate ::SubsystemSender< #wrapper_message >, #wrapper_message: From, { type Message = M; @@ -147,9 +148,9 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result type Error = #error_ty; async fn try_recv(&mut self) -> ::std::result::Result>, ()> { - match ::polkadot_overseer_gen::poll!(self.recv()) { - ::polkadot_overseer_gen::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), - ::polkadot_overseer_gen::Poll::Pending => Ok(None), + match #support_crate ::poll!(self.recv()) { + #support_crate ::Poll::Ready(msg) => Ok(Some(msg.map_err(|_| ())?)), + #support_crate ::Poll::Pending => Ok(None), } } @@ -159,19 +160,19 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result // in the meantime. if let Some((needs_signals_received, msg)) = self.pending_incoming.take() { if needs_signals_received <= self.signals_received.load() { - return Ok(::polkadot_overseer_gen::FromOverseer::Communication { msg }); + return Ok(#support_crate ::FromOverseer::Communication { msg }); } else { self.pending_incoming = Some((needs_signals_received, msg)); // wait for next signal. let signal = self.signals.next().await - .ok_or(::polkadot_overseer_gen::OverseerError::Context( + .ok_or(#support_crate ::OverseerError::Context( "Signal channel is terminated and empty." .to_owned() ))?; self.signals_received.inc(); - return Ok(::polkadot_overseer_gen::FromOverseer::Signal(signal)) + return Ok(#support_crate ::FromOverseer::Signal(signal)) } } @@ -181,19 +182,19 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result let pending_incoming = &mut self.pending_incoming; // Otherwise, wait for the next signal or incoming message. - let from_overseer = ::polkadot_overseer_gen::futures::select_biased! { + let from_overseer = #support_crate ::futures::select_biased! { signal = await_signal => { let signal = signal - .ok_or(::polkadot_overseer_gen::OverseerError::Context( + .ok_or(#support_crate ::OverseerError::Context( "Signal channel is terminated and empty." .to_owned() ))?; - ::polkadot_overseer_gen::FromOverseer::Signal(signal) + #support_crate ::FromOverseer::Signal(signal) } msg = await_message => { let packet = msg - .ok_or(::polkadot_overseer_gen::OverseerError::Context( + .ok_or(#support_crate ::OverseerError::Context( "Message channel is terminated and empty." .to_owned() ))?; @@ -204,12 +205,12 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result continue; } else { // we know enough to return this message. - ::polkadot_overseer_gen::FromOverseer::Communication { msg: packet.message} + #support_crate ::FromOverseer::Communication { msg: packet.message} } } }; - if let ::polkadot_overseer_gen::FromOverseer::Signal(_) = from_overseer { + if let #support_crate ::FromOverseer::Signal(_) = from_overseer { self.signals_received.inc(); } @@ -224,7 +225,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result fn spawn(&mut self, name: &'static str, s: Pin + Send>>) -> ::std::result::Result<(), #error_ty> { - self.to_overseer.unbounded_send(::polkadot_overseer_gen::ToOverseer::SpawnJob { + self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnJob { name, s, }).map_err(|_| SubsystemError::TaskSpawn(name)) @@ -233,7 +234,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) -> ::std::result::Result<(), #error_ty> { - self.to_overseer.unbounded_send(::polkadot_overseer_gen::ToOverseer::SpawnBlockingJob { + self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnBlockingJob { name, s, }).map_err(|_| SubsystemError::TaskSpawn(name)) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index 5968386bab55..ca27daec84bb 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -23,6 +23,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result Result Result> >, /// Gather running subsystems' outbound streams into one. - to_overseer_rx: ::polkadot_overseer_gen::stream::Fuse< - ::polkadot_overseer_gen::metered::UnboundedMeteredReceiver< ToOverseer > + to_overseer_rx: #support_crate ::stream::Fuse< + #support_crate ::metered::UnboundedMeteredReceiver< ToOverseer > >, /// Events that are sent to the overseer from the outside world. - events_rx: ::polkadot_overseer_gen::metered::MeteredReceiver< #event_ty >, + events_rx: #support_crate ::metered::MeteredReceiver< #event_ty >, } impl #generics #overseer_name #generics #where_clause { @@ -109,7 +110,7 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result Result Result { let signal = &info.extern_signal_ty; let error_ty = &info.extern_error_ty; + let support_crate = info.support_crate_name(); let ts = quote::quote! { - use ::polkadot_overseer_gen::futures::SinkExt as _; + use #support_crate ::futures::SinkExt as _; /// A subsystem that the overseer oversees. /// @@ -198,7 +200,7 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { /// The instance. pub instance: std::option::Option< - ::polkadot_overseer_gen::SubsystemInstance + #support_crate ::SubsystemInstance >, } @@ -216,14 +218,14 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { - ::polkadot_overseer_gen::tracing::error!( + #support_crate ::tracing::error!( target: LOG_TARGET, %origin, "Subsystem {} appears unresponsive.", instance.name, ); Err(#error_ty :: from( - ::polkadot_overseer_gen::OverseerError::SubsystemStalled(instance.name) + #support_crate ::OverseerError::SubsystemStalled(instance.name) )) } Some(res) => res.map_err(Into::into), @@ -243,7 +245,7 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result { Err(#error_ty :: from( - ::polkadot_overseer_gen::OverseerError::SubsystemStalled(instance.name) + #support_crate ::OverseerError::SubsystemStalled(instance.name) )) } Some(res) => { diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 021bd55dba48..653e7cb3b5fe 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -16,8 +16,9 @@ #![deny(unused_crate_dependencies)] -use proc_macro2::TokenStream; +use proc_macro2::{Span, Ident, TokenStream}; use syn::{parse2, Result}; +use quote::{quote, ToTokens}; mod impl_builder; mod impl_misc; @@ -50,7 +51,16 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< let of: OverseerGuts = parse2(orig)?; + let support_crate_name = { + use proc_macro_crate::{crate_name, FoundCrate}; + let crate_name = crate_name("polkadot-overseer-gen").expect("Support crate polkadot-overseer-gen is present in `Cargo.toml`. qed"); + match crate_name { + FoundCrate::Itself => quote!{crate}, + FoundCrate::Name(name) => Ident::new(&name, Span::call_site()).to_token_stream(), + } + }; let info = OverseerInfo { + support_crate_name, subsystems: of.subsystems, baggage: of.baggage, overseer_name: of.name, diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 010235987ea3..38b47692eebd 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -121,6 +121,9 @@ pub(crate) struct BaggageField { #[derive(Clone, Debug)] pub(crate) struct OverseerInfo { + /// Where the support crate `::polkadot_overseer_gen` lives. + pub(crate) support_crate_name: TokenStream, + /// Fields annotated with `#[subsystem(..)]`. pub(crate) subsystems: Vec, /// Fields that do not define a subsystem, @@ -155,6 +158,10 @@ pub(crate) struct OverseerInfo { } impl OverseerInfo { + pub(crate) fn support_crate_name(&self) -> &TokenStream { + &self.support_crate_name + } + pub(crate) fn variant_names(&self) -> Vec { self.subsystems .iter() From 662215e827c93e955a1aee4c0671fc1d004e2286 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 12:09:05 +0200 Subject: [PATCH 140/161] chores --- node/overseer/overseer-gen/src/lib.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index a4b6b817cc96..76a693e9f55a 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -73,6 +73,8 @@ pub use sp_core::traits::SpawnNamed; pub use futures::{ self, select, + StreamExt, + FutureExt, poll, future::{ Fuse, Future, BoxFuture @@ -92,18 +94,15 @@ pub use async_trait::async_trait; #[doc(hidden)] pub use std::time::Duration; -use std::sync{Arc, atomic::{self, AtomicUsize}}; +use std::sync::{Arc, atomic::{self, AtomicUsize}}; #[doc(hidden)] pub use futures_timer::Delay; -#[doc(hidden)] -pub use futures_util::stream::StreamExt; -#[doc(hidden)] -pub use futures_util::future::FutureExt; - pub use polkadot_node_network_protocol::WrongVariant; +use std::fmt; + /// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. /// /// Used to launch jobs. @@ -128,8 +127,6 @@ pub enum ToOverseer { }, } -use std::fmt; - impl fmt::Debug for ToOverseer { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { From a3f3c4716ad3c37eaa691c4451b4b5059107a79d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 12:11:18 +0200 Subject: [PATCH 141/161] extra line breaks --- .../proc-macro/src/impl_builder.rs | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 878b6e8e1744..524fdcb18777 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -80,7 +80,10 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Result Self { // explicitly assure the required traits are implemented fn trait_from_must_be_implemented() - where E: std::error::Error + Send + Sync + 'static + From<#support_crate ::OverseerError> + where + E: std::error::Error + Send + Sync + 'static + From<#support_crate ::OverseerError> {} trait_from_must_be_implemented::< #error_ty >(); @@ -139,7 +143,10 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Self where S: #support_crate ::SpawnNamed + Send { + pub fn spawner(mut self, spawner: S) -> Self + where + S: #support_crate ::SpawnNamed + Send + { self.spawner = Some(spawner); self } @@ -236,7 +243,12 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Date: Thu, 8 Jul 2021 12:17:08 +0200 Subject: [PATCH 142/161] remove stale dependency --- Cargo.lock | 1 - node/overseer/overseer-gen/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a24c9e65cc8b..f034dcb2fffb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6524,7 +6524,6 @@ dependencies = [ "async-trait", "futures 0.3.15", "futures-timer 3.0.2", - "futures-util", "metered-channel", "pin-project 1.0.7", "polkadot-node-network-protocol", diff --git a/node/overseer/overseer-gen/Cargo.toml b/node/overseer/overseer-gen/Cargo.toml index 62e020b1e640..bb7f4836c6f1 100644 --- a/node/overseer/overseer-gen/Cargo.toml +++ b/node/overseer/overseer-gen/Cargo.toml @@ -17,7 +17,6 @@ polkadot-node-network-protocol = { path = "../../network/protocol"} sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } futures-timer = "3.0.2" pin-project = "1.0" -futures-util = "0.3" [dev-dependencies] trybuild = "1.0.41" From c90f8b48db74791174ab9ac1b9e8d810a24abe45 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 12:18:40 +0200 Subject: [PATCH 143/161] bitfield signing test fmt + license fixup --- node/core/bitfield-signing/src/tests.rs | 120 ++++++++++++++---------- 1 file changed, 68 insertions(+), 52 deletions(-) diff --git a/node/core/bitfield-signing/src/tests.rs b/node/core/bitfield-signing/src/tests.rs index af76351ccfda..a5f8e564599f 100644 --- a/node/core/bitfield-signing/src/tests.rs +++ b/node/core/bitfield-signing/src/tests.rs @@ -1,67 +1,83 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + use super::*; use futures::{pin_mut, executor::block_on}; use polkadot_primitives::v1::{CandidateHash, OccupiedCore}; use polkadot_node_subsystem::messages::AllMessages; fn occupied_core(para_id: u32, candidate_hash: CandidateHash) -> CoreState { - CoreState::Occupied(OccupiedCore { - group_responsible: para_id.into(), - next_up_on_available: None, - occupied_since: 100_u32, - time_out_at: 200_u32, - next_up_on_time_out: None, - availability: Default::default(), - candidate_hash, - candidate_descriptor: Default::default(), - }) + CoreState::Occupied(OccupiedCore { + group_responsible: para_id.into(), + next_up_on_available: None, + occupied_since: 100_u32, + time_out_at: 200_u32, + next_up_on_time_out: None, + availability: Default::default(), + candidate_hash, + candidate_descriptor: Default::default(), + }) } #[test] fn construct_availability_bitfield_works() { - block_on(async move { - let relay_parent = Hash::default(); - let validator_index = ValidatorIndex(1u32); + block_on(async move { + let relay_parent = Hash::default(); + let validator_index = ValidatorIndex(1u32); - let (mut sender, mut receiver) = polkadot_node_subsystem_test_helpers::sender_receiver(); - let future = construct_availability_bitfield( - relay_parent, - &jaeger::Span::Disabled, - validator_index, - &mut sender, - ).fuse(); - pin_mut!(future); + let (mut sender, mut receiver) = polkadot_node_subsystem_test_helpers::sender_receiver(); + let future = construct_availability_bitfield( + relay_parent, + &jaeger::Span::Disabled, + validator_index, + &mut sender, + ).fuse(); + pin_mut!(future); - let hash_a = CandidateHash(Hash::repeat_byte(1)); - let hash_b = CandidateHash(Hash::repeat_byte(2)); + let hash_a = CandidateHash(Hash::repeat_byte(1)); + let hash_b = CandidateHash(Hash::repeat_byte(2)); - loop { - futures::select! { - m = receiver.next() => match m.unwrap() { - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(rp, RuntimeApiRequest::AvailabilityCores(tx)), - ) => { - assert_eq!(relay_parent, rp); - tx.send(Ok(vec![CoreState::Free, occupied_core(1, hash_a), occupied_core(2, hash_b)])).unwrap(); - } - AllMessages::AvailabilityStore( - AvailabilityStoreMessage::QueryChunkAvailability(c_hash, vidx, tx), - ) => { - assert_eq!(validator_index, vidx); + loop { + futures::select! { + m = receiver.next() => match m.unwrap() { + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(rp, RuntimeApiRequest::AvailabilityCores(tx)), + ) => { + assert_eq!(relay_parent, rp); + tx.send(Ok(vec![CoreState::Free, occupied_core(1, hash_a), occupied_core(2, hash_b)])).unwrap(); + } + AllMessages::AvailabilityStore( + AvailabilityStoreMessage::QueryChunkAvailability(c_hash, vidx, tx), + ) => { + assert_eq!(validator_index, vidx); - tx.send(c_hash == hash_a).unwrap(); - }, - o => panic!("Unknown message: {:?}", o), - }, - r = future => match r { - Ok(r) => { - assert!(!r.0.get(0).unwrap()); - assert!(r.0.get(1).unwrap()); - assert!(!r.0.get(2).unwrap()); - break - }, - Err(e) => panic!("Failed: {:?}", e), - }, - } - } - }); + tx.send(c_hash == hash_a).unwrap(); + }, + o => panic!("Unknown message: {:?}", o), + }, + r = future => match r { + Ok(r) => { + assert!(!r.0.get(0).unwrap()); + assert!(r.0.get(1).unwrap()); + assert!(!r.0.get(2).unwrap()); + break + }, + Err(e) => panic!("Failed: {:?}", e), + }, + } + } + }); } From 9662033847358987535c7330cf9076c095146389 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 12:21:36 +0200 Subject: [PATCH 144/161] tabs --- node/overseer/overseer-gen/src/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/overseer/overseer-gen/src/tests.rs b/node/overseer/overseer-gen/src/tests.rs index b14e2695a939..1dc1d2e86d59 100644 --- a/node/overseer/overseer-gen/src/tests.rs +++ b/node/overseer/overseer-gen/src/tests.rs @@ -5,6 +5,6 @@ #[test] fn ui_compile_fail() { - let t = trybuild::TestCases::new(); - t.compile_fail("tests/ui/err-*.rs"); + let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/err-*.rs"); } From 7dd86d720744e414c9c20384acc6cb9a28e878c0 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 12:23:41 +0200 Subject: [PATCH 145/161] more tabs --- node/core/chain-api/src/tests.rs | 470 +++--- .../tests/ui/ok-01-w-generics.rs | 14 +- node/overseer/src/tests.rs | 1390 ++++++++--------- 3 files changed, 937 insertions(+), 937 deletions(-) diff --git a/node/core/chain-api/src/tests.rs b/node/core/chain-api/src/tests.rs index 015030cfab24..730a6a2ae69d 100644 --- a/node/core/chain-api/src/tests.rs +++ b/node/core/chain-api/src/tests.rs @@ -12,10 +12,10 @@ use sp_core::testing::TaskExecutor; #[derive(Clone)] struct TestClient { - blocks: BTreeMap, - block_weights: BTreeMap, - finalized_blocks: BTreeMap, - headers: BTreeMap, + blocks: BTreeMap, + block_weights: BTreeMap, + finalized_blocks: BTreeMap, + headers: BTreeMap, } const ONE: Hash = Hash::repeat_byte(0x01); @@ -25,280 +25,280 @@ const FOUR: Hash = Hash::repeat_byte(0x04); const ERROR_PATH: Hash = Hash::repeat_byte(0xFF); fn default_header() -> Header { - Header { - parent_hash: Hash::zero(), - number: 100500, - state_root: Hash::zero(), - extrinsics_root: Hash::zero(), - digest: Default::default(), - } + Header { + parent_hash: Hash::zero(), + number: 100500, + state_root: Hash::zero(), + extrinsics_root: Hash::zero(), + digest: Default::default(), + } } impl Default for TestClient { - fn default() -> Self { - Self { - blocks: maplit::btreemap! { - ONE => 1, - TWO => 2, - THREE => 3, - FOUR => 4, - }, - block_weights: maplit::btreemap! { - ONE => 0, - TWO => 1, - THREE => 1, - FOUR => 2, - }, - finalized_blocks: maplit::btreemap! { - 1 => ONE, - 3 => THREE, - }, - headers: maplit::btreemap! { - TWO => Header { - parent_hash: ONE, - number: 2, - ..default_header() - }, - THREE => Header { - parent_hash: TWO, - number: 3, - ..default_header() - }, - FOUR => Header { - parent_hash: THREE, - number: 4, - ..default_header() - }, - ERROR_PATH => Header { - ..default_header() - } - }, - } - } + fn default() -> Self { + Self { + blocks: maplit::btreemap! { + ONE => 1, + TWO => 2, + THREE => 3, + FOUR => 4, + }, + block_weights: maplit::btreemap! { + ONE => 0, + TWO => 1, + THREE => 1, + FOUR => 2, + }, + finalized_blocks: maplit::btreemap! { + 1 => ONE, + 3 => THREE, + }, + headers: maplit::btreemap! { + TWO => Header { + parent_hash: ONE, + number: 2, + ..default_header() + }, + THREE => Header { + parent_hash: TWO, + number: 3, + ..default_header() + }, + FOUR => Header { + parent_hash: THREE, + number: 4, + ..default_header() + }, + ERROR_PATH => Header { + ..default_header() + } + }, + } + } } fn last_key_value(map: &BTreeMap) -> (K, V) { - assert!(!map.is_empty()); - map.iter() - .last() - .map(|(k, v)| (k.clone(), v.clone())) - .unwrap() + assert!(!map.is_empty()); + map.iter() + .last() + .map(|(k, v)| (k.clone(), v.clone())) + .unwrap() } impl HeaderBackend for TestClient { - fn info(&self) -> BlockInfo { - let genesis_hash = self.blocks.iter().next().map(|(h, _)| *h).unwrap(); - let (best_hash, best_number) = last_key_value(&self.blocks); - let (finalized_number, finalized_hash) = last_key_value(&self.finalized_blocks); - - BlockInfo { - best_hash, - best_number, - genesis_hash, - finalized_hash, - finalized_number, - number_leaves: 0, - finalized_state: None, - } - } - fn number(&self, hash: Hash) -> sp_blockchain::Result> { - Ok(self.blocks.get(&hash).copied()) - } - fn hash(&self, number: BlockNumber) -> sp_blockchain::Result> { - Ok(self.finalized_blocks.get(&number).copied()) - } - fn header(&self, id: BlockId) -> sp_blockchain::Result> { - match id { - // for error path testing - BlockId::Hash(hash) if hash.is_zero() => { - Err(sp_blockchain::Error::Backend("Zero hashes are illegal!".into())) - } - BlockId::Hash(hash) => { - Ok(self.headers.get(&hash).cloned()) - } - _ => unreachable!(), - } - } - fn status(&self, _id: BlockId) -> sp_blockchain::Result { - unimplemented!() - } + fn info(&self) -> BlockInfo { + let genesis_hash = self.blocks.iter().next().map(|(h, _)| *h).unwrap(); + let (best_hash, best_number) = last_key_value(&self.blocks); + let (finalized_number, finalized_hash) = last_key_value(&self.finalized_blocks); + + BlockInfo { + best_hash, + best_number, + genesis_hash, + finalized_hash, + finalized_number, + number_leaves: 0, + finalized_state: None, + } + } + fn number(&self, hash: Hash) -> sp_blockchain::Result> { + Ok(self.blocks.get(&hash).copied()) + } + fn hash(&self, number: BlockNumber) -> sp_blockchain::Result> { + Ok(self.finalized_blocks.get(&number).copied()) + } + fn header(&self, id: BlockId) -> sp_blockchain::Result> { + match id { + // for error path testing + BlockId::Hash(hash) if hash.is_zero() => { + Err(sp_blockchain::Error::Backend("Zero hashes are illegal!".into())) + } + BlockId::Hash(hash) => { + Ok(self.headers.get(&hash).cloned()) + } + _ => unreachable!(), + } + } + fn status(&self, _id: BlockId) -> sp_blockchain::Result { + unimplemented!() + } } fn test_harness( - test: impl FnOnce(Arc, TestSubsystemContextHandle) - -> BoxFuture<'static, ()>, + test: impl FnOnce(Arc, TestSubsystemContextHandle) + -> BoxFuture<'static, ()>, ) { - let (ctx, ctx_handle) = make_subsystem_context(TaskExecutor::new()); - let client = Arc::new(TestClient::default()); + let (ctx, ctx_handle) = make_subsystem_context(TaskExecutor::new()); + let client = Arc::new(TestClient::default()); - let subsystem = ChainApiSubsystem::new(client.clone(), Metrics(None)); - let chain_api_task = run(ctx, subsystem).map(|x| x.unwrap()); - let test_task = test(client, ctx_handle); + let subsystem = ChainApiSubsystem::new(client.clone(), Metrics(None)); + let chain_api_task = run(ctx, subsystem).map(|x| x.unwrap()); + let test_task = test(client, ctx_handle); - futures::executor::block_on(future::join(chain_api_task, test_task)); + futures::executor::block_on(future::join(chain_api_task, test_task)); } impl AuxStore for TestClient { - fn insert_aux< - 'a, - 'b: 'a, - 'c: 'a, - I: IntoIterator, - D: IntoIterator, - >( - &self, - _insert: I, - _delete: D, - ) -> sp_blockchain::Result<()> { - unimplemented!() - } - - fn get_aux(&self, key: &[u8]) -> sp_blockchain::Result>> { - Ok(self - .block_weights - .iter() - .find(|(hash, _)| sc_consensus_babe::aux_schema::block_weight_key(hash) == key) - .map(|(_, weight)| weight.encode())) - } + fn insert_aux< + 'a, + 'b: 'a, + 'c: 'a, + I: IntoIterator, + D: IntoIterator, + >( + &self, + _insert: I, + _delete: D, + ) -> sp_blockchain::Result<()> { + unimplemented!() + } + + fn get_aux(&self, key: &[u8]) -> sp_blockchain::Result>> { + Ok(self + .block_weights + .iter() + .find(|(hash, _)| sc_consensus_babe::aux_schema::block_weight_key(hash) == key) + .map(|(_, weight)| weight.encode())) + } } #[test] fn request_block_number() { - test_harness(|client, mut sender| { - async move { - let zero = Hash::zero(); - let test_cases = [ - (TWO, client.number(TWO).unwrap()), - (zero, client.number(zero).unwrap()), // not here - ]; - for (hash, expected) in &test_cases { - let (tx, rx) = oneshot::channel(); - - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::BlockNumber(*hash, tx), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), *expected); - } - - sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }.boxed() - }) + test_harness(|client, mut sender| { + async move { + let zero = Hash::zero(); + let test_cases = [ + (TWO, client.number(TWO).unwrap()), + (zero, client.number(zero).unwrap()), // not here + ]; + for (hash, expected) in &test_cases { + let (tx, rx) = oneshot::channel(); + + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::BlockNumber(*hash, tx), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), *expected); + } + + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) } #[test] fn request_block_header() { - test_harness(|client, mut sender| { - async move { - const NOT_HERE: Hash = Hash::repeat_byte(0x5); - let test_cases = [ - (TWO, client.header(BlockId::Hash(TWO)).unwrap()), - (NOT_HERE, client.header(BlockId::Hash(NOT_HERE)).unwrap()), - ]; - for (hash, expected) in &test_cases { - let (tx, rx) = oneshot::channel(); - - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::BlockHeader(*hash, tx), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), *expected); - } - - sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }.boxed() - }) + test_harness(|client, mut sender| { + async move { + const NOT_HERE: Hash = Hash::repeat_byte(0x5); + let test_cases = [ + (TWO, client.header(BlockId::Hash(TWO)).unwrap()), + (NOT_HERE, client.header(BlockId::Hash(NOT_HERE)).unwrap()), + ]; + for (hash, expected) in &test_cases { + let (tx, rx) = oneshot::channel(); + + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::BlockHeader(*hash, tx), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), *expected); + } + + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) } #[test] fn request_block_weight() { - test_harness(|client, mut sender| { - async move { - const NOT_HERE: Hash = Hash::repeat_byte(0x5); - let test_cases = [ - (TWO, sc_consensus_babe::block_weight(&*client, TWO).unwrap()), - (FOUR, sc_consensus_babe::block_weight(&*client, FOUR).unwrap()), - (NOT_HERE, sc_consensus_babe::block_weight(&*client, NOT_HERE).unwrap()), - ]; - for (hash, expected) in &test_cases { - let (tx, rx) = oneshot::channel(); - - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::BlockWeight(*hash, tx), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), *expected); - } - - sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }.boxed() - }) + test_harness(|client, mut sender| { + async move { + const NOT_HERE: Hash = Hash::repeat_byte(0x5); + let test_cases = [ + (TWO, sc_consensus_babe::block_weight(&*client, TWO).unwrap()), + (FOUR, sc_consensus_babe::block_weight(&*client, FOUR).unwrap()), + (NOT_HERE, sc_consensus_babe::block_weight(&*client, NOT_HERE).unwrap()), + ]; + for (hash, expected) in &test_cases { + let (tx, rx) = oneshot::channel(); + + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::BlockWeight(*hash, tx), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), *expected); + } + + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) } #[test] fn request_finalized_hash() { - test_harness(|client, mut sender| { - async move { - let test_cases = [ - (1, client.hash(1).unwrap()), // not here - (2, client.hash(2).unwrap()), - ]; - for (number, expected) in &test_cases { - let (tx, rx) = oneshot::channel(); - - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::FinalizedBlockHash(*number, tx), - }).await; - - assert_eq!(rx.await.unwrap().unwrap(), *expected); - } - - sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }.boxed() - }) + test_harness(|client, mut sender| { + async move { + let test_cases = [ + (1, client.hash(1).unwrap()), // not here + (2, client.hash(2).unwrap()), + ]; + for (number, expected) in &test_cases { + let (tx, rx) = oneshot::channel(); + + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::FinalizedBlockHash(*number, tx), + }).await; + + assert_eq!(rx.await.unwrap().unwrap(), *expected); + } + + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) } #[test] fn request_last_finalized_number() { - test_harness(|client, mut sender| { - async move { - let (tx, rx) = oneshot::channel(); + test_harness(|client, mut sender| { + async move { + let (tx, rx) = oneshot::channel(); - let expected = client.info().finalized_number; - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::FinalizedBlockNumber(tx), - }).await; + let expected = client.info().finalized_number; + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::FinalizedBlockNumber(tx), + }).await; - assert_eq!(rx.await.unwrap().unwrap(), expected); + assert_eq!(rx.await.unwrap().unwrap(), expected); - sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }.boxed() - }) + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) } #[test] fn request_ancestors() { - test_harness(|_client, mut sender| { - async move { - let (tx, rx) = oneshot::channel(); - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::Ancestors { hash: THREE, k: 4, response_channel: tx }, - }).await; - assert_eq!(rx.await.unwrap().unwrap(), vec![TWO, ONE]); - - let (tx, rx) = oneshot::channel(); - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::Ancestors { hash: TWO, k: 1, response_channel: tx }, - }).await; - assert_eq!(rx.await.unwrap().unwrap(), vec![ONE]); - - let (tx, rx) = oneshot::channel(); - sender.send(FromOverseer::Communication { - msg: ChainApiMessage::Ancestors { hash: ERROR_PATH, k: 2, response_channel: tx }, - }).await; - assert!(rx.await.unwrap().is_err()); - - sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; - }.boxed() - }) + test_harness(|_client, mut sender| { + async move { + let (tx, rx) = oneshot::channel(); + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::Ancestors { hash: THREE, k: 4, response_channel: tx }, + }).await; + assert_eq!(rx.await.unwrap().unwrap(), vec![TWO, ONE]); + + let (tx, rx) = oneshot::channel(); + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::Ancestors { hash: TWO, k: 1, response_channel: tx }, + }).await; + assert_eq!(rx.await.unwrap().unwrap(), vec![ONE]); + + let (tx, rx) = oneshot::channel(); + sender.send(FromOverseer::Communication { + msg: ChainApiMessage::Ancestors { hash: ERROR_PATH, k: 2, response_channel: tx }, + }).await; + assert!(rx.await.unwrap().is_err()); + + sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await; + }.boxed() + }) } diff --git a/node/overseer/all-subsystems-gen/tests/ui/ok-01-w-generics.rs b/node/overseer/all-subsystems-gen/tests/ui/ok-01-w-generics.rs index b049d40ff486..370c4a872e25 100644 --- a/node/overseer/all-subsystems-gen/tests/ui/ok-01-w-generics.rs +++ b/node/overseer/all-subsystems-gen/tests/ui/ok-01-w-generics.rs @@ -4,14 +4,14 @@ use polkadot_overseer_all_subsystems_gen::AllSubsystemsGen; #[derive(Clone, AllSubsystemsGen)] struct AllSubsystems { - a: A, - b: B, + a: A, + b: B, } fn main() { - let all = AllSubsystems:: { - a: 0u8, - b: 1u16, - }; - let _all: AllSubsystems<_,_> = all.replace_a::(777_777u32); + let all = AllSubsystems:: { + a: 0u8, + b: 1u16, + }; + let _all: AllSubsystems<_,_> = all.replace_a::(777_777u32); } diff --git a/node/overseer/src/tests.rs b/node/overseer/src/tests.rs index 4d769b8072e8..a4dd0359c70c 100644 --- a/node/overseer/src/tests.rs +++ b/node/overseer/src/tests.rs @@ -23,19 +23,19 @@ use polkadot_primitives::v1::{CollatorPair, CandidateHash}; use polkadot_node_primitives::{CollationResult, CollationGenerationConfig, PoV, BlockData}; use polkadot_node_network_protocol::{PeerId, UnifiedReputationChange}; use polkadot_node_subsystem_types::{ - ActivatedLeaf, LeafStatus, - messages::{ - RuntimeApiRequest, - NetworkBridgeEvent, - }, - jaeger, + ActivatedLeaf, LeafStatus, + messages::{ + RuntimeApiRequest, + NetworkBridgeEvent, + }, + jaeger, }; use crate::{ - self as overseer, - Overseer, - HeadSupportsParachains, - gen::Delay, + self as overseer, + Overseer, + HeadSupportsParachains, + gen::Delay, }; use metered_channel as metered; @@ -52,160 +52,160 @@ struct TestSubsystem1(metered::MeteredSender); impl overseer::Subsystem for TestSubsystem1 where - C: overseer::SubsystemContext, + C: overseer::SubsystemContext, { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let mut sender = self.0; - SpawnedSubsystem { - name: "test-subsystem-1", - future: Box::pin(async move { - let mut i = 0; - loop { - match ctx.recv().await { - Ok(FromOverseer::Communication { .. }) => { - let _ = sender.send(i).await; - i += 1; - continue; - } - Ok(FromOverseer::Signal(OverseerSignal::Conclude)) => return Ok(()), - Err(_) => return Ok(()), - _ => (), - } - } - }), - } - } + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let mut sender = self.0; + SpawnedSubsystem { + name: "test-subsystem-1", + future: Box::pin(async move { + let mut i = 0; + loop { + match ctx.recv().await { + Ok(FromOverseer::Communication { .. }) => { + let _ = sender.send(i).await; + i += 1; + continue; + } + Ok(FromOverseer::Signal(OverseerSignal::Conclude)) => return Ok(()), + Err(_) => return Ok(()), + _ => (), + } + } + }), + } + } } struct TestSubsystem2(metered::MeteredSender); impl overseer::Subsystem for TestSubsystem2 where - C: overseer::SubsystemContext, + C: overseer::SubsystemContext, { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let sender = self.0.clone(); - SpawnedSubsystem { - name: "test-subsystem-2", - future: Box::pin(async move { - let _sender = sender; - let mut c: usize = 0; - loop { - if c < 10 { - let (tx, _) = oneshot::channel(); - ctx.send_message( - CandidateValidationMessage::ValidateFromChainState( - Default::default(), - PoV { - block_data: BlockData(Vec::new()), - }.into(), - tx, - ) - ).await; - c += 1; - continue; - } - match ctx.try_recv().await { - Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => { - break; - } - Ok(Some(_)) => { - continue; - } - Err(_) => return Ok(()), - _ => (), - } - pending!(); - } - - Ok(()) - }), - } - } + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let sender = self.0.clone(); + SpawnedSubsystem { + name: "test-subsystem-2", + future: Box::pin(async move { + let _sender = sender; + let mut c: usize = 0; + loop { + if c < 10 { + let (tx, _) = oneshot::channel(); + ctx.send_message( + CandidateValidationMessage::ValidateFromChainState( + Default::default(), + PoV { + block_data: BlockData(Vec::new()), + }.into(), + tx, + ) + ).await; + c += 1; + continue; + } + match ctx.try_recv().await { + Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => { + break; + } + Ok(Some(_)) => { + continue; + } + Err(_) => return Ok(()), + _ => (), + } + pending!(); + } + + Ok(()) + }), + } + } } struct ReturnOnStart; impl overseer::Subsystem for ReturnOnStart where - C: overseer::SubsystemContext, + C: overseer::SubsystemContext, { - fn start(self, mut _ctx: C) -> SpawnedSubsystem { - SpawnedSubsystem { - name: "test-subsystem-4", - future: Box::pin(async move { - // Do nothing and exit. - Ok(()) - }), - } - } + fn start(self, mut _ctx: C) -> SpawnedSubsystem { + SpawnedSubsystem { + name: "test-subsystem-4", + future: Box::pin(async move { + // Do nothing and exit. + Ok(()) + }), + } + } } struct MockSupportsParachains; impl HeadSupportsParachains for MockSupportsParachains { - fn head_supports_parachains(&self, _head: &Hash) -> bool { - true - } + fn head_supports_parachains(&self, _head: &Hash) -> bool { + true + } } // Checks that a minimal configuration of two jobs can run and exchange messages. #[test] fn overseer_works() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let (s1_tx, s1_rx) = metered::channel::(64); - let (s2_tx, s2_rx) = metered::channel::(64); - - let mut s1_rx = s1_rx.fuse(); - let mut s2_rx = s2_rx.fuse(); - - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_validation(TestSubsystem1(s1_tx)) - .replace_candidate_backing(TestSubsystem2(s2_tx)); - - let (overseer, mut handler) = Overseer::new( - vec![], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - let overseer_fut = overseer.run().fuse(); - - pin_mut!(overseer_fut); - - let mut s1_results = Vec::new(); - let mut s2_results = Vec::new(); - - loop { - select! { - _ = overseer_fut => break, - s1_next = s1_rx.next() => { - match s1_next { - Some(msg) => { - s1_results.push(msg); - if s1_results.len() == 10 { - handler.stop().await; - } - } - None => break, - } - }, - s2_next = s2_rx.next() => { - match s2_next { - Some(_) => s2_results.push(s2_next), - None => break, - } - }, - complete => break, - } - } - - assert_eq!(s1_results, (0..10).collect::>()); - }); + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let (s1_tx, s1_rx) = metered::channel::(64); + let (s2_tx, s2_rx) = metered::channel::(64); + + let mut s1_rx = s1_rx.fuse(); + let mut s2_rx = s2_rx.fuse(); + + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_validation(TestSubsystem1(s1_tx)) + .replace_candidate_backing(TestSubsystem2(s2_tx)); + + let (overseer, mut handler) = Overseer::new( + vec![], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + let overseer_fut = overseer.run().fuse(); + + pin_mut!(overseer_fut); + + let mut s1_results = Vec::new(); + let mut s2_results = Vec::new(); + + loop { + select! { + _ = overseer_fut => break, + s1_next = s1_rx.next() => { + match s1_next { + Some(msg) => { + s1_results.push(msg); + if s1_results.len() == 10 { + handler.stop().await; + } + } + None => break, + } + }, + s2_next = s2_rx.next() => { + match s2_next { + Some(_) => s2_results.push(s2_next), + None => break, + } + }, + complete => break, + } + } + + assert_eq!(s1_results, (0..10).collect::>()); + }); } // Checks activated/deactivated metrics are updated properly. @@ -266,18 +266,18 @@ fn overseer_metrics_work() { } fn extract_metrics(registry: &prometheus::Registry) -> HashMap<&'static str, u64> { - let gather = registry.gather(); - assert_eq!(gather[0].get_name(), "parachain_activated_heads_total"); - assert_eq!(gather[1].get_name(), "parachain_deactivated_heads_total"); - assert_eq!(gather[2].get_name(), "parachain_messages_relayed_total"); - let activated = gather[0].get_metric()[0].get_counter().get_value() as u64; - let deactivated = gather[1].get_metric()[0].get_counter().get_value() as u64; - let relayed = gather[2].get_metric()[0].get_counter().get_value() as u64; - let mut result = HashMap::new(); - result.insert("activated", activated); - result.insert("deactivated", deactivated); - result.insert("relayed", relayed); - result + let gather = registry.gather(); + assert_eq!(gather[0].get_name(), "parachain_activated_heads_total"); + assert_eq!(gather[1].get_name(), "parachain_deactivated_heads_total"); + assert_eq!(gather[2].get_name(), "parachain_messages_relayed_total"); + let activated = gather[0].get_metric()[0].get_counter().get_value() as u64; + let deactivated = gather[1].get_metric()[0].get_counter().get_value() as u64; + let relayed = gather[2].get_metric()[0].get_counter().get_value() as u64; + let mut result = HashMap::new(); + result.insert("activated", activated); + result.insert("deactivated", deactivated); + result.insert("relayed", relayed); + result } // Spawn a subsystem that immediately exits. @@ -285,528 +285,528 @@ fn extract_metrics(registry: &prometheus::Registry) -> HashMap<&'static str, u64 // Should immediately conclude the overseer itself. #[test] fn overseer_ends_on_subsystem_exit() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_backing(ReturnOnStart); - let (overseer, _handle) = Overseer::new( - vec![], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - - overseer.run().await.unwrap(); - }) + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_backing(ReturnOnStart); + let (overseer, _handle) = Overseer::new( + vec![], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + + overseer.run().await.unwrap(); + }) } struct TestSubsystem5(metered::MeteredSender); impl overseer::Subsystem for TestSubsystem5 where - C: overseer::SubsystemContext, + C: overseer::SubsystemContext, { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let mut sender = self.0.clone(); - - SpawnedSubsystem { - name: "test-subsystem-5", - future: Box::pin(async move { - loop { - match ctx.try_recv().await { - Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => break, - Ok(Some(FromOverseer::Signal(s))) => { - sender.send(s).await.unwrap(); - continue; - }, - Ok(Some(_)) => continue, - Err(_) => break, - _ => (), - } - pending!(); - } - - Ok(()) - }), - } - } + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let mut sender = self.0.clone(); + + SpawnedSubsystem { + name: "test-subsystem-5", + future: Box::pin(async move { + loop { + match ctx.try_recv().await { + Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => break, + Ok(Some(FromOverseer::Signal(s))) => { + sender.send(s).await.unwrap(); + continue; + }, + Ok(Some(_)) => continue, + Err(_) => break, + _ => (), + } + pending!(); + } + + Ok(()) + }), + } + } } struct TestSubsystem6(metered::MeteredSender); impl Subsystem for TestSubsystem6 where - C: overseer::SubsystemContext, + C: overseer::SubsystemContext, { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - let mut sender = self.0.clone(); - - SpawnedSubsystem { - name: "test-subsystem-6", - future: Box::pin(async move { - loop { - match ctx.try_recv().await { - Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => break, - Ok(Some(FromOverseer::Signal(s))) => { - sender.send(s).await.unwrap(); - continue; - }, - Ok(Some(_)) => continue, - Err(_) => break, - _ => (), - } - pending!(); - } - - Ok(()) - }), - } - } + fn start(self, mut ctx: C) -> SpawnedSubsystem { + let mut sender = self.0.clone(); + + SpawnedSubsystem { + name: "test-subsystem-6", + future: Box::pin(async move { + loop { + match ctx.try_recv().await { + Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => break, + Ok(Some(FromOverseer::Signal(s))) => { + sender.send(s).await.unwrap(); + continue; + }, + Ok(Some(_)) => continue, + Err(_) => break, + _ => (), + } + pending!(); + } + + Ok(()) + }), + } + } } // Tests that starting with a defined set of leaves and receiving // notifications on imported blocks triggers expected `StartWork` and `StopWork` heartbeats. #[test] fn overseer_start_stop_works() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let first_block_hash = [1; 32].into(); - let second_block_hash = [2; 32].into(); - let third_block_hash = [3; 32].into(); - - let first_block = BlockInfo { - hash: first_block_hash, - parent_hash: [0; 32].into(), - number: 1, - }; - let second_block = BlockInfo { - hash: second_block_hash, - parent_hash: first_block_hash, - number: 2, - }; - let third_block = BlockInfo { - hash: third_block_hash, - parent_hash: second_block_hash, - number: 3, - }; - - let (tx_5, mut rx_5) = metered::channel(64); - let (tx_6, mut rx_6) = metered::channel(64); - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_validation(TestSubsystem5(tx_5)) - .replace_candidate_backing(TestSubsystem6(tx_6)); - let (overseer, mut handler) = Overseer::new( - vec![first_block], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - - let overseer_fut = overseer.run().fuse(); - pin_mut!(overseer_fut); - - let mut ss5_results = Vec::new(); - let mut ss6_results = Vec::new(); - - handler.block_imported(second_block).await; - handler.block_imported(third_block).await; - - let expected_heartbeats = vec![ - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: first_block_hash, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - })), - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: [ActivatedLeaf { - hash: second_block_hash, - number: 2, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - }].as_ref().into(), - deactivated: [first_block_hash].as_ref().into(), - }), - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: [ActivatedLeaf { - hash: third_block_hash, - number: 3, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - }].as_ref().into(), - deactivated: [second_block_hash].as_ref().into(), - }), - ]; - - loop { - select! { - res = overseer_fut => { - assert!(res.is_ok()); - break; - }, - res = rx_5.next() => { - if let Some(res) = res { - ss5_results.push(res); - } - } - res = rx_6.next() => { - if let Some(res) = res { - ss6_results.push(res); - } - } - complete => break, - } - - if ss5_results.len() == expected_heartbeats.len() && - ss6_results.len() == expected_heartbeats.len() { - handler.stop().await; - } - } - - assert_eq!(ss5_results, expected_heartbeats); - assert_eq!(ss6_results, expected_heartbeats); - }); + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let first_block_hash = [1; 32].into(); + let second_block_hash = [2; 32].into(); + let third_block_hash = [3; 32].into(); + + let first_block = BlockInfo { + hash: first_block_hash, + parent_hash: [0; 32].into(), + number: 1, + }; + let second_block = BlockInfo { + hash: second_block_hash, + parent_hash: first_block_hash, + number: 2, + }; + let third_block = BlockInfo { + hash: third_block_hash, + parent_hash: second_block_hash, + number: 3, + }; + + let (tx_5, mut rx_5) = metered::channel(64); + let (tx_6, mut rx_6) = metered::channel(64); + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_validation(TestSubsystem5(tx_5)) + .replace_candidate_backing(TestSubsystem6(tx_6)); + let (overseer, mut handler) = Overseer::new( + vec![first_block], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + + let overseer_fut = overseer.run().fuse(); + pin_mut!(overseer_fut); + + let mut ss5_results = Vec::new(); + let mut ss6_results = Vec::new(); + + handler.block_imported(second_block).await; + handler.block_imported(third_block).await; + + let expected_heartbeats = vec![ + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { + hash: first_block_hash, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + status: LeafStatus::Fresh, + })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: [ActivatedLeaf { + hash: second_block_hash, + number: 2, + span: Arc::new(jaeger::Span::Disabled), + status: LeafStatus::Fresh, + }].as_ref().into(), + deactivated: [first_block_hash].as_ref().into(), + }), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: [ActivatedLeaf { + hash: third_block_hash, + number: 3, + span: Arc::new(jaeger::Span::Disabled), + status: LeafStatus::Fresh, + }].as_ref().into(), + deactivated: [second_block_hash].as_ref().into(), + }), + ]; + + loop { + select! { + res = overseer_fut => { + assert!(res.is_ok()); + break; + }, + res = rx_5.next() => { + if let Some(res) = res { + ss5_results.push(res); + } + } + res = rx_6.next() => { + if let Some(res) = res { + ss6_results.push(res); + } + } + complete => break, + } + + if ss5_results.len() == expected_heartbeats.len() && + ss6_results.len() == expected_heartbeats.len() { + handler.stop().await; + } + } + + assert_eq!(ss5_results, expected_heartbeats); + assert_eq!(ss6_results, expected_heartbeats); + }); } // Tests that starting with a defined set of leaves and receiving // notifications on imported blocks triggers expected `StartWork` and `StopWork` heartbeats. #[test] fn overseer_finalize_works() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let first_block_hash = [1; 32].into(); - let second_block_hash = [2; 32].into(); - let third_block_hash = [3; 32].into(); - - let first_block = BlockInfo { - hash: first_block_hash, - parent_hash: [0; 32].into(), - number: 1, - }; - let second_block = BlockInfo { - hash: second_block_hash, - parent_hash: [42; 32].into(), - number: 2, - }; - let third_block = BlockInfo { - hash: third_block_hash, - parent_hash: second_block_hash, - number: 3, - }; - - let (tx_5, mut rx_5) = metered::channel(64); - let (tx_6, mut rx_6) = metered::channel(64); - - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_validation(TestSubsystem5(tx_5)) - .replace_candidate_backing(TestSubsystem6(tx_6)); - - // start with two forks of different height. - let (overseer, mut handler) = Overseer::new( - vec![first_block, second_block], - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - - let overseer_fut = overseer.run().fuse(); - pin_mut!(overseer_fut); - - let mut ss5_results = Vec::new(); - let mut ss6_results = Vec::new(); - - // this should stop work on both forks we started with earlier. - handler.block_finalized(third_block).await; - - let expected_heartbeats = vec![ - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: [ - ActivatedLeaf { - hash: first_block_hash, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - }, - ActivatedLeaf { - hash: second_block_hash, - number: 2, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - }, - ].as_ref().into(), - ..Default::default() - }), - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - deactivated: [first_block_hash, second_block_hash].as_ref().into(), - ..Default::default() - }), - OverseerSignal::BlockFinalized(third_block_hash, 3), - ]; - - loop { - select! { - res = overseer_fut => { - assert!(res.is_ok()); - break; - }, - res = rx_5.next() => { - if let Some(res) = res { - ss5_results.push(res); - } - } - res = rx_6.next() => { - if let Some(res) = res { - ss6_results.push(res); - } - } - complete => break, - } - - if ss5_results.len() == expected_heartbeats.len() && ss6_results.len() == expected_heartbeats.len() { - handler.stop().await; - } - } - - assert_eq!(ss5_results.len(), expected_heartbeats.len()); - assert_eq!(ss6_results.len(), expected_heartbeats.len()); - - // Notifications on finality for multiple blocks at once - // may be received in different orders. - for expected in expected_heartbeats { - assert!(ss5_results.contains(&expected)); - assert!(ss6_results.contains(&expected)); - } - }); + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let first_block_hash = [1; 32].into(); + let second_block_hash = [2; 32].into(); + let third_block_hash = [3; 32].into(); + + let first_block = BlockInfo { + hash: first_block_hash, + parent_hash: [0; 32].into(), + number: 1, + }; + let second_block = BlockInfo { + hash: second_block_hash, + parent_hash: [42; 32].into(), + number: 2, + }; + let third_block = BlockInfo { + hash: third_block_hash, + parent_hash: second_block_hash, + number: 3, + }; + + let (tx_5, mut rx_5) = metered::channel(64); + let (tx_6, mut rx_6) = metered::channel(64); + + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_validation(TestSubsystem5(tx_5)) + .replace_candidate_backing(TestSubsystem6(tx_6)); + + // start with two forks of different height. + let (overseer, mut handler) = Overseer::new( + vec![first_block, second_block], + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + + let overseer_fut = overseer.run().fuse(); + pin_mut!(overseer_fut); + + let mut ss5_results = Vec::new(); + let mut ss6_results = Vec::new(); + + // this should stop work on both forks we started with earlier. + handler.block_finalized(third_block).await; + + let expected_heartbeats = vec![ + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: [ + ActivatedLeaf { + hash: first_block_hash, + number: 1, + span: Arc::new(jaeger::Span::Disabled), + status: LeafStatus::Fresh, + }, + ActivatedLeaf { + hash: second_block_hash, + number: 2, + span: Arc::new(jaeger::Span::Disabled), + status: LeafStatus::Fresh, + }, + ].as_ref().into(), + ..Default::default() + }), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + deactivated: [first_block_hash, second_block_hash].as_ref().into(), + ..Default::default() + }), + OverseerSignal::BlockFinalized(third_block_hash, 3), + ]; + + loop { + select! { + res = overseer_fut => { + assert!(res.is_ok()); + break; + }, + res = rx_5.next() => { + if let Some(res) = res { + ss5_results.push(res); + } + } + res = rx_6.next() => { + if let Some(res) = res { + ss6_results.push(res); + } + } + complete => break, + } + + if ss5_results.len() == expected_heartbeats.len() && ss6_results.len() == expected_heartbeats.len() { + handler.stop().await; + } + } + + assert_eq!(ss5_results.len(), expected_heartbeats.len()); + assert_eq!(ss6_results.len(), expected_heartbeats.len()); + + // Notifications on finality for multiple blocks at once + // may be received in different orders. + for expected in expected_heartbeats { + assert!(ss5_results.contains(&expected)); + assert!(ss6_results.contains(&expected)); + } + }); } #[test] fn do_not_send_empty_leaves_update_on_block_finalization() { - let spawner = sp_core::testing::TaskExecutor::new(); - - executor::block_on(async move { - let imported_block = BlockInfo { - hash: Hash::random(), - parent_hash: Hash::random(), - number: 1, - }; - - let finalized_block = BlockInfo { - hash: Hash::random(), - parent_hash: Hash::random(), - number: 1, - }; - - let (tx_5, mut rx_5) = metered::channel(64); - - let all_subsystems = AllSubsystems::<()>::dummy() - .replace_candidate_backing(TestSubsystem6(tx_5)); - - let (overseer, mut handler) = Overseer::new( - Vec::new(), - all_subsystems, - None, - MockSupportsParachains, - spawner, - ).unwrap(); - - let overseer_fut = overseer.run().fuse(); - pin_mut!(overseer_fut); - - let mut ss5_results = Vec::new(); - - handler.block_finalized(finalized_block.clone()).await; - handler.block_imported(imported_block.clone()).await; - - let expected_heartbeats = vec![ - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: [ - ActivatedLeaf { - hash: imported_block.hash, - number: imported_block.number, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - } - ].as_ref().into(), - ..Default::default() - }), - OverseerSignal::BlockFinalized(finalized_block.hash, 1), - ]; - - loop { - select! { - res = overseer_fut => { - assert!(res.is_ok()); - break; - }, - res = rx_5.next() => { - if let Some(res) = dbg!(res) { - ss5_results.push(res); - } - } - } - - if ss5_results.len() == expected_heartbeats.len() { - handler.stop().await; - } - } - - assert_eq!(ss5_results.len(), expected_heartbeats.len()); - - for expected in expected_heartbeats { - assert!(ss5_results.contains(&expected)); - } - }); + let spawner = sp_core::testing::TaskExecutor::new(); + + executor::block_on(async move { + let imported_block = BlockInfo { + hash: Hash::random(), + parent_hash: Hash::random(), + number: 1, + }; + + let finalized_block = BlockInfo { + hash: Hash::random(), + parent_hash: Hash::random(), + number: 1, + }; + + let (tx_5, mut rx_5) = metered::channel(64); + + let all_subsystems = AllSubsystems::<()>::dummy() + .replace_candidate_backing(TestSubsystem6(tx_5)); + + let (overseer, mut handler) = Overseer::new( + Vec::new(), + all_subsystems, + None, + MockSupportsParachains, + spawner, + ).unwrap(); + + let overseer_fut = overseer.run().fuse(); + pin_mut!(overseer_fut); + + let mut ss5_results = Vec::new(); + + handler.block_finalized(finalized_block.clone()).await; + handler.block_imported(imported_block.clone()).await; + + let expected_heartbeats = vec![ + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { + activated: [ + ActivatedLeaf { + hash: imported_block.hash, + number: imported_block.number, + span: Arc::new(jaeger::Span::Disabled), + status: LeafStatus::Fresh, + } + ].as_ref().into(), + ..Default::default() + }), + OverseerSignal::BlockFinalized(finalized_block.hash, 1), + ]; + + loop { + select! { + res = overseer_fut => { + assert!(res.is_ok()); + break; + }, + res = rx_5.next() => { + if let Some(res) = dbg!(res) { + ss5_results.push(res); + } + } + } + + if ss5_results.len() == expected_heartbeats.len() { + handler.stop().await; + } + } + + assert_eq!(ss5_results.len(), expected_heartbeats.len()); + + for expected in expected_heartbeats { + assert!(ss5_results.contains(&expected)); + } + }); } #[derive(Clone)] struct CounterSubsystem { - stop_signals_received: Arc, - signals_received: Arc, - msgs_received: Arc, + stop_signals_received: Arc, + signals_received: Arc, + msgs_received: Arc, } impl CounterSubsystem { - fn new( - stop_signals_received: Arc, - signals_received: Arc, - msgs_received: Arc, - ) -> Self { - Self { - stop_signals_received, - signals_received, - msgs_received, - } - } + fn new( + stop_signals_received: Arc, + signals_received: Arc, + msgs_received: Arc, + ) -> Self { + Self { + stop_signals_received, + signals_received, + msgs_received, + } + } } impl Subsystem for CounterSubsystem where - C: overseer::SubsystemContext, - M: Send, + C: overseer::SubsystemContext, + M: Send, { - fn start(self, mut ctx: C) -> SpawnedSubsystem { - SpawnedSubsystem { - name: "counter-subsystem", - future: Box::pin(async move { - loop { - match ctx.try_recv().await { - Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => { - self.stop_signals_received.fetch_add(1, atomic::Ordering::SeqCst); - break; - }, - Ok(Some(FromOverseer::Signal(_))) => { - self.signals_received.fetch_add(1, atomic::Ordering::SeqCst); - continue; - }, - Ok(Some(FromOverseer::Communication { .. })) => { - self.msgs_received.fetch_add(1, atomic::Ordering::SeqCst); - continue; - }, - Err(_) => (), - _ => (), - } - pending!(); - } - - Ok(()) - }), - } - } + fn start(self, mut ctx: C) -> SpawnedSubsystem { + SpawnedSubsystem { + name: "counter-subsystem", + future: Box::pin(async move { + loop { + match ctx.try_recv().await { + Ok(Some(FromOverseer::Signal(OverseerSignal::Conclude))) => { + self.stop_signals_received.fetch_add(1, atomic::Ordering::SeqCst); + break; + }, + Ok(Some(FromOverseer::Signal(_))) => { + self.signals_received.fetch_add(1, atomic::Ordering::SeqCst); + continue; + }, + Ok(Some(FromOverseer::Communication { .. })) => { + self.msgs_received.fetch_add(1, atomic::Ordering::SeqCst); + continue; + }, + Err(_) => (), + _ => (), + } + pending!(); + } + + Ok(()) + }), + } + } } fn test_candidate_validation_msg() -> CandidateValidationMessage { - let (sender, _) = oneshot::channel(); - let pov = Arc::new(PoV { block_data: BlockData(Vec::new()) }); - CandidateValidationMessage::ValidateFromChainState(Default::default(), pov, sender) + let (sender, _) = oneshot::channel(); + let pov = Arc::new(PoV { block_data: BlockData(Vec::new()) }); + CandidateValidationMessage::ValidateFromChainState(Default::default(), pov, sender) } fn test_candidate_backing_msg() -> CandidateBackingMessage { - let (sender, _) = oneshot::channel(); - CandidateBackingMessage::GetBackedCandidates(Default::default(), Vec::new(), sender) + let (sender, _) = oneshot::channel(); + CandidateBackingMessage::GetBackedCandidates(Default::default(), Vec::new(), sender) } fn test_chain_api_msg() -> ChainApiMessage { - let (sender, _) = oneshot::channel(); - ChainApiMessage::FinalizedBlockNumber(sender) + let (sender, _) = oneshot::channel(); + ChainApiMessage::FinalizedBlockNumber(sender) } fn test_collator_generation_msg() -> CollationGenerationMessage { - CollationGenerationMessage::Initialize(CollationGenerationConfig { - key: CollatorPair::generate().0, - collator: Box::new(|_, _| TestCollator.boxed()), - para_id: Default::default(), - }) + CollationGenerationMessage::Initialize(CollationGenerationConfig { + key: CollatorPair::generate().0, + collator: Box::new(|_, _| TestCollator.boxed()), + para_id: Default::default(), + }) } struct TestCollator; impl Future for TestCollator { - type Output = Option; + type Output = Option; - fn poll(self: Pin<&mut Self>, _cx: &mut futures::task::Context) -> Poll { - panic!("at the Disco") - } + fn poll(self: Pin<&mut Self>, _cx: &mut futures::task::Context) -> Poll { + panic!("at the Disco") + } } impl Unpin for TestCollator {} fn test_collator_protocol_msg() -> CollatorProtocolMessage { - CollatorProtocolMessage::CollateOn(Default::default()) + CollatorProtocolMessage::CollateOn(Default::default()) } fn test_network_bridge_event() -> NetworkBridgeEvent { - NetworkBridgeEvent::PeerDisconnected(PeerId::random()) + NetworkBridgeEvent::PeerDisconnected(PeerId::random()) } fn test_statement_distribution_msg() -> StatementDistributionMessage { - StatementDistributionMessage::NetworkBridgeUpdateV1(test_network_bridge_event()) + StatementDistributionMessage::NetworkBridgeUpdateV1(test_network_bridge_event()) } fn test_availability_recovery_msg() -> AvailabilityRecoveryMessage { - let (sender, _) = oneshot::channel(); - AvailabilityRecoveryMessage::RecoverAvailableData( - Default::default(), - Default::default(), - None, - sender, - ) + let (sender, _) = oneshot::channel(); + AvailabilityRecoveryMessage::RecoverAvailableData( + Default::default(), + Default::default(), + None, + sender, + ) } fn test_bitfield_distribution_msg() -> BitfieldDistributionMessage { - BitfieldDistributionMessage::NetworkBridgeUpdateV1(test_network_bridge_event()) + BitfieldDistributionMessage::NetworkBridgeUpdateV1(test_network_bridge_event()) } fn test_provisioner_msg() -> ProvisionerMessage { - let (sender, _) = oneshot::channel(); - ProvisionerMessage::RequestInherentData(Default::default(), sender) + let (sender, _) = oneshot::channel(); + ProvisionerMessage::RequestInherentData(Default::default(), sender) } fn test_runtime_api_msg() -> RuntimeApiMessage { - let (sender, _) = oneshot::channel(); - RuntimeApiMessage::Request(Default::default(), RuntimeApiRequest::Validators(sender)) + let (sender, _) = oneshot::channel(); + RuntimeApiMessage::Request(Default::default(), RuntimeApiRequest::Validators(sender)) } fn test_availability_store_msg() -> AvailabilityStoreMessage { - let (sender, _) = oneshot::channel(); - AvailabilityStoreMessage::QueryAvailableData(CandidateHash(Default::default()), sender) + let (sender, _) = oneshot::channel(); + AvailabilityStoreMessage::QueryAvailableData(CandidateHash(Default::default()), sender) } fn test_network_bridge_msg() -> NetworkBridgeMessage { - NetworkBridgeMessage::ReportPeer(PeerId::random(), UnifiedReputationChange::BenefitMinor("")) + NetworkBridgeMessage::ReportPeer(PeerId::random(), UnifiedReputationChange::BenefitMinor("")) } fn test_approval_distribution_msg() -> ApprovalDistributionMessage { - ApprovalDistributionMessage::NewBlocks(Default::default()) + ApprovalDistributionMessage::NewBlocks(Default::default()) } fn test_approval_voting_msg() -> ApprovalVotingMessage { - let (sender, _) = oneshot::channel(); - ApprovalVotingMessage::ApprovedAncestor(Default::default(), 0, sender) + let (sender, _) = oneshot::channel(); + ApprovalVotingMessage::ApprovedAncestor(Default::default(), 0, sender) } // Checks that `stop`, `broadcast_signal` and `broadcast_message` are implemented correctly. @@ -916,121 +916,121 @@ fn overseer_all_subsystems_receive_signals_and_messages() { #[test] fn context_holds_onto_message_until_enough_signals_received() { - let (candidate_validation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (candidate_backing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (statement_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (availability_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (availability_recovery_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (bitfield_signing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (bitfield_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (provisioner_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (runtime_api_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (availability_store_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (network_bridge_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (chain_api_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (collator_protocol_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (collation_generation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (approval_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (approval_voting_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - let (gossip_support_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); - - let (candidate_validation_unbounded_tx, _) = metered::unbounded(); - let (candidate_backing_unbounded_tx, _) = metered::unbounded(); - let (statement_distribution_unbounded_tx, _) = metered::unbounded(); - let (availability_distribution_unbounded_tx, _) = metered::unbounded(); - let (availability_recovery_unbounded_tx, _) = metered::unbounded(); - let (bitfield_signing_unbounded_tx, _) = metered::unbounded(); - let (bitfield_distribution_unbounded_tx, _) = metered::unbounded(); - let (provisioner_unbounded_tx, _) = metered::unbounded(); - let (runtime_api_unbounded_tx, _) = metered::unbounded(); - let (availability_store_unbounded_tx, _) = metered::unbounded(); - let (network_bridge_unbounded_tx, _) = metered::unbounded(); - let (chain_api_unbounded_tx, _) = metered::unbounded(); - let (collator_protocol_unbounded_tx, _) = metered::unbounded(); - let (collation_generation_unbounded_tx, _) = metered::unbounded(); - let (approval_distribution_unbounded_tx, _) = metered::unbounded(); - let (approval_voting_unbounded_tx, _) = metered::unbounded(); - let (gossip_support_unbounded_tx, _) = metered::unbounded(); - - let channels_out = ChannelsOut { - candidate_validation: candidate_validation_bounded_tx.clone(), - candidate_backing: candidate_backing_bounded_tx.clone(), - statement_distribution: statement_distribution_bounded_tx.clone(), - availability_distribution: availability_distribution_bounded_tx.clone(), - availability_recovery: availability_recovery_bounded_tx.clone(), - bitfield_signing: bitfield_signing_bounded_tx.clone(), - bitfield_distribution: bitfield_distribution_bounded_tx.clone(), - provisioner: provisioner_bounded_tx.clone(), - runtime_api: runtime_api_bounded_tx.clone(), - availability_store: availability_store_bounded_tx.clone(), - network_bridge: network_bridge_bounded_tx.clone(), - chain_api: chain_api_bounded_tx.clone(), - collator_protocol: collator_protocol_bounded_tx.clone(), - collation_generation: collation_generation_bounded_tx.clone(), - approval_distribution: approval_distribution_bounded_tx.clone(), - approval_voting: approval_voting_bounded_tx.clone(), - gossip_support: gossip_support_bounded_tx.clone(), - - candidate_validation_unbounded: candidate_validation_unbounded_tx.clone(), - candidate_backing_unbounded: candidate_backing_unbounded_tx.clone(), - statement_distribution_unbounded: statement_distribution_unbounded_tx.clone(), - availability_distribution_unbounded: availability_distribution_unbounded_tx.clone(), - availability_recovery_unbounded: availability_recovery_unbounded_tx.clone(), - bitfield_signing_unbounded: bitfield_signing_unbounded_tx.clone(), - bitfield_distribution_unbounded: bitfield_distribution_unbounded_tx.clone(), - provisioner_unbounded: provisioner_unbounded_tx.clone(), - runtime_api_unbounded: runtime_api_unbounded_tx.clone(), - availability_store_unbounded: availability_store_unbounded_tx.clone(), - network_bridge_unbounded: network_bridge_unbounded_tx.clone(), - chain_api_unbounded: chain_api_unbounded_tx.clone(), - collator_protocol_unbounded: collator_protocol_unbounded_tx.clone(), - collation_generation_unbounded: collation_generation_unbounded_tx.clone(), - approval_distribution_unbounded: approval_distribution_unbounded_tx.clone(), - approval_voting_unbounded: approval_voting_unbounded_tx.clone(), - gossip_support_unbounded: gossip_support_unbounded_tx.clone(), - }; - - let (mut signal_tx, signal_rx) = metered::channel(CHANNEL_CAPACITY); - let (mut bounded_tx, bounded_rx) = metered::channel(CHANNEL_CAPACITY); - let (unbounded_tx, unbounded_rx) = metered::unbounded(); - let (to_overseer_tx, _to_overseer_rx) = metered::unbounded(); - - let mut ctx = OverseerSubsystemContext::new( - signal_rx, - stream::select(bounded_rx, unbounded_rx), - channels_out, - to_overseer_tx, - ); - - assert_eq!(ctx.signals_received.load(), 0); - - let test_fut = async move { - signal_tx.send(OverseerSignal::Conclude).await.unwrap(); - assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Signal(OverseerSignal::Conclude)); - - assert_eq!(ctx.signals_received.load(), 1); - bounded_tx.send(MessagePacket { - signals_received: 2, - message: (), - }).await.unwrap(); - unbounded_tx.unbounded_send(MessagePacket { - signals_received: 2, - message: (), - }).unwrap(); - - match poll!(ctx.recv()) { - Poll::Pending => {} - Poll::Ready(_) => panic!("ready too early"), - }; - - assert!(ctx.pending_incoming.is_some()); - - signal_tx.send(OverseerSignal::Conclude).await.unwrap(); - assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Signal(OverseerSignal::Conclude)); - assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Communication { msg: () }); - assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Communication { msg: () }); - assert!(ctx.pending_incoming.is_none()); - }; - - futures::executor::block_on(test_fut); + let (candidate_validation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (candidate_backing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (statement_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (availability_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (availability_recovery_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (bitfield_signing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (bitfield_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (provisioner_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (runtime_api_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (availability_store_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (network_bridge_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (chain_api_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (collator_protocol_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (collation_generation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (approval_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (approval_voting_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + let (gossip_support_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); + + let (candidate_validation_unbounded_tx, _) = metered::unbounded(); + let (candidate_backing_unbounded_tx, _) = metered::unbounded(); + let (statement_distribution_unbounded_tx, _) = metered::unbounded(); + let (availability_distribution_unbounded_tx, _) = metered::unbounded(); + let (availability_recovery_unbounded_tx, _) = metered::unbounded(); + let (bitfield_signing_unbounded_tx, _) = metered::unbounded(); + let (bitfield_distribution_unbounded_tx, _) = metered::unbounded(); + let (provisioner_unbounded_tx, _) = metered::unbounded(); + let (runtime_api_unbounded_tx, _) = metered::unbounded(); + let (availability_store_unbounded_tx, _) = metered::unbounded(); + let (network_bridge_unbounded_tx, _) = metered::unbounded(); + let (chain_api_unbounded_tx, _) = metered::unbounded(); + let (collator_protocol_unbounded_tx, _) = metered::unbounded(); + let (collation_generation_unbounded_tx, _) = metered::unbounded(); + let (approval_distribution_unbounded_tx, _) = metered::unbounded(); + let (approval_voting_unbounded_tx, _) = metered::unbounded(); + let (gossip_support_unbounded_tx, _) = metered::unbounded(); + + let channels_out = ChannelsOut { + candidate_validation: candidate_validation_bounded_tx.clone(), + candidate_backing: candidate_backing_bounded_tx.clone(), + statement_distribution: statement_distribution_bounded_tx.clone(), + availability_distribution: availability_distribution_bounded_tx.clone(), + availability_recovery: availability_recovery_bounded_tx.clone(), + bitfield_signing: bitfield_signing_bounded_tx.clone(), + bitfield_distribution: bitfield_distribution_bounded_tx.clone(), + provisioner: provisioner_bounded_tx.clone(), + runtime_api: runtime_api_bounded_tx.clone(), + availability_store: availability_store_bounded_tx.clone(), + network_bridge: network_bridge_bounded_tx.clone(), + chain_api: chain_api_bounded_tx.clone(), + collator_protocol: collator_protocol_bounded_tx.clone(), + collation_generation: collation_generation_bounded_tx.clone(), + approval_distribution: approval_distribution_bounded_tx.clone(), + approval_voting: approval_voting_bounded_tx.clone(), + gossip_support: gossip_support_bounded_tx.clone(), + + candidate_validation_unbounded: candidate_validation_unbounded_tx.clone(), + candidate_backing_unbounded: candidate_backing_unbounded_tx.clone(), + statement_distribution_unbounded: statement_distribution_unbounded_tx.clone(), + availability_distribution_unbounded: availability_distribution_unbounded_tx.clone(), + availability_recovery_unbounded: availability_recovery_unbounded_tx.clone(), + bitfield_signing_unbounded: bitfield_signing_unbounded_tx.clone(), + bitfield_distribution_unbounded: bitfield_distribution_unbounded_tx.clone(), + provisioner_unbounded: provisioner_unbounded_tx.clone(), + runtime_api_unbounded: runtime_api_unbounded_tx.clone(), + availability_store_unbounded: availability_store_unbounded_tx.clone(), + network_bridge_unbounded: network_bridge_unbounded_tx.clone(), + chain_api_unbounded: chain_api_unbounded_tx.clone(), + collator_protocol_unbounded: collator_protocol_unbounded_tx.clone(), + collation_generation_unbounded: collation_generation_unbounded_tx.clone(), + approval_distribution_unbounded: approval_distribution_unbounded_tx.clone(), + approval_voting_unbounded: approval_voting_unbounded_tx.clone(), + gossip_support_unbounded: gossip_support_unbounded_tx.clone(), + }; + + let (mut signal_tx, signal_rx) = metered::channel(CHANNEL_CAPACITY); + let (mut bounded_tx, bounded_rx) = metered::channel(CHANNEL_CAPACITY); + let (unbounded_tx, unbounded_rx) = metered::unbounded(); + let (to_overseer_tx, _to_overseer_rx) = metered::unbounded(); + + let mut ctx = OverseerSubsystemContext::new( + signal_rx, + stream::select(bounded_rx, unbounded_rx), + channels_out, + to_overseer_tx, + ); + + assert_eq!(ctx.signals_received.load(), 0); + + let test_fut = async move { + signal_tx.send(OverseerSignal::Conclude).await.unwrap(); + assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Signal(OverseerSignal::Conclude)); + + assert_eq!(ctx.signals_received.load(), 1); + bounded_tx.send(MessagePacket { + signals_received: 2, + message: (), + }).await.unwrap(); + unbounded_tx.unbounded_send(MessagePacket { + signals_received: 2, + message: (), + }).unwrap(); + + match poll!(ctx.recv()) { + Poll::Pending => {} + Poll::Ready(_) => panic!("ready too early"), + }; + + assert!(ctx.pending_incoming.is_some()); + + signal_tx.send(OverseerSignal::Conclude).await.unwrap(); + assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Signal(OverseerSignal::Conclude)); + assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Communication { msg: () }); + assert_matches!(ctx.recv().await.unwrap(), FromOverseer::Communication { msg: () }); + assert!(ctx.pending_incoming.is_none()); + }; + + futures::executor::block_on(test_fut); } From 4a73141024afe9b289774d397ad2fc4276a0c158 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 12:28:06 +0200 Subject: [PATCH 146/161] up they go --- node/core/bitfield-signing/src/lib.rs | 7 ++++--- node/core/chain-api/src/lib.rs | 6 +++--- node/network/bitfield-distribution/src/lib.rs | 7 +++---- node/network/bridge/src/lib.rs | 6 +++--- node/overseer/overseer-gen/proc-macro/src/lib.rs | 6 +++--- node/overseer/overseer-gen/src/lib.rs | 7 ++++--- node/overseer/src/lib.rs | 8 ++++---- node/subsystem-util/src/lib.rs | 6 +++--- 8 files changed, 27 insertions(+), 26 deletions(-) diff --git a/node/core/bitfield-signing/src/lib.rs b/node/core/bitfield-signing/src/lib.rs index 4462b2d378c7..8da711da649a 100644 --- a/node/core/bitfield-signing/src/lib.rs +++ b/node/core/bitfield-signing/src/lib.rs @@ -38,10 +38,14 @@ use polkadot_primitives::v1::{AvailabilityBitfield, CoreState, Hash, ValidatorIn use std::{pin::Pin, time::Duration, iter::FromIterator, sync::Arc}; use wasm_timer::{Delay, Instant}; +#[cfg(test)] +mod tests; + /// Delay between starting a bitfield signing job and its attempting to create a bitfield. const JOB_DELAY: Duration = Duration::from_millis(1500); const LOG_TARGET: &str = "parachain::bitfield-signing"; + /// Each `BitfieldSigningJob` prepares a signed bitfield for a single relay parent. pub struct BitfieldSigningJob; @@ -310,6 +314,3 @@ impl JobTrait for BitfieldSigningJob { /// BitfieldSigningSubsystem manages a number of bitfield signing jobs. pub type BitfieldSigningSubsystem = JobSubsystem; - -#[cfg(test)] -mod tests; diff --git a/node/core/chain-api/src/lib.rs b/node/core/chain-api/src/lib.rs index 856f7f823e71..264a27644d1f 100644 --- a/node/core/chain-api/src/lib.rs +++ b/node/core/chain-api/src/lib.rs @@ -46,6 +46,9 @@ use polkadot_subsystem::{ SubsystemContext, SubsystemError, SubsystemResult, }; +#[cfg(test)] +mod tests; + const LOG_TARGET: &str = "parachain::chain-api"; /// The Chain API Subsystem implementation. @@ -296,6 +299,3 @@ impl metrics::Metrics for Metrics { Ok(Metrics(Some(metrics))) } } - -#[cfg(test)] -mod tests; diff --git a/node/network/bitfield-distribution/src/lib.rs b/node/network/bitfield-distribution/src/lib.rs index d2a5ebe78b36..e339e6dc66f2 100644 --- a/node/network/bitfield-distribution/src/lib.rs +++ b/node/network/bitfield-distribution/src/lib.rs @@ -39,6 +39,9 @@ use polkadot_primitives::v1::{Hash, SignedAvailabilityBitfield, SigningContext, use polkadot_node_network_protocol::{v1 as protocol_v1, PeerId, View, UnifiedReputationChange as Rep, OurView}; use std::collections::{HashMap, HashSet}; +#[cfg(test)] +mod tests; + const COST_SIGNATURE_INVALID: Rep = Rep::CostMajor("Bitfield signature invalid"); const COST_VALIDATOR_INDEX_INVALID: Rep = Rep::CostMajor("Bitfield validator index invalid"); const COST_MISSING_PEER_SESSION_KEY: Rep = Rep::CostMinor("Missing peer session key"); @@ -836,7 +839,3 @@ impl metrics::Metrics for Metrics { Ok(Metrics(Some(metrics))) } } - - -#[cfg(test)] -mod tests; diff --git a/node/network/bridge/src/lib.rs b/node/network/bridge/src/lib.rs index 11574bca5d6f..101ec01466a5 100644 --- a/node/network/bridge/src/lib.rs +++ b/node/network/bridge/src/lib.rs @@ -72,6 +72,9 @@ use network::{Network, send_message, get_peer_id_by_authority_id}; mod multiplexer; pub use multiplexer::RequestMultiplexer; +#[cfg(test)] +mod tests; + /// The maximum amount of heads a peer is allowed to have in their view at any time. /// /// We use the same limit to compute the view sent to peers locally. @@ -1162,6 +1165,3 @@ async fn dispatch_collation_events_to_all( ctx.send_messages(events.into_iter().flat_map(messages_for)).await } - -#[cfg(test)] -mod tests; diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 653e7cb3b5fe..ef4a116b9346 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -38,6 +38,9 @@ use impl_overseer::*; use parse_attr::*; use parse_struct::*; +#[cfg(test)] +mod tests; + #[proc_macro_attribute] pub fn overlord(attr: proc_macro::TokenStream, item: proc_macro::TokenStream) -> proc_macro::TokenStream { let attr: TokenStream = attr.into(); @@ -86,6 +89,3 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< Ok(additive) } - -#[cfg(test)] -mod tests; diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 76a693e9f55a..5c9ee5c76a1f 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -103,6 +103,10 @@ pub use polkadot_node_network_protocol::WrongVariant; use std::fmt; + +#[cfg(test)] +mod tests; + /// A type of messages that are sent from [`Subsystem`] to [`Overseer`]. /// /// Used to launch jobs. @@ -510,6 +514,3 @@ impl Future for Timeout where F: Future { Poll::Pending } } - -#[cfg(test)] -mod tests; diff --git a/node/overseer/src/lib.rs b/node/overseer/src/lib.rs index 3ff7c7ca9ba3..65c9f8ea3eaf 100644 --- a/node/overseer/src/lib.rs +++ b/node/overseer/src/lib.rs @@ -137,6 +137,10 @@ pub use polkadot_overseer_gen as gen; /// in the LRU cache. Assumes a 6-second block time. const KNOWN_LEAVES_CACHE_SIZE: usize = 2 * 24 * 3600 / 6; +#[cfg(test)] +mod tests; + + /// Whether a header supports parachain consensus or not. pub trait HeadSupportsParachains { /// Return true if the given header supports parachain consensus. Otherwise, false. @@ -879,7 +883,3 @@ impl From> for AllMess From::::from(From::from(req)) } } - - -#[cfg(test)] -mod tests; diff --git a/node/subsystem-util/src/lib.rs b/node/subsystem-util/src/lib.rs index 98c93467a250..096957167b45 100644 --- a/node/subsystem-util/src/lib.rs +++ b/node/subsystem-util/src/lib.rs @@ -100,6 +100,9 @@ pub mod rolling_session_window; mod determine_new_blocks; mod error_handling; +#[cfg(test)] +mod tests; + /// Duration a job will wait after sending a stop signal before hard-aborting. pub const JOB_GRACEFUL_STOP_DURATION: Duration = Duration::from_secs(1); /// Capacity of channels to and from individual jobs @@ -783,6 +786,3 @@ where } } } - -#[cfg(test)] -mod tests; From 9962bc5ca9fa874edeaa46c335af6125a1a0f255 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 12:31:07 +0200 Subject: [PATCH 147/161] Handler -> Handle --- node/core/parachains-inherent/src/lib.rs | 4 ++-- node/malus/src/variant-a.rs | 4 ++-- .../overseer-gen/proc-macro/src/impl_builder.rs | 12 ++++++------ node/overseer/src/lib.rs | 16 ++++++++-------- node/service/src/grandpa_support.rs | 6 +++--- node/service/src/lib.rs | 4 ++-- node/service/src/overseer.rs | 6 +++--- node/service/src/relay_chain_selection.rs | 8 ++++---- node/test/service/src/lib.rs | 4 ++-- 9 files changed, 32 insertions(+), 32 deletions(-) diff --git a/node/core/parachains-inherent/src/lib.rs b/node/core/parachains-inherent/src/lib.rs index 3e433a1eabec..0ba09a86ace9 100644 --- a/node/core/parachains-inherent/src/lib.rs +++ b/node/core/parachains-inherent/src/lib.rs @@ -26,7 +26,7 @@ use futures::{select, FutureExt}; use polkadot_node_subsystem::{ - overseer::Handler, + overseer::Handle, messages::ProvisionerMessage, errors::SubsystemError, }; use polkadot_primitives::v1::{ @@ -48,7 +48,7 @@ impl ParachainsInherentDataProvider { /// Create a new instance of the [`ParachainsInherentDataProvider`]. pub async fn create>( client: &C, - mut overseer: Handler, + mut overseer: Handle, parent: Hash, ) -> Result { let pid = async { diff --git a/node/malus/src/variant-a.rs b/node/malus/src/variant-a.rs index 7406a3cb509b..6b89f64071ec 100644 --- a/node/malus/src/variant-a.rs +++ b/node/malus/src/variant-a.rs @@ -27,7 +27,7 @@ use polkadot_cli::{ create_default_subsystems, service::{ AuthorityDiscoveryApi, AuxStore, BabeApi, Block, Error, HeaderBackend, Overseer, - OverseerGen, OverseerGenArgs, Handler, ParachainHost, ProvideRuntimeApi, + OverseerGen, OverseerGenArgs, Handle, ParachainHost, ProvideRuntimeApi, SpawnNamed, }, Cli, @@ -73,7 +73,7 @@ impl OverseerGen for BehaveMaleficient { fn generate<'a, Spawner, RuntimeClient>( &self, args: OverseerGenArgs<'a, Spawner, RuntimeClient>, - ) -> Result<(Overseer>, Handler), Error> + ) -> Result<(Overseer>, Handle), Error> where RuntimeClient: 'static + ProvideRuntimeApi + HeaderBackend + AuxStore, RuntimeClient::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index 524fdcb18777..f1d575db0d21 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -26,7 +26,7 @@ use super::*; pub(crate) fn impl_builder(info: &OverseerInfo) -> Result { let overseer_name = info.overseer_name.clone(); let builder = Ident::new(&(overseer_name.to_string() + "Builder"), overseer_name.span()); - let handler = Ident::new(&(overseer_name.to_string() + "Handler"), overseer_name.span()); + let handle = Ident::new(&(overseer_name.to_string() + "Handle"), overseer_name.span()); let subsystem_name = &info.subsystem_names_without_wip(); let builder_generic_ty = &info.builder_generic_types(); @@ -105,8 +105,8 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result; + /// Handle for an overseer. + pub type #handle = #support_crate ::metered::MeteredSender< #event >; #[allow(missing_docs)] pub struct #builder #builder_generics { @@ -168,13 +168,13 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result ::std::result::Result<(#overseer_name #generics, #handler), #error_ty> + pub fn build(mut self) -> ::std::result::Result<(#overseer_name #generics, #handle), #error_ty> { let (events_tx, events_rx) = #support_crate ::metered::channel::< #event >(SIGNAL_CHANNEL_CAPACITY); - let handler: #handler = events_tx.clone(); + let handle: #handle = events_tx.clone(); let (to_overseer_tx, to_overseer_rx) = #support_crate ::metered::unbounded::< ToOverseer @@ -269,7 +269,7 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result HeadSupportsParachains for Arc where /// /// [`Overseer`]: struct.Overseer.html #[derive(Clone)] -pub struct Handler(pub OverseerHandler); +pub struct Handle(pub OverseerHandle); -impl Handler { +impl Handle { /// Inform the `Overseer` that that some block was imported. pub async fn block_imported(&mut self, block: BlockInfo) { self.send_and_log_error(Event::BlockImported(block)).await @@ -223,7 +223,7 @@ impl Handler { /// Using this handler, connect another handler to the same /// overseer, if any. - pub fn connect_other(&self, other: &mut Handler) { + pub fn connect_other(&self, other: &mut Handle) { *other = self.clone(); } } @@ -297,10 +297,10 @@ pub enum ExternalRequest { } /// Glues together the [`Overseer`] and `BlockchainEvents` by forwarding -/// import and finality notifications into the [`OverseerHandler`]. +/// import and finality notifications into the [`OverseerHandle`]. pub async fn forward_events>( client: Arc

, - mut handler: Handler, + mut handler: Handle, ) { let mut finality = client.finality_notification_stream(); let mut imports = client.import_notification_stream(); @@ -429,7 +429,7 @@ where { /// Create a new instance of the [`Overseer`] with a fixed set of [`Subsystem`]s. /// - /// This returns the overseer along with an [`OverseerHandler`] which can + /// This returns the overseer along with an [`OverseerHandle`] which can /// be used to send messages from external parts of the codebase. /// /// The [`OverseerHandler`] returned from this function is connected to @@ -551,7 +551,7 @@ where prometheus_registry: Option<&prometheus::Registry>, supports_parachains: SupportsParachains, s: S, - ) -> SubsystemResult<(Self, Handler)> + ) -> SubsystemResult<(Self, Handle)> where CV: Subsystem, SubsystemError> + Send, CB: Subsystem, SubsystemError> + Send, @@ -643,7 +643,7 @@ where overseer.spawner().spawn("metrics_metronome", Box::pin(metronome)); } - Ok((overseer, Handler(handler))) + Ok((overseer, Handle(handler))) } /// Stop the overseer. diff --git a/node/service/src/grandpa_support.rs b/node/service/src/grandpa_support.rs index 7974d1c537a4..7736ce3a18cf 100644 --- a/node/service/src/grandpa_support.rs +++ b/node/service/src/grandpa_support.rs @@ -27,7 +27,7 @@ use { polkadot_primitives::v1::{Hash, Block as PolkadotBlock, Header as PolkadotHeader}, polkadot_subsystem::messages::ApprovalVotingMessage, prometheus_endpoint::{self, Registry}, - polkadot_overseer::Handler, + polkadot_overseer::Handle, futures::channel::oneshot, }; @@ -41,13 +41,13 @@ use { #[derive(Clone)] pub(crate) struct ApprovalCheckingVotingRule { checking_lag: Option>, - overseer: Handler, + overseer: Handle, } #[cfg(feature = "full-node")] impl ApprovalCheckingVotingRule { /// Create a new approval checking diagnostic voting rule. - pub fn new(overseer: Handler, registry: Option<&Registry>) + pub fn new(overseer: Handle, registry: Option<&Registry>) -> Result { Ok(ApprovalCheckingVotingRule { diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 6d5a1f95f6bb..50c2af707198 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -56,7 +56,7 @@ pub use { sp_authority_discovery::AuthorityDiscoveryApi, sc_client_api::AuxStore, polkadot_primitives::v1::ParachainHost, - polkadot_overseer::{Overseer, Handler}, + polkadot_overseer::{Overseer, Handle}, }; pub use sp_core::traits::SpawnNamed; @@ -427,7 +427,7 @@ fn new_partial( pub struct NewFull { pub task_manager: TaskManager, pub client: C, - pub overseer_handler: Option, + pub overseer_handler: Option, pub network: Arc::Hash>>, pub rpc_handlers: RpcHandlers, pub backend: Arc, diff --git a/node/service/src/overseer.rs b/node/service/src/overseer.rs index be9c9d5dab0e..9c2081538f50 100644 --- a/node/service/src/overseer.rs +++ b/node/service/src/overseer.rs @@ -28,7 +28,7 @@ use polkadot_network_bridge::RequestMultiplexer; use polkadot_node_core_av_store::Config as AvailabilityConfig; use polkadot_node_core_approval_voting::Config as ApprovalVotingConfig; use polkadot_node_core_candidate_validation::Config as CandidateValidationConfig; -use polkadot_overseer::{AllSubsystems, BlockInfo, Overseer, Handler}; +use polkadot_overseer::{AllSubsystems, BlockInfo, Overseer, Handle}; use polkadot_primitives::v1::ParachainHost; use sc_authority_discovery::Service as AuthorityDiscoveryService; use sp_api::ProvideRuntimeApi; @@ -237,7 +237,7 @@ where /// would do. pub trait OverseerGen { /// Overwrite the full generation of the overseer, including the subsystems. - fn generate<'a, Spawner, RuntimeClient>(&self, args: OverseerGenArgs<'a, Spawner, RuntimeClient>) -> Result<(Overseer>, Handler), Error> + fn generate<'a, Spawner, RuntimeClient>(&self, args: OverseerGenArgs<'a, Spawner, RuntimeClient>) -> Result<(Overseer>, Handle), Error> where RuntimeClient: 'static + ProvideRuntimeApi + HeaderBackend + AuxStore, RuntimeClient::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, @@ -256,7 +256,7 @@ pub struct RealOverseerGen; impl OverseerGen for RealOverseerGen { fn generate<'a, Spawner, RuntimeClient>(&self, args : OverseerGenArgs<'a, Spawner, RuntimeClient> - ) -> Result<(Overseer>, Handler), Error> + ) -> Result<(Overseer>, Handle), Error> where RuntimeClient: 'static + ProvideRuntimeApi + HeaderBackend + AuxStore, RuntimeClient::Api: ParachainHost + BabeApi + AuthorityDiscoveryApi, diff --git a/node/service/src/relay_chain_selection.rs b/node/service/src/relay_chain_selection.rs index a070ac758c74..eecbc4b5eda1 100644 --- a/node/service/src/relay_chain_selection.rs +++ b/node/service/src/relay_chain_selection.rs @@ -41,7 +41,7 @@ use { }, polkadot_subsystem::messages::{ApprovalVotingMessage, ChainSelectionMessage}, polkadot_node_subsystem_util::metrics::{self, prometheus}, - polkadot_overseer::Handler, + polkadot_overseer::Handle, futures::channel::oneshot, consensus_common::{Error as ConsensusError, SelectChain}, sp_blockchain::HeaderBackend, @@ -111,7 +111,7 @@ impl Metrics { /// A chain-selection implementation which provides safety for relay chains. pub struct SelectRelayChain { backend: Arc, - overseer: Handler, + overseer: Handle, // A fallback to use in case the overseer is disconnected. // // This is used on relay chains which have not yet enabled @@ -126,7 +126,7 @@ impl SelectRelayChain /// Create a new [`SelectRelayChain`] wrapping the given chain backend /// and a handle to the overseer. #[allow(unused)] - pub fn new(backend: Arc, overseer: Handler, metrics: Metrics) -> Self { + pub fn new(backend: Arc, overseer: Handle, metrics: Metrics) -> Self { SelectRelayChain { fallback: sc_consensus::LongestChain::new(backend.clone()), backend, @@ -172,7 +172,7 @@ impl SelectRelayChain { #[allow(unused)] pub fn connect_overseer_handler( &mut self, - other_handler: &Handler, + other_handler: &Handle, ) { other_handler.connect_other(&mut self.overseer); } diff --git a/node/test/service/src/lib.rs b/node/test/service/src/lib.rs index f69091436a40..0c5b2e3738eb 100644 --- a/node/test/service/src/lib.rs +++ b/node/test/service/src/lib.rs @@ -22,7 +22,7 @@ pub mod chain_spec; pub use chain_spec::*; use futures::future::Future; -use polkadot_overseer::Handler; +use polkadot_overseer::Handle; use polkadot_primitives::v1::{ Id as ParaId, HeadData, ValidationCode, Balance, CollatorPair, }; @@ -290,7 +290,7 @@ pub struct PolkadotTestNode { /// Client's instance. pub client: Arc, /// The overseer handler. - pub overseer_handler: Handler, + pub overseer_handler: Handle, /// The `MultiaddrWithPeerId` to this node. This is useful if you want to pass it as "boot node" to other nodes. pub addr: MultiaddrWithPeerId, /// RPCHandlers to make RPC queries. From 21c9f76bd300b0be1cc385e7956b2aedd77218d0 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 13:03:14 +0200 Subject: [PATCH 148/161] use statements and results --- .../overseer-gen/proc-macro/src/impl_builder.rs | 12 ++++++------ .../overseer-gen/proc-macro/src/impl_dispatch.rs | 6 +++--- .../overseer-gen/proc-macro/src/impl_misc.rs | 6 +++--- .../overseer-gen/proc-macro/src/impl_overseer.rs | 9 ++++----- node/overseer/overseer-gen/proc-macro/src/lib.rs | 12 ++++++------ .../overseer-gen/proc-macro/src/parse_attr.rs | 10 ++-------- .../overseer-gen/proc-macro/src/parse_struct.rs | 16 +++++----------- 7 files changed, 29 insertions(+), 42 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs index f1d575db0d21..256288368b8e 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_builder.rs @@ -15,7 +15,7 @@ // along with Polkadot. If not, see . use quote::quote; -use syn::{Ident, Result}; +use syn::Ident; use super::*; @@ -23,7 +23,7 @@ use super::*; /// which acts as the gateway to constructing the overseer. /// /// Elements tagged with `wip` are not covered here. -pub(crate) fn impl_builder(info: &OverseerInfo) -> Result { +pub(crate) fn impl_builder(info: &OverseerInfo) -> proc_macro2::TokenStream { let overseer_name = info.overseer_name.clone(); let builder = Ident::new(&(overseer_name.to_string() + "Builder"), overseer_name.span()); let handle = Ident::new(&(overseer_name.to_string() + "Handle"), overseer_name.span()); @@ -273,11 +273,11 @@ pub(crate) fn impl_builder(info: &OverseerInfo) -> Result Result { +pub(crate) fn impl_task_kind(info: &OverseerInfo) -> proc_macro2::TokenStream { let signal = &info.extern_signal_ty; let error_ty = &info.extern_error_ty; let support_crate = info.support_crate_name(); @@ -369,5 +369,5 @@ pub(crate) fn impl_task_kind(info: &OverseerInfo) -> Result Result { +pub(crate) fn impl_dispatch(info: &OverseerInfo) -> TokenStream { let message_wrapper = &info.message_wrapper; let dispatchable_variant = info @@ -71,5 +71,5 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> Result { } }); } - Ok(ts) + ts } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index 8ab714f37a1f..1b46d2a2d385 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -15,13 +15,13 @@ // along with Polkadot. If not, see . use quote::quote; -use syn::{Ident, Result}; +use syn::Ident; use super::*; /// Implement a builder pattern for the `Overseer`-type, /// which acts as the gateway to constructing the overseer. -pub(crate) fn impl_misc(info: &OverseerInfo) -> Result { +pub(crate) fn impl_misc(info: &OverseerInfo) -> proc_macro2::TokenStream { let overseer_name = info.overseer_name.clone(); let subsystem_sender_name = Ident::new(&(overseer_name.to_string() + "SubsystemSender"), overseer_name.span()); let subsystem_ctx_name = Ident::new(&(overseer_name.to_string() + "SubsystemContext"), overseer_name.span()); @@ -242,5 +242,5 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> Result } }; - Ok(ts) + ts } diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs index ca27daec84bb..3c578fdf4a51 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_overseer.rs @@ -15,11 +15,10 @@ // along with Polkadot. If not, see . use quote::quote; -use syn::Result; use super::*; -pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result { +pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> proc_macro2::TokenStream { let message_wrapper = &info.message_wrapper.clone(); let overseer_name = info.overseer_name.clone(); let subsystem_name = &info.subsystem_names_without_wip(); @@ -178,10 +177,10 @@ pub(crate) fn impl_overseer_struct(info: &OverseerInfo) -> Result Result { +pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> proc_macro2::TokenStream { let signal = &info.extern_signal_ty; let error_ty = &info.extern_error_ty; let support_crate = info.support_crate_name(); @@ -262,5 +261,5 @@ pub(crate) fn impl_overseen_subsystem(info: &OverseerInfo) -> Result Result< outgoing_ty: args.outgoing_ty, }; - let mut additive = impl_overseer_struct(&info)?; - additive.extend(impl_builder(&info)?); + let mut additive = impl_overseer_struct(&info); + additive.extend(impl_builder(&info)); - additive.extend(impl_overseen_subsystem(&info)?); - additive.extend(impl_channels_out_struct(&info)?); - additive.extend(impl_misc(&info)?); + additive.extend(impl_overseen_subsystem(&info)); + additive.extend(impl_channels_out_struct(&info)); + additive.extend(impl_misc(&info)); additive.extend(impl_message_wrapper_enum(&info)?); - additive.extend(impl_dispatch(&info)?); + additive.extend(impl_dispatch(&info)); Ok(additive) } diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs index d7dc05c86d55..472349ef2707 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs @@ -16,16 +16,10 @@ use proc_macro2::Span; use std::collections::{hash_map::RandomState, HashMap}; -use syn::parse::Parse; -use syn::parse::ParseBuffer; +use syn::parse::{Parse, ParseBuffer}; use syn::punctuated::Punctuated; use syn::spanned::Spanned; -use syn::Error; -use syn::Ident; -use syn::LitInt; -use syn::Path; -use syn::Result; -use syn::Token; +use syn::{Error, Ident, LitInt, Path, Result, Token}; #[derive(Clone, Debug)] enum AttrItem { diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 38b47692eebd..9f4d6b50f0a0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -16,20 +16,14 @@ use proc_macro2::{Span, TokenStream}; use quote::quote; -use syn::Visibility; use std::collections::{hash_map::RandomState, HashSet}; -use syn::parse::Parse; -use syn::parse::ParseStream; +use syn::parse::{Parse, ParseStream}; use syn::punctuated::Punctuated; use syn::spanned::Spanned; -use syn::Attribute; -use syn::Field; -use syn::FieldsNamed; -use syn::Ident; -use syn::Token; -use syn::Type; -use syn::{AttrStyle, Path}; -use syn::{Error, GenericParam, ItemStruct, Result}; +use syn::{ + Attribute, Field, FieldsNamed, Ident, Token, Type, AttrStyle, Path, + Error, GenericParam, ItemStruct, Result, Visibility +}; /// A field of the struct annotated with /// `#[subsystem(no_dispatch, , A | B | C)]` From c82cfc469f7f9eddef8efcc8fbe0bfabfd10cbae Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 13:03:26 +0200 Subject: [PATCH 149/161] print the subsystem name on send fail --- .../overseer-gen/proc-macro/src/impl_channels_out.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 3ad4bc4287fb..6878bbe4728a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -68,7 +68,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result { self. #channel_name .send( #support_crate ::make_packet(signals_received, inner) - ).await + ).await.map_err(|_| stringify!( #channel_name )) } )* // subsystems that are wip @@ -79,10 +79,11 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Ok(()), }; - if res.is_err() { + if let Err(subsystem_name) = res { #support_crate ::tracing::debug!( target: LOG_TARGET, - "Failed to send a message to another subsystem", + "Failed to send a message to {} subsystem", + subsystem_name ); } } From 0270602a2d2923d4f1106dbef40ade14f6b329fe Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 13:04:01 +0200 Subject: [PATCH 150/161] avoid iter chain --- .../proc-macro/src/impl_dispatch.rs | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs index b44dfad36a2f..687d094be532 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_dispatch.rs @@ -44,28 +44,23 @@ pub(crate) fn impl_dispatch(info: &OverseerInfo) -> TokenStream { impl #message_wrapper { /// Generated dispatch iterator generator. pub fn dispatch_iter(extern_msg: #extern_network_ty) -> impl Iterator + Send { - None - .into_iter() - + ::std::array::IntoIter::new([ #( - .chain( - ::std::iter::once( - extern_msg - // focuses on a `NetworkBridgeEvent< protocol_v1::* >` - // TODO do not require this to be hardcoded, either externalize or ... - // https://github.com/paritytech/polkadot/issues/3427 - .focus() - .ok() - .map(|event| { - #message_wrapper :: #dispatchable_variant ( - // the inner type of the enum variant - #dispatchable_message :: from( event ) - ) - }) - ) - ) + extern_msg + // focuses on a `NetworkBridgeEvent< protocol_v1::* >` + // TODO do not require this to be hardcoded, either externalize or ... + // https://github.com/paritytech/polkadot/issues/3427 + .focus() + .ok() + .map(|event| { + #message_wrapper :: #dispatchable_variant ( + // the inner type of the enum variant + #dispatchable_message :: from( event ) + ) + }), )* - + ]) + .into_iter() .filter_map(|x: Option<_>| x) } } From 1a036bf0bde613b7ce2145b0445b9a32ea5aa509 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 13:04:15 +0200 Subject: [PATCH 151/161] give unreachables a msg --- node/overseer/overseer-gen/proc-macro/src/parse_attr.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs index 472349ef2707..5a648650d516 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs @@ -141,7 +141,7 @@ impl Parse for AttrArgs { if let AttrItem::SignalChannelCapacity(lit) = item { lit.base10_parse::()? } else { - unreachable!() + unreachable!("Parsing guarantees the correct AttrItem::SignalChannelCapacity variant. qed") } } else { 64_usize @@ -151,7 +151,7 @@ impl Parse for AttrArgs { if let AttrItem::MessageChannelCapacity(lit) = item { lit.base10_parse::()? } else { - unreachable!() + unreachable!("Parsing guarantees the correct AttrItem::MessageChannelCapacity variant. qed") } } else { 1024_usize From 4fa3ff2d6edcef246fa969c763864fb47f6ae825 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 13:04:25 +0200 Subject: [PATCH 152/161] simplify --- node/overseer/overseer-gen/proc-macro/src/parse_struct.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 9f4d6b50f0a0..785c25e17aa3 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -292,11 +292,7 @@ impl OverseerGuts { if variant.consumes.len() != 1 { return Err(Error::new(attr_tokens.span(), "Exactly one message can be consumed per subsystem.")); } - consumes_paths.extend(variant.consumes.into_iter()); - - if consumes_paths.is_empty() { - return Err(Error::new(span, "Subsystem must consume at least one message")); - } + consumes_paths.push(variant.consumes[0].clone()); let field_ty = try_type_to_path(ty, span)?; let generic = field_ty.get_ident().ok_or_else(|| Error::new(field_ty.span(), "Must be an identifier, not a path."))?.clone(); From 8ff6a6370b0d06beb9128f0d9b383afec7728e52 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 13:08:08 +0200 Subject: [PATCH 153/161] better msg for unbounded too --- .../overseer-gen/proc-macro/src/impl_channels_out.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs index 6878bbe4728a..f2d6e88b360b 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_channels_out.rs @@ -82,7 +82,7 @@ pub(crate) fn impl_channels_out_struct(info: &OverseerInfo) -> Result Result Result Ok(()) }; - if res.is_err() { + if let Err(subsystem_name) = res { #support_crate ::tracing::debug!( target: LOG_TARGET, - "Failed to send a message to another subsystem", + "Failed to send_unbounded a message to {} subsystem", + subsystem_name ); } } From 450ae2df2fb1e356f26fbc2cbccec6f701390809 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 13:08:53 +0200 Subject: [PATCH 154/161] Update node/overseer/overseer-gen/proc-macro/src/parse_struct.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Köcher --- node/overseer/overseer-gen/proc-macro/src/parse_struct.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 785c25e17aa3..5249dfaa42cf 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -312,7 +312,7 @@ impl OverseerGuts { blocking: variant.blocking, }); } else { - let field_ty: Path = try_type_to_path(ty, ident.span())?; + let field_ty = try_type_to_path(ty, ident.span())?; let generic: bool = if let Some(ident) = field_ty.get_ident() { baggage_generics.contains(ident) } else { false }; baggage.push(BaggageField { field_name: ident, generic, field_ty, vis }); From 538e159331229a5353f7f5f8494f5535656344a8 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 13:09:22 +0200 Subject: [PATCH 155/161] Update node/overseer/overseer-gen/proc-macro/src/parse_struct.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Köcher --- node/overseer/overseer-gen/proc-macro/src/parse_struct.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 5249dfaa42cf..2f4a522e0324 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -313,8 +313,7 @@ impl OverseerGuts { }); } else { let field_ty = try_type_to_path(ty, ident.span())?; - let generic: bool = - if let Some(ident) = field_ty.get_ident() { baggage_generics.contains(ident) } else { false }; + let generic = field_ty.get_ident().map(|i| baggage_generics.contains(ident)).unwrap_or_default(); baggage.push(BaggageField { field_name: ident, generic, field_ty, vis }); } } From a06634cbbdd1cd865f15d6bd616057b34ec435e7 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 15:11:39 +0200 Subject: [PATCH 156/161] rework parse attr --- .../overseer-gen/proc-macro/src/parse_attr.rs | 297 +++++++++--------- 1 file changed, 157 insertions(+), 140 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs index 5a648650d516..29b97bc7ce09 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs @@ -20,90 +20,134 @@ use syn::parse::{Parse, ParseBuffer}; use syn::punctuated::Punctuated; use syn::spanned::Spanned; use syn::{Error, Ident, LitInt, Path, Result, Token}; +use quote::{quote, ToTokens}; + +mod kw { + syn::custom_keyword!(event); + syn::custom_keyword!(signal); + syn::custom_keyword!(error); + syn::custom_keyword!(network); + syn::custom_keyword!(outgoing); + syn::custom_keyword!(gen); + syn::custom_keyword!(signal_capacity); + syn::custom_keyword!(message_capacity); +} + #[derive(Clone, Debug)] enum AttrItem { - ExternEventType(Path), - ExternNetworkType(Path), - ExternOverseerSignalType(Path), - ExternErrorType(Path), - OutgoingType(Path), - MessageWrapperName(Ident), - SignalChannelCapacity(LitInt), - MessageChannelCapacity(LitInt), -} - -impl Spanned for AttrItem { - fn span(&self) -> Span { - match self { - AttrItem::ExternEventType(x) => x.span(), - AttrItem::ExternNetworkType(x) => x.span(), - AttrItem::ExternOverseerSignalType(x) => x.span(), - AttrItem::ExternErrorType(x) => x.span(), - AttrItem::OutgoingType(x) => x.span(), - AttrItem::MessageWrapperName(x) => x.span(), - AttrItem::SignalChannelCapacity(x) => x.span(), - AttrItem::MessageChannelCapacity(x) => x.span(), - } - } + ExternEventType { + tag: kw::event, + eq_token: Token![=], + value: Path + }, + ExternNetworkType { + tag: kw::network, + eq_token: Token![=], + value: Path + }, + ExternOverseerSignalType { + tag: kw::signal, + eq_token: Token![=], + value: Path + }, + ExternErrorType { + tag: kw::error, + eq_token: Token![=], + value: Path + }, + OutgoingType { + tag: kw::outgoing, + eq_token: Token![=], + value: Path + }, + MessageWrapperName { + tag: kw::gen, + eq_token: Token![=], + value: Ident + }, + SignalChannelCapacity { + tag: kw::signal_capacity, + eq_token: Token![=], + value: usize + }, + MessageChannelCapacity { + tag: kw::message_capacity, + eq_token: Token![=], + value: usize + }, } -const TAG_EXT_EVENT_TY: &str = "event"; -const TAG_EXT_SIGNAL_TY: &str = "signal"; -const TAG_EXT_ERROR_TY: &str = "error"; -const TAG_EXT_NETWORK_TY: &str = "network"; -const TAG_OUTGOING_TY: &str = "outgoing"; -const TAG_GEN_TY: &str = "gen"; -const TAG_SIGNAL_CAPACITY: &str = "signal_capacity"; -const TAG_MESSAGE_CAPACITY: &str = "message_capacity"; - -impl AttrItem { - fn key(&self) -> &'static str { - match self { - AttrItem::ExternEventType(_) => TAG_EXT_EVENT_TY, - AttrItem::ExternOverseerSignalType(_) => TAG_EXT_SIGNAL_TY, - AttrItem::ExternErrorType(_) => TAG_EXT_ERROR_TY, - AttrItem::ExternNetworkType(_) => TAG_EXT_NETWORK_TY, - AttrItem::OutgoingType(_) => TAG_OUTGOING_TY, - AttrItem::MessageWrapperName(_) => TAG_GEN_TY, - AttrItem::SignalChannelCapacity(_) => TAG_SIGNAL_CAPACITY, - AttrItem::MessageChannelCapacity(_) => TAG_MESSAGE_CAPACITY, - } +impl ToTokens for AttrItem { + fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { + let ts = match self { + Self::ExternEventType { tag, eq_token, value } => { quote!{ #tag #eq_token, #value } } + Self::ExternNetworkType { tag, eq_token, value } => { quote!{ #tag #eq_token, #value } } + Self::ExternOverseerSignalType { tag, eq_token, value } => { quote!{ #tag #eq_token, #value } } + Self::ExternErrorType { tag, eq_token, value } => { quote!{ #tag #eq_token, #value } } + Self::OutgoingType { tag, eq_token, value } => { quote!{ #tag #eq_token, #value } } + Self::MessageWrapperName { tag, eq_token, value } => { quote!{ #tag #eq_token, #value } } + Self::SignalChannelCapacity { tag, eq_token, value } => { quote!{ #tag #eq_token, #value } } + Self::MessageChannelCapacity { tag, eq_token, value } => { quote!{ #tag #eq_token, #value } } + }; + tokens.extend(ts.into_iter()); } } impl Parse for AttrItem { fn parse(input: &ParseBuffer) -> Result { - let key = input.parse::()?; - let span = key.span(); - let _ = input.parse::()?; - Ok(if key == TAG_EXT_SIGNAL_TY { - let path = input.parse::()?; - AttrItem::ExternOverseerSignalType(path) - } else if key == TAG_EXT_NETWORK_TY { - let path = input.parse::()?; - AttrItem::ExternNetworkType(path) - } else if key == TAG_OUTGOING_TY { - let path = input.parse::()?; - AttrItem::OutgoingType(path) - } else if key == TAG_EXT_EVENT_TY { - let path = input.parse::()?; - AttrItem::ExternEventType(path) - } else if key == TAG_EXT_ERROR_TY { - let path = input.parse::()?; - AttrItem::ExternErrorType(path) - } else if key == TAG_GEN_TY { - let wrapper_message = input.parse::()?; - AttrItem::MessageWrapperName(wrapper_message) - } else if key == TAG_SIGNAL_CAPACITY { - let value = input.parse::()?; - AttrItem::SignalChannelCapacity(value) - } else if key == TAG_MESSAGE_CAPACITY { - let value = input.parse::()?; - AttrItem::MessageChannelCapacity(value) + let lookahead = input.lookahead1(); + if lookahead.peek(kw::event) { + Ok(AttrItem::ExternEventType { + tag: input.parse::()?, + eq_token: input.parse()?, + value: input.parse()?, + }) + } else if lookahead.peek(kw::signal) { + Ok(AttrItem::ExternOverseerSignalType { + tag: input.parse::()?, + eq_token: input.parse()?, + value: input.parse()?, + }) + } else if lookahead.peek(kw::error) { + Ok(AttrItem::ExternErrorType { + tag: input.parse::()?, + eq_token: input.parse()?, + value: input.parse()?, + }) + } else if lookahead.peek(kw::network) { + Ok(AttrItem::ExternNetworkType { + tag: input.parse::()?, + eq_token: input.parse()?, + value: input.parse()?, + }) + } else if lookahead.peek(kw::outgoing) { + Ok(AttrItem::OutgoingType { + tag: input.parse::()?, + eq_token: input.parse()?, + value: input.parse()?, + }) + } else if lookahead.peek(kw::gen) { + Ok(AttrItem::MessageWrapperName { + tag: input.parse::()?, + eq_token: input.parse()?, + value: input.parse()?, + }) + } else if lookahead.peek(kw::signal_capacity) { + Ok(AttrItem::SignalChannelCapacity { + tag: input.parse::()?, + eq_token: input.parse()?, + value: input.parse::()?.base10_parse::()? + }) + } else if lookahead.peek(kw::message_capacity) { + Ok(AttrItem::MessageChannelCapacity { + tag: input.parse::()?, + eq_token: input.parse()?, + value: input.parse::()?.base10_parse::()?, + }) } else { - return Err(Error::new(span, "Expected one of `gen`, `signal_capacity`, or `message_capacity`.")); - }) + Err(lookahead.error()) + } } } @@ -123,88 +167,61 @@ pub(crate) struct AttrArgs { pub(crate) message_channel_capacity: usize, } +macro_rules! extract_variant { + ($unique:expr, $variant:ident ; default = $fallback:expr) => { + extract_variant!($unique, $variant) + .unwrap_or_else(|| { $fallback }) + }; + ($unique:expr, $variant:ident ; err = $err:expr) => { + extract_variant!($unique, $variant) + .ok_or_else(|| { + Error::new(Span::call_site(), $err) + }) + }; + ($unique:expr, $variant:ident) => { + $unique.values() + .find_map(|item| { + if let AttrItem:: $variant { value, ..} = item { + Some(value.clone()) + } else { + None + } + }) + }; +} + impl Parse for AttrArgs { fn parse(input: &ParseBuffer) -> Result { - let span = input.span(); let items: Punctuated = input.parse_terminated(AttrItem::parse)?; - let mut unique = HashMap::<&str, AttrItem, RandomState>::default(); + let mut unique = HashMap::, AttrItem, RandomState>::default(); for item in items { - if let Some(first) = unique.insert(item.key(), item.clone()) { - let mut e = Error::new(item.span(), format!("Duplicate definition of `{}` found", item.key())); + if let Some(first) = unique.insert(std::mem::discriminant(&item), item.clone()) { + let mut e = Error::new(item.span(), format!("Duplicate definition of subsystem attribute found")); e.combine(Error::new(first.span(), "previously defined here.")); return Err(e); } } - let signal_channel_capacity = if let Some(item) = unique.remove(TAG_SIGNAL_CAPACITY) { - if let AttrItem::SignalChannelCapacity(lit) = item { - lit.base10_parse::()? - } else { - unreachable!("Parsing guarantees the correct AttrItem::SignalChannelCapacity variant. qed") - } - } else { - 64_usize - }; - - let message_channel_capacity = if let Some(item) = unique.remove(TAG_MESSAGE_CAPACITY) { - if let AttrItem::MessageChannelCapacity(lit) = item { - lit.base10_parse::()? - } else { - unreachable!("Parsing guarantees the correct AttrItem::MessageChannelCapacity variant. qed") - } - } else { - 1024_usize - }; - let extern_error_ty = unique - .remove(TAG_EXT_ERROR_TY) - .map(|x| if let AttrItem::ExternErrorType(x) = x { x.clone() } else { unreachable!() }) - .ok_or_else(|| { - Error::new(span, format!("Must declare the overseer error type via `{}=..`.", TAG_EXT_ERROR_TY)) - })?; - - let extern_signal_ty = unique - .remove(TAG_EXT_SIGNAL_TY) - .map(|x| if let AttrItem::ExternOverseerSignalType(x) = x { x.clone() } else { unreachable!() }) - .ok_or_else(|| { - Error::new(span, format!("Must declare the overseer signals type via `{}=..`.", TAG_EXT_SIGNAL_TY)) - })?; + let signal_channel_capacity = extract_variant!(unique, SignalChannelCapacity; default = 64_usize); + let message_channel_capacity = extract_variant!(unique, MessageChannelCapacity; default = 1024_usize); - let extern_event_ty = unique - .remove(TAG_EXT_EVENT_TY) - .map(|x| if let AttrItem::ExternEventType(x) = x { x.clone() } else { unreachable!() }) - .ok_or_else(|| { - Error::new(span, format!("Must declare the external network type via `{}=..`.", TAG_EXT_NETWORK_TY)) - })?; - - let extern_network_ty = unique - .remove(TAG_EXT_NETWORK_TY) - .map(|x| if let AttrItem::ExternNetworkType(x) = x { x.clone() } else { unreachable!() }); - - let outgoing_ty = unique - .remove(TAG_OUTGOING_TY) - .map(|x| if let AttrItem::OutgoingType(x) = x { x.clone() } else { unreachable!() }); - - let message_wrapper = unique - .remove(TAG_GEN_TY) - .map(|x| if let AttrItem::MessageWrapperName(x) = x { x.clone() } else { unreachable!() }) - .ok_or_else(|| Error::new(span, format!("Must declare the generated type via `{}=..`.", TAG_GEN_TY)))?; - - if !unique.is_empty() { - let v = unique.into_iter().map(|(tag, _attr)| -> String { format!("`{}`", tag) }).collect::>(); - let s = v.join(", "); - - return Err(Error::new(span, format!("Found unknown arguments to the overseer macro {}.", s))); - } + dbg!(quote!{kw::error(span)}.to_string()); + let error = extract_variant!(unique, ExternErrorType; err = format!("Must declare the overseer error type via `{}=..`.", quote!{kw::error(span)}))?; + let event = extract_variant!(unique, ExternEventType; err = format!("Must declare the overseer event type via `{}=..`.", quote!{kw::event(span)}))?; + let signal = extract_variant!(unique, ExternOverseerSignalType; err = format!("Must declare the overseer signal type via `{}=..`.", quote!{kw::signal(span)}))?; + let message_wrapper = extract_variant!(unique, MessageWrapperName; err = format!("Must declare the overseer generated wrapping message type via `{}=..`.", quote!{kw::gen(span)}))?; + let network = extract_variant!(unique, ExternNetworkType); + let outgoing = extract_variant!(unique, OutgoingType); Ok(AttrArgs { signal_channel_capacity, message_channel_capacity, - extern_event_ty, - extern_signal_ty, - extern_error_ty, - extern_network_ty, - outgoing_ty, + extern_event_ty: event, + extern_signal_ty: signal, + extern_error_ty: error, + extern_network_ty: network, + outgoing_ty: outgoing, message_wrapper, }) } From aa515e949fe581dc67491de141e1720e3f62ef99 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 17:02:09 +0200 Subject: [PATCH 157/161] more lookahead usage --- .../overseer-gen/proc-macro/src/impl_misc.rs | 6 +- .../overseer-gen/proc-macro/src/parse_attr.rs | 39 +++-- .../proc-macro/src/parse_struct.rs | 134 +++++++++++++----- node/overseer/overseer-gen/src/lib.rs | 4 +- .../tests/ui/err-04-missing-error.stderr | 6 +- 5 files changed, 130 insertions(+), 59 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index 1b46d2a2d385..71b6ab33acaf 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -228,7 +228,8 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> proc_macro2::TokenStream { self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnJob { name, s, - }).map_err(|_| SubsystemError::TaskSpawn(name)) + }).map_err(|_| OverseerError::TaskSpawn(name))?; + Ok(()) } fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) @@ -237,7 +238,8 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> proc_macro2::TokenStream { self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnBlockingJob { name, s, - }).map_err(|_| SubsystemError::TaskSpawn(name)) + }).map_err(|_| OverseerError::TaskSpawn(name))?; + Ok(()) } } }; diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs index 29b97bc7ce09..60fa6ab22765 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_attr.rs @@ -35,7 +35,7 @@ mod kw { #[derive(Clone, Debug)] -enum AttrItem { +enum OverseerAttrItem { ExternEventType { tag: kw::event, eq_token: Token![=], @@ -78,7 +78,7 @@ enum AttrItem { }, } -impl ToTokens for AttrItem { +impl ToTokens for OverseerAttrItem { fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { let ts = match self { Self::ExternEventType { tag, eq_token, value } => { quote!{ #tag #eq_token, #value } } @@ -94,53 +94,53 @@ impl ToTokens for AttrItem { } } -impl Parse for AttrItem { +impl Parse for OverseerAttrItem { fn parse(input: &ParseBuffer) -> Result { let lookahead = input.lookahead1(); if lookahead.peek(kw::event) { - Ok(AttrItem::ExternEventType { + Ok(OverseerAttrItem::ExternEventType { tag: input.parse::()?, eq_token: input.parse()?, value: input.parse()?, }) } else if lookahead.peek(kw::signal) { - Ok(AttrItem::ExternOverseerSignalType { + Ok(OverseerAttrItem::ExternOverseerSignalType { tag: input.parse::()?, eq_token: input.parse()?, value: input.parse()?, }) } else if lookahead.peek(kw::error) { - Ok(AttrItem::ExternErrorType { + Ok(OverseerAttrItem::ExternErrorType { tag: input.parse::()?, eq_token: input.parse()?, value: input.parse()?, }) } else if lookahead.peek(kw::network) { - Ok(AttrItem::ExternNetworkType { + Ok(OverseerAttrItem::ExternNetworkType { tag: input.parse::()?, eq_token: input.parse()?, value: input.parse()?, }) } else if lookahead.peek(kw::outgoing) { - Ok(AttrItem::OutgoingType { + Ok(OverseerAttrItem::OutgoingType { tag: input.parse::()?, eq_token: input.parse()?, value: input.parse()?, }) } else if lookahead.peek(kw::gen) { - Ok(AttrItem::MessageWrapperName { + Ok(OverseerAttrItem::MessageWrapperName { tag: input.parse::()?, eq_token: input.parse()?, value: input.parse()?, }) } else if lookahead.peek(kw::signal_capacity) { - Ok(AttrItem::SignalChannelCapacity { + Ok(OverseerAttrItem::SignalChannelCapacity { tag: input.parse::()?, eq_token: input.parse()?, value: input.parse::()?.base10_parse::()? }) } else if lookahead.peek(kw::message_capacity) { - Ok(AttrItem::MessageChannelCapacity { + Ok(OverseerAttrItem::MessageChannelCapacity { tag: input.parse::()?, eq_token: input.parse()?, value: input.parse::()?.base10_parse::()?, @@ -181,7 +181,7 @@ macro_rules! extract_variant { ($unique:expr, $variant:ident) => { $unique.values() .find_map(|item| { - if let AttrItem:: $variant { value, ..} = item { + if let OverseerAttrItem:: $variant { value, ..} = item { Some(value.clone()) } else { None @@ -192,12 +192,12 @@ macro_rules! extract_variant { impl Parse for AttrArgs { fn parse(input: &ParseBuffer) -> Result { - let items: Punctuated = input.parse_terminated(AttrItem::parse)?; + let items: Punctuated = input.parse_terminated(OverseerAttrItem::parse)?; - let mut unique = HashMap::, AttrItem, RandomState>::default(); + let mut unique = HashMap::, OverseerAttrItem, RandomState>::default(); for item in items { if let Some(first) = unique.insert(std::mem::discriminant(&item), item.clone()) { - let mut e = Error::new(item.span(), format!("Duplicate definition of subsystem attribute found")); + let mut e = Error::new(item.span(), format!("Duplicate definition of overseer generation type found")); e.combine(Error::new(first.span(), "previously defined here.")); return Err(e); } @@ -206,11 +206,10 @@ impl Parse for AttrArgs { let signal_channel_capacity = extract_variant!(unique, SignalChannelCapacity; default = 64_usize); let message_channel_capacity = extract_variant!(unique, MessageChannelCapacity; default = 1024_usize); - dbg!(quote!{kw::error(span)}.to_string()); - let error = extract_variant!(unique, ExternErrorType; err = format!("Must declare the overseer error type via `{}=..`.", quote!{kw::error(span)}))?; - let event = extract_variant!(unique, ExternEventType; err = format!("Must declare the overseer event type via `{}=..`.", quote!{kw::event(span)}))?; - let signal = extract_variant!(unique, ExternOverseerSignalType; err = format!("Must declare the overseer signal type via `{}=..`.", quote!{kw::signal(span)}))?; - let message_wrapper = extract_variant!(unique, MessageWrapperName; err = format!("Must declare the overseer generated wrapping message type via `{}=..`.", quote!{kw::gen(span)}))?; + let error = extract_variant!(unique, ExternErrorType; err = "Must declare the overseer error type via `error=..`.")?; + let event = extract_variant!(unique, ExternEventType; err = "Must declare the overseer event type via `event=..`.")?; + let signal = extract_variant!(unique, ExternOverseerSignalType; err = "Must declare the overseer signal type via `span=..`.")?; + let message_wrapper = extract_variant!(unique, MessageWrapperName; err = "Must declare the overseer generated wrapping message type via `gen=..`.")?; let network = extract_variant!(unique, ExternNetworkType); let outgoing = extract_variant!(unique, OutgoingType); diff --git a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs index 2f4a522e0324..2448cfb143b2 100644 --- a/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs +++ b/node/overseer/overseer-gen/proc-macro/src/parse_struct.rs @@ -15,16 +15,64 @@ // along with Polkadot. If not, see . use proc_macro2::{Span, TokenStream}; -use quote::quote; -use std::collections::{hash_map::RandomState, HashSet}; -use syn::parse::{Parse, ParseStream}; +use std::collections::{hash_map::RandomState, HashSet, HashMap}; use syn::punctuated::Punctuated; use syn::spanned::Spanned; +use syn::parse::{Parse, ParseStream}; use syn::{ Attribute, Field, FieldsNamed, Ident, Token, Type, AttrStyle, Path, Error, GenericParam, ItemStruct, Result, Visibility }; +use quote::{quote, ToTokens}; + +mod kw { + syn::custom_keyword!(wip); + syn::custom_keyword!(no_dispatch); + syn::custom_keyword!(blocking); +} + + +#[derive(Clone, Debug)] +enum SubSysAttrItem { + /// The subsystem is still a work in progress + /// and should not be communicated with. + Wip(kw::wip), + /// The subsystem is blocking and requires to be + /// spawned on an exclusive thread. + Blocking(kw::blocking), + /// External messages should not be - after being converted - + /// be dispatched to the annotated subsystem. + NoDispatch(kw::no_dispatch), +} + +impl Parse for SubSysAttrItem { + fn parse(input: ParseStream) -> Result { + let lookahead = input.lookahead1(); + Ok(if lookahead.peek(kw::wip) { + Self::Wip(input.parse::()?) + } else if lookahead.peek(kw::blocking) { + Self::Blocking(input.parse::()?) + } else if lookahead.peek(kw::no_dispatch) { + Self::NoDispatch(input.parse::()?) + } else { + return Err(lookahead.error()) + }) + } +} + +impl ToTokens for SubSysAttrItem { + fn to_tokens(&self, tokens: &mut proc_macro2::TokenStream) { + let ts = match self { + Self::Wip(wip) => { quote!{ #wip } } + Self::Blocking(blocking) => { quote!{ #blocking } } + Self::NoDispatch(no_dispatch) => { quote!{ #no_dispatch } } + }; + tokens.extend(ts.into_iter()); + } +} + + /// A field of the struct annotated with /// `#[subsystem(no_dispatch, , A | B | C)]` #[derive(Clone, Debug)] @@ -56,7 +104,31 @@ fn try_type_to_path(ty: Type, span: Span) -> Result { } } -pub(crate) struct SubSystemTag { +macro_rules! extract_variant { + ($unique:expr, $variant:ident ; default = $fallback:expr) => { + extract_variant!($unique, $variant) + .unwrap_or_else(|| { $fallback }) + }; + ($unique:expr, $variant:ident ; err = $err:expr) => { + extract_variant!($unique, $variant) + .ok_or_else(|| { + Error::new(Span::call_site(), $err) + }) + }; + ($unique:expr, $variant:ident) => { + $unique.values() + .find_map(|item| { + if let SubSysAttrItem:: $variant ( _ ) = item { + Some(true) + } else { + None + } + }) + }; +} + + +pub(crate) struct SubSystemTags { #[allow(dead_code)] pub(crate) attrs: Vec, #[allow(dead_code)] @@ -65,42 +137,41 @@ pub(crate) struct SubSystemTag { /// and also not include the subsystem in the list of subsystems. pub(crate) wip: bool, pub(crate) blocking: bool, - pub(crate) consumes: Punctuated, + pub(crate) consumes: Path, } -impl Parse for SubSystemTag { +impl Parse for SubSystemTags { fn parse(input: syn::parse::ParseStream) -> Result { let attrs = Attribute::parse_outer(input)?; let input = input; let content; let _ = syn::parenthesized!(content in input); - let parse_tags = || -> Result> { - if content.peek(Ident) && content.peek2(Token![,]) { - let ident = content.parse::()?; - let _ = content.parse::()?; - Ok(Some(ident)) - } else { - Ok(None) - } - }; - let mut unique = HashSet::<_, RandomState>::default(); - while let Some(ident) = parse_tags()? { - if ident != "no_dispatch" && ident != "blocking" && ident != "wip" { - return Err(Error::new(ident.span(), "Allowed tags are only `no_dispatch` or `blocking`.")); - } - if !unique.insert(ident.to_string()) { - return Err(Error::new(ident.span(), "Found duplicate tag.")); + let mut items = Punctuated::new(); + while let Ok(tag) = content.call(SubSysAttrItem::parse) { + items.push_value(tag); + items.push_punct(content.call(::parse)?); + } + + assert!(items.empty_or_trailing(), "Always followed by the message type to consume. qed"); + + let consumes = content.parse::()?; + + let mut unique = HashMap::, SubSysAttrItem, RandomState>::default(); + for item in items { + if let Some(first) = unique.insert(std::mem::discriminant(&item), item.clone()) { + let mut e = Error::new(item.span(), format!("Duplicate definition of subsystem attribute found")); + e.combine(Error::new(first.span(), "previously defined here.")); + return Err(e); } } - let no_dispatch = unique.take("no_dispatch").is_some(); - let blocking = unique.take("blocking").is_some(); - let wip = unique.take("wip").is_some(); - let consumes = content.parse_terminated(Path::parse)?; + let no_dispatch = extract_variant!(unique, NoDispatch; default = false); + let blocking = extract_variant!(unique, Blocking; default = false); + let wip = extract_variant!(unique, Wip; default = false); - Ok(Self { attrs, no_dispatch, blocking, consumes, wip}) + Ok(Self { attrs, no_dispatch, blocking, consumes, wip }) } } @@ -288,11 +359,8 @@ impl OverseerGuts { } let mut consumes_paths = Vec::with_capacity(attrs.len()); let attr_tokens = attr_tokens.clone(); - let variant: SubSystemTag = syn::parse2(attr_tokens.clone())?; - if variant.consumes.len() != 1 { - return Err(Error::new(attr_tokens.span(), "Exactly one message can be consumed per subsystem.")); - } - consumes_paths.push(variant.consumes[0].clone()); + let variant: SubSystemTags = syn::parse2(attr_tokens.clone())?; + consumes_paths.push(variant.consumes); let field_ty = try_type_to_path(ty, span)?; let generic = field_ty.get_ident().ok_or_else(|| Error::new(field_ty.span(), "Must be an identifier, not a path."))?.clone(); @@ -313,7 +381,7 @@ impl OverseerGuts { }); } else { let field_ty = try_type_to_path(ty, ident.span())?; - let generic = field_ty.get_ident().map(|i| baggage_generics.contains(ident)).unwrap_or_default(); + let generic = field_ty.get_ident().map(|ident| baggage_generics.contains(ident)).unwrap_or_default(); baggage.push(BaggageField { field_name: ident, generic, field_ty, vis }); } } diff --git a/node/overseer/overseer-gen/src/lib.rs b/node/overseer/overseer-gen/src/lib.rs index 5c9ee5c76a1f..4b6abdb2b20a 100644 --- a/node/overseer/overseer-gen/src/lib.rs +++ b/node/overseer/overseer-gen/src/lib.rs @@ -256,8 +256,8 @@ pub enum OverseerError { #[error(transparent)] QueueError(#[from] mpsc::SendError), - #[error(transparent)] - TaskSpawn(#[from] futures::task::SpawnError), + #[error("Failed to spawn task {0}")] + TaskSpawn(&'static str), #[error(transparent)] Infallible(#[from] std::convert::Infallible), diff --git a/node/overseer/overseer-gen/tests/ui/err-04-missing-error.stderr b/node/overseer/overseer-gen/tests/ui/err-04-missing-error.stderr index 92ddb2e26af0..39e6a4a8fdc0 100644 --- a/node/overseer/overseer-gen/tests/ui/err-04-missing-error.stderr +++ b/node/overseer/overseer-gen/tests/ui/err-04-missing-error.stderr @@ -1,8 +1,10 @@ error: Must declare the overseer error type via `error=..`. - --> $DIR/err-04-missing-error.rs:16:12 + --> $DIR/err-04-missing-error.rs:16:1 | 16 | #[overlord(signal=SigSigSig, event=Event, gen=AllMessages)] - | ^^^^^^ + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in an attribute macro (in Nightly builds, run with -Z macro-backtrace for more info) error[E0433]: failed to resolve: use of undeclared type `Overseer` --> $DIR/err-04-missing-error.rs:30:10 From 1fa00e60e85ea42e1eeaffb4171ec5fe3b7eb3c3 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 17:09:46 +0200 Subject: [PATCH 158/161] path --- node/overseer/overseer-gen/proc-macro/src/impl_misc.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs index 71b6ab33acaf..b3406b62f7eb 100644 --- a/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs +++ b/node/overseer/overseer-gen/proc-macro/src/impl_misc.rs @@ -228,7 +228,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> proc_macro2::TokenStream { self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnJob { name, s, - }).map_err(|_| OverseerError::TaskSpawn(name))?; + }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; Ok(()) } @@ -238,7 +238,7 @@ pub(crate) fn impl_misc(info: &OverseerInfo) -> proc_macro2::TokenStream { self.to_overseer.unbounded_send(#support_crate ::ToOverseer::SpawnBlockingJob { name, s, - }).map_err(|_| OverseerError::TaskSpawn(name))?; + }).map_err(|_| #support_crate ::OverseerError::TaskSpawn(name))?; Ok(()) } } From 1434404690be3a1d63da821b53cfabcba15512cd Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 17:16:42 +0200 Subject: [PATCH 159/161] slips --- node/malus/src/lib.rs | 8 ++++---- node/subsystem-test-helpers/src/lib.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/node/malus/src/lib.rs b/node/malus/src/lib.rs index 38982b8964eb..516d4840b3c4 100644 --- a/node/malus/src/lib.rs +++ b/node/malus/src/lib.rs @@ -138,20 +138,20 @@ where } } - async fn spawn( + fn spawn( &mut self, name: &'static str, s: Pin + Send>>, ) -> SubsystemResult<()> { - self.inner.spawn(name, s).await + self.inner.spawn(name, s) } - async fn spawn_blocking( + fn spawn_blocking( &mut self, name: &'static str, s: Pin + Send>>, ) -> SubsystemResult<()> { - self.inner.spawn_blocking(name, s).await + self.inner.spawn_blocking(name, s) } fn sender(&mut self) -> &mut Self::Sender { diff --git a/node/subsystem-test-helpers/src/lib.rs b/node/subsystem-test-helpers/src/lib.rs index 66bfa4cefa80..87b28d3963e4 100644 --- a/node/subsystem-test-helpers/src/lib.rs +++ b/node/subsystem-test-helpers/src/lib.rs @@ -232,7 +232,7 @@ where .ok_or_else(|| SubsystemError::Context("Receiving end closed".to_owned())) } - async fn spawn( + fn spawn( &mut self, name: &'static str, s: Pin + Send>>, @@ -241,7 +241,7 @@ where Ok(()) } - async fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) + fn spawn_blocking(&mut self, name: &'static str, s: Pin + Send>>) -> SubsystemResult<()> { self.spawn.spawn_blocking(name, s); From d176007867bb3747fa5fcb5b6c2376b0427ab5e3 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 18:43:11 +0200 Subject: [PATCH 160/161] minor tweaks, fix silly test --- node/overseer/overseer-gen/proc-macro/src/lib.rs | 7 +++++-- node/overseer/overseer-gen/proc-macro/src/tests.rs | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index 0bbdb33b5c14..a30eac7d1105 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -54,9 +54,12 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< let of: OverseerGuts = parse2(orig)?; - let support_crate_name = { + let support_crate_name = if cfg!(test) { + quote!{crate} + } else { use proc_macro_crate::{crate_name, FoundCrate}; - let crate_name = crate_name("polkadot-overseer-gen").expect("Support crate polkadot-overseer-gen is present in `Cargo.toml`. qed"); + let crate_name = crate_name("polkadot-overseer-gen") + .expect("Support crate polkadot-overseer-gen is present in `Cargo.toml`. qed"); match crate_name { FoundCrate::Itself => quote!{crate}, FoundCrate::Name(name) => Ident::new(&name, Span::call_site()).to_token_stream(), diff --git a/node/overseer/overseer-gen/proc-macro/src/tests.rs b/node/overseer/overseer-gen/proc-macro/src/tests.rs index 79c901518947..40df210fb6f0 100644 --- a/node/overseer/overseer-gen/proc-macro/src/tests.rs +++ b/node/overseer/overseer-gen/proc-macro/src/tests.rs @@ -22,7 +22,11 @@ use syn::parse_quote; #[test] fn print() { let attr = quote! { - gen=AllMessage, event=::some::why::ExternEvent, signal=SigSigSig, signal_capacity=111, message_capacity=222, + gen=AllMessage, + event=::some::why::ExternEvent, + signal=SigSigSig, + signal_capacity=111, + message_capacity=222, error=OverseerError, }; From a630a3eed854815dc3c79ffb391685efad8a1665 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 8 Jul 2021 18:43:50 +0200 Subject: [PATCH 161/161] one space --- node/overseer/overseer-gen/proc-macro/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/overseer/overseer-gen/proc-macro/src/lib.rs b/node/overseer/overseer-gen/proc-macro/src/lib.rs index a30eac7d1105..9fecfd19241a 100644 --- a/node/overseer/overseer-gen/proc-macro/src/lib.rs +++ b/node/overseer/overseer-gen/proc-macro/src/lib.rs @@ -54,7 +54,7 @@ pub(crate) fn impl_overseer_gen(attr: TokenStream, orig: TokenStream) -> Result< let of: OverseerGuts = parse2(orig)?; - let support_crate_name = if cfg!(test) { + let support_crate_name = if cfg!(test) { quote!{crate} } else { use proc_macro_crate::{crate_name, FoundCrate};