diff --git a/.github/workflows/check-weights.yml b/.github/workflows/check-weights.yml new file mode 100644 index 000000000000..e6a6c43e0a6a --- /dev/null +++ b/.github/workflows/check-weights.yml @@ -0,0 +1,49 @@ +name: Check updated weights + +on: + pull_request: + paths: +  - 'runtime/*/src/weights/**' + +jobs: + check_weights_files: + strategy: + fail-fast: false + matrix: + runtime: [westend, kusama, polkadot, rococo] + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v3 + - name: Check weights files + shell: bash + run: | + scripts/ci/github/verify_updated_weights.sh ${{ matrix.runtime }} + + # This job uses https://github.com/ggwpez/substrate-weight-compare to compare the weights of the current + # release with the last release, then adds them as a comment to the PR. + check_weight_changes: + strategy: + fail-fast: false + matrix: + runtime: [westend, kusama, polkadot, rococo] + runs-on: ubuntu-latest + steps: + - name: Get latest release + run: | + LAST_RELEASE=$(curl -s https://api.github.com/repos/paritytech/polkadot/releases/latest | jq -r .tag_name) + echo "LAST_RELEASE=$LAST_RELEASE" >> $GITHUB_ENV + - name: Checkout current sources + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Check weight changes + shell: bash + run: | + cargo install --git https://github.com/ggwpez/substrate-weight-compare swc + ./scripts/ci/github/check_weights_swc.sh ${{ matrix.runtime }} "$LAST_RELEASE" | tee swc_output_${{ matrix.runtime }}.md + - name: Add comment + uses: thollander/actions-comment-pull-request@v2 + with: + filePath: ./swc_output_${{ matrix.runtime }}.md + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 52470bc1e7ef..64b1b2f92733 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,7 @@ # moment of time. stages: + - .pre - weights - check - test @@ -185,6 +186,23 @@ default: after_script: - buildah logout --all +#### stage: .pre + +# By default our pipelines are interruptible, but some special pipelines shouldn't be interrupted: +# * multi-project pipelines such as the ones triggered by the scripts repo +# +# In those cases, we add an uninterruptible .pre job; once that one has started, +# the entire pipeline becomes uninterruptible. +uninterruptible-pipeline: + extends: .kubernetes-env + variables: + CI_IMAGE: "paritytech/tools:latest" + stage: .pre + interruptible: false + rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + script: "true" + include: # weights jobs - scripts/ci/gitlab/pipeline/weights.yml diff --git a/Cargo.lock b/Cargo.lock index 84a28e94b6f0..d53e89163464 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -362,6 +362,17 @@ dependencies = [ "event-listener", ] +[[package]] +name = "async-recursion" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + [[package]] name = "async-trait" version = "0.1.68" @@ -370,7 +381,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -466,7 +477,7 @@ dependencies = [ [[package]] name = "binary-merkle-tree" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "hash-db", "log", @@ -483,9 +494,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.60.1" +version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6" +checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" dependencies = [ "bitflags", "cexpr", @@ -498,6 +509,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", + "syn 1.0.109", ] [[package]] @@ -738,9 +750,9 @@ dependencies = [ [[package]] name = "cargo_metadata" -version = "0.15.3" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a1ec454bc3eead8719cb56e15dbbfecdbc14e4b3a3ae4936cc6e31f5fc0d07" +checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" dependencies = [ "camino", "cargo-platform", @@ -2128,7 +2140,6 @@ dependencies = [ "fs-err", "proc-macro2", "quote", - "syn 1.0.109", ] [[package]] @@ -2144,6 +2155,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "expander" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f86a749cf851891866c10515ef6c299b5c69661465e9c3bbe7e07a2b77fb0f7" +dependencies = [ + "blake2", + "fs-err", + "proc-macro2", + "quote", + "syn 2.0.15", +] + [[package]] name = "eyre" version = "0.6.5" @@ -2365,7 +2389,7 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "fork-tree" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", ] @@ -2388,7 +2412,7 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" [[package]] name = "frame-benchmarking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "frame-support-procedural", @@ -2413,7 +2437,7 @@ dependencies = [ [[package]] name = "frame-benchmarking-cli" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "Inflector", "array-bytes", @@ -2460,18 +2484,18 @@ dependencies = [ [[package]] name = "frame-election-provider-solution-type" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "frame-election-provider-support" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-election-provider-solution-type", "frame-support", @@ -2488,7 +2512,7 @@ dependencies = [ [[package]] name = "frame-executive" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "frame-system", @@ -2504,9 +2528,9 @@ dependencies = [ [[package]] name = "frame-metadata" -version = "15.0.0" +version = "15.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6bb8542ef006ef0de09a5c4420787d79823c0ed7924225822362fd2bf2ff2d" +checksum = "878babb0b136e731cc77ec2fd883ff02745ff21e6fb662729953d44923df009c" dependencies = [ "cfg-if", "parity-scale-codec", @@ -2517,9 +2541,11 @@ dependencies = [ [[package]] name = "frame-remote-externalities" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ + "async-recursion", "futures", + "jsonrpsee", "log", "parity-scale-codec", "serde", @@ -2533,7 +2559,7 @@ dependencies = [ [[package]] name = "frame-support" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "bitflags", "environmental", @@ -2566,7 +2592,7 @@ dependencies = [ [[package]] name = "frame-support-procedural" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "Inflector", "cfg-expr", @@ -2576,35 +2602,35 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "frame-support-procedural-tools" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support-procedural-tools-derive", "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "frame-support-procedural-tools-derive" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "frame-support-test" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -2615,6 +2641,7 @@ dependencies = [ "rustversion", "scale-info", "serde", + "sp-api", "sp-arithmetic", "sp-core", "sp-io", @@ -2628,7 +2655,7 @@ dependencies = [ [[package]] name = "frame-support-test-pallet" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "frame-system", @@ -2639,7 +2666,7 @@ dependencies = [ [[package]] name = "frame-system" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "log", @@ -2657,7 +2684,7 @@ dependencies = [ [[package]] name = "frame-system-benchmarking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -2672,7 +2699,7 @@ dependencies = [ [[package]] name = "frame-system-rpc-runtime-api" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "sp-api", @@ -2681,7 +2708,7 @@ dependencies = [ [[package]] name = "frame-try-runtime" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "parity-scale-codec", @@ -2801,7 +2828,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -2863,7 +2890,7 @@ dependencies = [ [[package]] name = "generate-bags" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "chrono", "frame-election-provider-support", @@ -3019,9 +3046,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.11" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f1f717ddc7b2ba36df7e871fd88db79326551d3d6f1fc406fbfd28b582ff8e" +checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" dependencies = [ "bytes", "fnv", @@ -3032,7 +3059,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.6.9", + "tokio-util", "tracing", ] @@ -3116,6 +3143,12 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" +[[package]] +name = "hex-literal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" + [[package]] name = "hkdf" version = "0.12.3" @@ -3290,6 +3323,7 @@ dependencies = [ "rustls-native-certs", "tokio", "tokio-rustls", + "webpki-roots", ] [[package]] @@ -3530,6 +3564,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d291e3a5818a2384645fd9756362e6d89cf0541b0b916fa7702ea4a9833608e" dependencies = [ "jsonrpsee-core", + "jsonrpsee-http-client", "jsonrpsee-proc-macros", "jsonrpsee-server", "jsonrpsee-types", @@ -3553,7 +3588,7 @@ dependencies = [ "thiserror", "tokio", "tokio-rustls", - "tokio-util 0.7.1", + "tokio-util", "tracing", "webpki-roots", ] @@ -3586,6 +3621,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "jsonrpsee-http-client" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc345b0a43c6bc49b947ebeb936e886a419ee3d894421790c969cc56040542ad" +dependencies = [ + "async-trait", + "hyper", + "hyper-rustls", + "jsonrpsee-core", + "jsonrpsee-types", + "rustc-hash", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "jsonrpsee-proc-macros" version = "0.16.2" @@ -3616,7 +3670,7 @@ dependencies = [ "soketto", "tokio", "tokio-stream", - "tokio-util 0.7.1", + "tokio-util", "tower", "tracing", ] @@ -3668,7 +3722,7 @@ checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" [[package]] name = "kusama-runtime" -version = "0.9.39" +version = "0.9.41" dependencies = [ "bitvec", "frame-benchmarking", @@ -3680,7 +3734,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal", + "hex-literal 0.4.1", "kusama-runtime-constants", "log", "pallet-authority-discovery", @@ -3773,7 +3827,7 @@ dependencies = [ [[package]] name = "kusama-runtime-constants" -version = "0.9.39" +version = "0.9.41" dependencies = [ "frame-support", "polkadot-primitives", @@ -3805,9 +3859,9 @@ dependencies = [ [[package]] name = "kvdb-rocksdb" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2182b8219fee6bd83aacaab7344e840179ae079d5216aa4e249b4d704646a844" +checksum = "fe7a749456510c45f795e8b04a6a3e0976d0139213ecbf465843830ad55e2217" dependencies = [ "kvdb", "num_cpus", @@ -4247,7 +4301,7 @@ dependencies = [ "thiserror", "tinytemplate", "tokio", - "tokio-util 0.7.1", + "tokio-util", "webrtc", ] @@ -4286,9 +4340,9 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "0.8.0+7.4.4" +version = "0.10.0+7.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d" +checksum = "0fe4d5874f5ff2bc616e55e8c6086d478fcda13faf9495768a4aa1c22042d30b" dependencies = [ "bindgen", "bzip2-sys", @@ -4661,7 +4715,7 @@ dependencies = [ [[package]] name = "mmr-gadget" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "futures", "log", @@ -4680,7 +4734,7 @@ dependencies = [ [[package]] name = "mmr-rpc" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "anyhow", "jsonrpsee", @@ -5264,7 +5318,7 @@ dependencies = [ [[package]] name = "pallet-assets" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5279,7 +5333,7 @@ dependencies = [ [[package]] name = "pallet-authority-discovery" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "frame-system", @@ -5295,7 +5349,7 @@ dependencies = [ [[package]] name = "pallet-authorship" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "frame-system", @@ -5309,7 +5363,7 @@ dependencies = [ [[package]] name = "pallet-babe" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5322,7 +5376,7 @@ dependencies = [ "scale-info", "sp-application-crypto", "sp-consensus-babe", - "sp-consensus-vrf", + "sp-core", "sp-io", "sp-runtime", "sp-session", @@ -5333,7 +5387,7 @@ dependencies = [ [[package]] name = "pallet-bags-list" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -5353,7 +5407,7 @@ dependencies = [ [[package]] name = "pallet-bags-list-remote-tests" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-election-provider-support", "frame-remote-externalities", @@ -5372,7 +5426,7 @@ dependencies = [ [[package]] name = "pallet-balances" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5387,7 +5441,7 @@ dependencies = [ [[package]] name = "pallet-beefy" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "frame-system", @@ -5406,7 +5460,7 @@ dependencies = [ [[package]] name = "pallet-beefy-mmr" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "binary-merkle-tree", @@ -5430,7 +5484,7 @@ dependencies = [ [[package]] name = "pallet-bounties" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5448,7 +5502,7 @@ dependencies = [ [[package]] name = "pallet-child-bounties" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5467,7 +5521,7 @@ dependencies = [ [[package]] name = "pallet-collective" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5484,7 +5538,7 @@ dependencies = [ [[package]] name = "pallet-conviction-voting" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "assert_matches", "frame-benchmarking", @@ -5501,7 +5555,7 @@ dependencies = [ [[package]] name = "pallet-democracy" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5519,7 +5573,7 @@ dependencies = [ [[package]] name = "pallet-election-provider-multi-phase" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -5542,7 +5596,7 @@ dependencies = [ [[package]] name = "pallet-election-provider-support-benchmarking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -5555,7 +5609,7 @@ dependencies = [ [[package]] name = "pallet-elections-phragmen" version = "5.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5573,7 +5627,7 @@ dependencies = [ [[package]] name = "pallet-fast-unstake" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -5591,7 +5645,7 @@ dependencies = [ [[package]] name = "pallet-grandpa" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5614,7 +5668,7 @@ dependencies = [ [[package]] name = "pallet-identity" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "enumflags2", "frame-benchmarking", @@ -5630,7 +5684,7 @@ dependencies = [ [[package]] name = "pallet-im-online" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5650,7 +5704,7 @@ dependencies = [ [[package]] name = "pallet-indices" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5667,7 +5721,7 @@ dependencies = [ [[package]] name = "pallet-membership" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5684,7 +5738,7 @@ dependencies = [ [[package]] name = "pallet-mmr" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5701,7 +5755,7 @@ dependencies = [ [[package]] name = "pallet-multisig" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5717,7 +5771,7 @@ dependencies = [ [[package]] name = "pallet-nis" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5733,7 +5787,7 @@ dependencies = [ [[package]] name = "pallet-nomination-pools" version = "1.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "frame-system", @@ -5750,7 +5804,7 @@ dependencies = [ [[package]] name = "pallet-nomination-pools-benchmarking" version = "1.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -5770,7 +5824,7 @@ dependencies = [ [[package]] name = "pallet-nomination-pools-runtime-api" version = "1.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "pallet-nomination-pools", "parity-scale-codec", @@ -5781,7 +5835,7 @@ dependencies = [ [[package]] name = "pallet-offences" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "frame-system", @@ -5798,7 +5852,7 @@ dependencies = [ [[package]] name = "pallet-offences-benchmarking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -5822,7 +5876,7 @@ dependencies = [ [[package]] name = "pallet-preimage" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5839,7 +5893,7 @@ dependencies = [ [[package]] name = "pallet-proxy" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5854,7 +5908,7 @@ dependencies = [ [[package]] name = "pallet-ranked-collective" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5872,7 +5926,7 @@ dependencies = [ [[package]] name = "pallet-recovery" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5887,7 +5941,7 @@ dependencies = [ [[package]] name = "pallet-referenda" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "assert_matches", "frame-benchmarking", @@ -5906,7 +5960,7 @@ dependencies = [ [[package]] name = "pallet-scheduler" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5923,7 +5977,7 @@ dependencies = [ [[package]] name = "pallet-session" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "frame-system", @@ -5944,7 +5998,7 @@ dependencies = [ [[package]] name = "pallet-session-benchmarking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -5960,7 +6014,7 @@ dependencies = [ [[package]] name = "pallet-society" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "frame-system", @@ -5974,7 +6028,7 @@ dependencies = [ [[package]] name = "pallet-staking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-election-provider-support", @@ -5997,18 +6051,18 @@ dependencies = [ [[package]] name = "pallet-staking-reward-curve" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "pallet-staking-reward-fn" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "log", "sp-arithmetic", @@ -6017,7 +6071,7 @@ dependencies = [ [[package]] name = "pallet-staking-runtime-api" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "sp-api", @@ -6026,7 +6080,7 @@ dependencies = [ [[package]] name = "pallet-state-trie-migration" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -6043,7 +6097,7 @@ dependencies = [ [[package]] name = "pallet-sudo" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "frame-system", @@ -6057,7 +6111,7 @@ dependencies = [ [[package]] name = "pallet-timestamp" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -6075,7 +6129,7 @@ dependencies = [ [[package]] name = "pallet-tips" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -6094,7 +6148,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-support", "frame-system", @@ -6110,7 +6164,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "jsonrpsee", "pallet-transaction-payment-rpc-runtime-api", @@ -6126,7 +6180,7 @@ dependencies = [ [[package]] name = "pallet-transaction-payment-rpc-runtime-api" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "pallet-transaction-payment", "parity-scale-codec", @@ -6138,7 +6192,7 @@ dependencies = [ [[package]] name = "pallet-treasury" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -6155,7 +6209,7 @@ dependencies = [ [[package]] name = "pallet-uniques" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -6170,7 +6224,7 @@ dependencies = [ [[package]] name = "pallet-utility" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -6186,7 +6240,7 @@ dependencies = [ [[package]] name = "pallet-vesting" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -6201,7 +6255,7 @@ dependencies = [ [[package]] name = "pallet-whitelist" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-benchmarking", "frame-support", @@ -6215,7 +6269,7 @@ dependencies = [ [[package]] name = "pallet-xcm" -version = "0.9.39" +version = "0.9.41" dependencies = [ "bounded-collections", "frame-benchmarking", @@ -6239,7 +6293,7 @@ dependencies = [ [[package]] name = "pallet-xcm-benchmarks" -version = "0.9.39" +version = "0.9.41" dependencies = [ "frame-benchmarking", "frame-support", @@ -6560,14 +6614,14 @@ checksum = "e3d7ddaed09e0eb771a79ab0fd64609ba0afb0a8366421957936ad14cbd13630" [[package]] name = "polkadot" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_cmd", "color-eyre", "nix 0.26.2", "polkadot-cli", "polkadot-core-primitives", - "polkadot-node-core-pvf", + "polkadot-node-core-pvf-worker", "polkadot-overseer", "substrate-rpc-client", "tempfile", @@ -6577,7 +6631,7 @@ dependencies = [ [[package]] name = "polkadot-approval-distribution" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "env_logger 0.9.0", @@ -6603,7 +6657,7 @@ dependencies = [ [[package]] name = "polkadot-availability-bitfield-distribution" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "bitvec", @@ -6628,7 +6682,7 @@ dependencies = [ [[package]] name = "polkadot-availability-distribution" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "derive_more", @@ -6657,7 +6711,7 @@ dependencies = [ [[package]] name = "polkadot-availability-recovery" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "env_logger 0.9.0", @@ -6686,14 +6740,14 @@ dependencies = [ [[package]] name = "polkadot-cli" -version = "0.9.39" +version = "0.9.41" dependencies = [ "clap 4.0.15", "frame-benchmarking-cli", "futures", "log", "polkadot-client", - "polkadot-node-core-pvf", + "polkadot-node-core-pvf-worker", "polkadot-node-metrics", "polkadot-performance-test", "polkadot-service", @@ -6707,6 +6761,7 @@ dependencies = [ "sp-core", "sp-io", "sp-keyring", + "sp-maybe-compressed-blob", "substrate-build-script-utils", "thiserror", "try-runtime-cli", @@ -6714,7 +6769,7 @@ dependencies = [ [[package]] name = "polkadot-client" -version = "0.9.39" +version = "0.9.41" dependencies = [ "async-trait", "frame-benchmarking", @@ -6758,7 +6813,7 @@ dependencies = [ [[package]] name = "polkadot-collator-protocol" -version = "0.9.39" +version = "0.9.41" dependencies = [ "always-assert", "assert_matches", @@ -6787,7 +6842,7 @@ dependencies = [ [[package]] name = "polkadot-core-primitives" -version = "0.9.39" +version = "0.9.41" dependencies = [ "parity-scale-codec", "scale-info", @@ -6798,7 +6853,7 @@ dependencies = [ [[package]] name = "polkadot-dispute-distribution" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "async-trait", @@ -6830,7 +6885,7 @@ dependencies = [ [[package]] name = "polkadot-erasure-coding" -version = "0.9.39" +version = "0.9.41" dependencies = [ "criterion", "parity-scale-codec", @@ -6844,7 +6899,7 @@ dependencies = [ [[package]] name = "polkadot-gossip-support" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "async-trait", @@ -6871,7 +6926,7 @@ dependencies = [ [[package]] name = "polkadot-network-bridge" -version = "0.9.39" +version = "0.9.41" dependencies = [ "always-assert", "assert_matches", @@ -6900,7 +6955,7 @@ dependencies = [ [[package]] name = "polkadot-node-collation-generation" -version = "0.9.39" +version = "0.9.41" dependencies = [ "futures", "parity-scale-codec", @@ -6919,7 +6974,7 @@ dependencies = [ [[package]] name = "polkadot-node-core-approval-voting" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "async-trait", @@ -6958,7 +7013,7 @@ dependencies = [ [[package]] name = "polkadot-node-core-av-store" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "bitvec", @@ -6987,7 +7042,7 @@ dependencies = [ [[package]] name = "polkadot-node-core-backing" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "bitvec", @@ -7013,7 +7068,7 @@ dependencies = [ [[package]] name = "polkadot-node-core-bitfield-signing" -version = "0.9.39" +version = "0.9.41" dependencies = [ "futures", "polkadot-node-subsystem", @@ -7029,7 +7084,7 @@ dependencies = [ [[package]] name = "polkadot-node-core-candidate-validation" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "async-trait", @@ -7053,7 +7108,7 @@ dependencies = [ [[package]] name = "polkadot-node-core-chain-api" -version = "0.9.39" +version = "0.9.41" dependencies = [ "futures", "maplit", @@ -7072,7 +7127,7 @@ dependencies = [ [[package]] name = "polkadot-node-core-chain-selection" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "futures", @@ -7093,7 +7148,7 @@ dependencies = [ [[package]] name = "polkadot-node-core-dispute-coordinator" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "fatality", @@ -7121,7 +7176,7 @@ dependencies = [ [[package]] name = "polkadot-node-core-parachains-inherent" -version = "0.9.39" +version = "0.9.41" dependencies = [ "async-trait", "futures", @@ -7137,7 +7192,7 @@ dependencies = [ [[package]] name = "polkadot-node-core-provisioner" -version = "0.9.39" +version = "0.9.41" dependencies = [ "bitvec", "fatality", @@ -7158,14 +7213,13 @@ dependencies = [ [[package]] name = "polkadot-node-core-pvf" -version = "0.9.39" +version = "0.9.41" dependencies = [ "always-assert", "assert_matches", - "cpu-time", "futures", "futures-timer", - "hex-literal", + "hex-literal 0.3.4", "libc", "parity-scale-codec", "pin-project", @@ -7175,29 +7229,20 @@ dependencies = [ "polkadot-parachain", "polkadot-primitives", "rand 0.8.5", - "rayon", - "sc-executor", - "sc-executor-common", - "sc-executor-wasmtime", "slotmap", "sp-core", - "sp-externalities", - "sp-io", "sp-maybe-compressed-blob", "sp-tracing", "sp-wasm-interface", "substrate-build-script-utils", "tempfile", - "test-parachain-adder", - "test-parachain-halt", - "tikv-jemalloc-ctl", "tokio", "tracing-gum", ] [[package]] name = "polkadot-node-core-pvf-checker" -version = "0.9.39" +version = "0.9.41" dependencies = [ "futures", "futures-timer", @@ -7218,9 +7263,39 @@ dependencies = [ "tracing-gum", ] +[[package]] +name = "polkadot-node-core-pvf-worker" +version = "0.9.41" +dependencies = [ + "assert_matches", + "cpu-time", + "futures", + "libc", + "parity-scale-codec", + "polkadot-node-core-pvf", + "polkadot-parachain", + "polkadot-primitives", + "rayon", + "sc-executor", + "sc-executor-common", + "sc-executor-wasmtime", + "sp-core", + "sp-externalities", + "sp-io", + "sp-maybe-compressed-blob", + "sp-tracing", + "substrate-build-script-utils", + "tempfile", + "test-parachain-adder", + "test-parachain-halt", + "tikv-jemalloc-ctl", + "tokio", + "tracing-gum", +] + [[package]] name = "polkadot-node-core-runtime-api" -version = "0.9.39" +version = "0.9.41" dependencies = [ "futures", "lru 0.9.0", @@ -7241,7 +7316,7 @@ dependencies = [ [[package]] name = "polkadot-node-jaeger" -version = "0.9.39" +version = "0.9.41" dependencies = [ "lazy_static", "log", @@ -7258,7 +7333,7 @@ dependencies = [ [[package]] name = "polkadot-node-metrics" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_cmd", "bs58", @@ -7285,7 +7360,7 @@ dependencies = [ [[package]] name = "polkadot-node-network-protocol" -version = "0.9.39" +version = "0.9.41" dependencies = [ "async-trait", "derive_more", @@ -7307,7 +7382,7 @@ dependencies = [ [[package]] name = "polkadot-node-primitives" -version = "0.9.39" +version = "0.9.41" dependencies = [ "bounded-vec", "futures", @@ -7319,18 +7394,17 @@ dependencies = [ "serde", "sp-application-crypto", "sp-consensus-babe", - "sp-consensus-vrf", "sp-core", "sp-keystore", "sp-maybe-compressed-blob", "sp-runtime", "thiserror", - "zstd", + "zstd 0.11.2+zstd.1.5.2", ] [[package]] name = "polkadot-node-subsystem" -version = "0.9.39" +version = "0.9.41" dependencies = [ "polkadot-node-jaeger", "polkadot-node-subsystem-types", @@ -7339,7 +7413,7 @@ dependencies = [ [[package]] name = "polkadot-node-subsystem-test-helpers" -version = "0.9.39" +version = "0.9.41" dependencies = [ "async-trait", "futures", @@ -7357,7 +7431,7 @@ dependencies = [ [[package]] name = "polkadot-node-subsystem-types" -version = "0.9.39" +version = "0.9.41" dependencies = [ "async-trait", "derive_more", @@ -7379,7 +7453,7 @@ dependencies = [ [[package]] name = "polkadot-node-subsystem-util" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "async-trait", @@ -7420,7 +7494,7 @@ dependencies = [ [[package]] name = "polkadot-overseer" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "async-trait", @@ -7446,7 +7520,7 @@ dependencies = [ [[package]] name = "polkadot-parachain" -version = "0.9.39" +version = "0.9.41" dependencies = [ "bounded-collections", "derive_more", @@ -7462,25 +7536,27 @@ dependencies = [ [[package]] name = "polkadot-performance-test" -version = "0.9.39" +version = "0.9.41" dependencies = [ "env_logger 0.9.0", "kusama-runtime", "log", "polkadot-erasure-coding", - "polkadot-node-core-pvf", + "polkadot-node-core-pvf-worker", "polkadot-node-primitives", "polkadot-primitives", "quote", + "sc-executor-common", + "sp-maybe-compressed-blob", "thiserror", ] [[package]] name = "polkadot-primitives" -version = "0.9.39" +version = "0.9.41" dependencies = [ "bitvec", - "hex-literal", + "hex-literal 0.4.1", "parity-scale-codec", "polkadot-core-primitives", "polkadot-parachain", @@ -7502,7 +7578,7 @@ dependencies = [ [[package]] name = "polkadot-primitives-test-helpers" -version = "0.9.39" +version = "0.9.41" dependencies = [ "polkadot-primitives", "rand 0.8.5", @@ -7514,7 +7590,7 @@ dependencies = [ [[package]] name = "polkadot-rpc" -version = "0.9.39" +version = "0.9.41" dependencies = [ "jsonrpsee", "mmr-rpc", @@ -7545,7 +7621,7 @@ dependencies = [ [[package]] name = "polkadot-runtime" -version = "0.9.39" +version = "0.9.41" dependencies = [ "bitvec", "frame-benchmarking", @@ -7557,7 +7633,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal", + "hex-literal 0.4.1", "log", "pallet-authority-discovery", "pallet-authorship", @@ -7646,7 +7722,7 @@ dependencies = [ [[package]] name = "polkadot-runtime-common" -version = "0.9.39" +version = "0.9.41" dependencies = [ "bitvec", "frame-benchmarking", @@ -7654,7 +7730,7 @@ dependencies = [ "frame-support", "frame-support-test", "frame-system", - "hex-literal", + "hex-literal 0.4.1", "impl-trait-for-tuples", "libsecp256k1", "log", @@ -7696,7 +7772,7 @@ dependencies = [ [[package]] name = "polkadot-runtime-constants" -version = "0.9.39" +version = "0.9.41" dependencies = [ "frame-support", "polkadot-primitives", @@ -7709,7 +7785,7 @@ dependencies = [ [[package]] name = "polkadot-runtime-metrics" -version = "0.9.39" +version = "0.9.41" dependencies = [ "bs58", "frame-benchmarking", @@ -7721,7 +7797,7 @@ dependencies = [ [[package]] name = "polkadot-runtime-parachains" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "bitflags", @@ -7732,7 +7808,7 @@ dependencies = [ "frame-support-test", "frame-system", "futures", - "hex-literal", + "hex-literal 0.4.1", "log", "pallet-authority-discovery", "pallet-authorship", @@ -7774,7 +7850,7 @@ dependencies = [ [[package]] name = "polkadot-service" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "async-trait", @@ -7783,7 +7859,7 @@ dependencies = [ "frame-support", "frame-system-rpc-runtime-api", "futures", - "hex-literal", + "hex-literal 0.4.1", "kusama-runtime", "kusama-runtime-constants", "kvdb", @@ -7890,7 +7966,7 @@ dependencies = [ [[package]] name = "polkadot-statement-distribution" -version = "0.9.39" +version = "0.9.41" dependencies = [ "arrayvec 0.5.2", "assert_matches", @@ -7921,7 +7997,7 @@ dependencies = [ [[package]] name = "polkadot-statement-table" -version = "0.9.39" +version = "0.9.41" dependencies = [ "parity-scale-codec", "polkadot-primitives", @@ -7930,7 +8006,7 @@ dependencies = [ [[package]] name = "polkadot-test-client" -version = "0.9.39" +version = "0.9.41" dependencies = [ "futures", "parity-scale-codec", @@ -7956,7 +8032,7 @@ dependencies = [ [[package]] name = "polkadot-test-malus" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "async-trait", @@ -7969,7 +8045,7 @@ dependencies = [ "polkadot-node-core-backing", "polkadot-node-core-candidate-validation", "polkadot-node-core-dispute-coordinator", - "polkadot-node-core-pvf", + "polkadot-node-core-pvf-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", @@ -7984,7 +8060,7 @@ dependencies = [ [[package]] name = "polkadot-test-runtime" -version = "0.9.39" +version = "0.9.41" dependencies = [ "bitvec", "frame-election-provider-support", @@ -7992,7 +8068,7 @@ dependencies = [ "frame-support", "frame-system", "frame-system-rpc-runtime-api", - "hex-literal", + "hex-literal 0.4.1", "log", "pallet-authority-discovery", "pallet-authorship", @@ -8049,7 +8125,7 @@ dependencies = [ [[package]] name = "polkadot-test-service" -version = "0.9.39" +version = "0.9.41" dependencies = [ "frame-benchmarking", "frame-system", @@ -8103,7 +8179,7 @@ dependencies = [ [[package]] name = "polkadot-voter-bags" -version = "0.9.39" +version = "0.9.41" dependencies = [ "clap 4.0.15", "generate-bags", @@ -8293,9 +8369,9 @@ dependencies = [ [[package]] name = "proc-macro-warning" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d4f284d87b9cedc2ff57223cbc4e3937cd6063c01e92c8e2a8c080df0013933" +checksum = "f0e25495609acefcaeb5052edad8ac91017c9bc98fc38ef321ed524e50b68bac" dependencies = [ "proc-macro2", "quote", @@ -8743,7 +8819,7 @@ dependencies = [ [[package]] name = "remote-ext-tests-bags-list" -version = "0.9.39" +version = "0.9.41" dependencies = [ "clap 4.0.15", "frame-system", @@ -8854,9 +8930,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.19.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc" +checksum = "015439787fce1e75d55f279078d33ff14b4af5d93d995e8838ee4631301c8a99" dependencies = [ "libc", "librocksdb-sys", @@ -8864,7 +8940,7 @@ dependencies = [ [[package]] name = "rococo-runtime" -version = "0.9.39" +version = "0.9.41" dependencies = [ "binary-merkle-tree", "frame-benchmarking", @@ -8875,7 +8951,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal", + "hex-literal 0.4.1", "log", "pallet-authority-discovery", "pallet-authorship", @@ -8957,7 +9033,7 @@ dependencies = [ [[package]] name = "rococo-runtime-constants" -version = "0.9.39" +version = "0.9.41" dependencies = [ "frame-support", "polkadot-primitives", @@ -9172,7 +9248,7 @@ dependencies = [ [[package]] name = "sc-allocator" version = "4.1.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "log", "sp-core", @@ -9183,7 +9259,7 @@ dependencies = [ [[package]] name = "sc-authority-discovery" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "futures", @@ -9211,7 +9287,7 @@ dependencies = [ [[package]] name = "sc-basic-authorship" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "futures", "futures-timer", @@ -9234,7 +9310,7 @@ dependencies = [ [[package]] name = "sc-block-builder" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "sc-client-api", @@ -9249,7 +9325,7 @@ dependencies = [ [[package]] name = "sc-chain-spec" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "memmap2", "sc-chain-spec-derive", @@ -9268,18 +9344,18 @@ dependencies = [ [[package]] name = "sc-chain-spec-derive" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "sc-cli" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "chrono", @@ -9319,7 +9395,7 @@ dependencies = [ [[package]] name = "sc-client-api" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "fnv", "futures", @@ -9345,7 +9421,7 @@ dependencies = [ [[package]] name = "sc-client-db" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "hash-db", "kvdb", @@ -9371,7 +9447,7 @@ dependencies = [ [[package]] name = "sc-consensus" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "futures", @@ -9396,13 +9472,12 @@ dependencies = [ [[package]] name = "sc-consensus-babe" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "fork-tree", "futures", "log", - "merlin", "num-bigint", "num-rational", "num-traits", @@ -9415,7 +9490,6 @@ dependencies = [ "sc-keystore", "sc-telemetry", "scale-info", - "schnorrkel", "sp-api", "sp-application-crypto", "sp-block-builder", @@ -9423,7 +9497,6 @@ dependencies = [ "sp-consensus", "sp-consensus-babe", "sp-consensus-slots", - "sp-consensus-vrf", "sp-core", "sp-inherents", "sp-keystore", @@ -9435,7 +9508,7 @@ dependencies = [ [[package]] name = "sc-consensus-babe-rpc" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "futures", "jsonrpsee", @@ -9457,7 +9530,7 @@ dependencies = [ [[package]] name = "sc-consensus-beefy" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "async-trait", @@ -9492,7 +9565,7 @@ dependencies = [ [[package]] name = "sc-consensus-beefy-rpc" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "futures", "jsonrpsee", @@ -9511,7 +9584,7 @@ dependencies = [ [[package]] name = "sc-consensus-epochs" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "fork-tree", "parity-scale-codec", @@ -9524,7 +9597,7 @@ dependencies = [ [[package]] name = "sc-consensus-grandpa" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "ahash 0.8.2", "array-bytes", @@ -9564,7 +9637,7 @@ dependencies = [ [[package]] name = "sc-consensus-grandpa-rpc" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "finality-grandpa", "futures", @@ -9584,7 +9657,7 @@ dependencies = [ [[package]] name = "sc-consensus-slots" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "futures", @@ -9607,7 +9680,7 @@ dependencies = [ [[package]] name = "sc-executor" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "lru 0.8.1", "parity-scale-codec", @@ -9631,7 +9704,7 @@ dependencies = [ [[package]] name = "sc-executor-common" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "sc-allocator", "sp-maybe-compressed-blob", @@ -9644,7 +9717,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmi" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "log", "sc-allocator", @@ -9657,7 +9730,7 @@ dependencies = [ [[package]] name = "sc-executor-wasmtime" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "anyhow", "cfg-if", @@ -9675,7 +9748,7 @@ dependencies = [ [[package]] name = "sc-informant" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "ansi_term", "futures", @@ -9691,7 +9764,7 @@ dependencies = [ [[package]] name = "sc-keystore" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "async-trait", @@ -9706,7 +9779,7 @@ dependencies = [ [[package]] name = "sc-network" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "async-channel", @@ -9751,7 +9824,7 @@ dependencies = [ [[package]] name = "sc-network-bitswap" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "cid", "futures", @@ -9771,7 +9844,7 @@ dependencies = [ [[package]] name = "sc-network-common" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "async-trait", @@ -9799,7 +9872,7 @@ dependencies = [ [[package]] name = "sc-network-gossip" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "ahash 0.8.2", "futures", @@ -9818,7 +9891,7 @@ dependencies = [ [[package]] name = "sc-network-light" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "futures", @@ -9840,7 +9913,7 @@ dependencies = [ [[package]] name = "sc-network-sync" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "async-trait", @@ -9874,7 +9947,7 @@ dependencies = [ [[package]] name = "sc-network-transactions" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "futures", @@ -9894,7 +9967,7 @@ dependencies = [ [[package]] name = "sc-offchain" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "bytes", @@ -9925,7 +9998,7 @@ dependencies = [ [[package]] name = "sc-peerset" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "futures", "libp2p", @@ -9938,7 +10011,7 @@ dependencies = [ [[package]] name = "sc-proposer-metrics" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "log", "substrate-prometheus-endpoint", @@ -9947,7 +10020,7 @@ dependencies = [ [[package]] name = "sc-rpc" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "futures", "jsonrpsee", @@ -9977,7 +10050,7 @@ dependencies = [ [[package]] name = "sc-rpc-api" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "jsonrpsee", "parity-scale-codec", @@ -9996,7 +10069,7 @@ dependencies = [ [[package]] name = "sc-rpc-server" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "http", "jsonrpsee", @@ -10011,7 +10084,7 @@ dependencies = [ [[package]] name = "sc-rpc-spec-v2" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "futures", @@ -10037,7 +10110,7 @@ dependencies = [ [[package]] name = "sc-service" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "directories", @@ -10103,7 +10176,7 @@ dependencies = [ [[package]] name = "sc-state-db" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "log", "parity-scale-codec", @@ -10114,7 +10187,7 @@ dependencies = [ [[package]] name = "sc-storage-monitor" version = "0.1.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "clap 4.0.15", "fs4", @@ -10130,7 +10203,7 @@ dependencies = [ [[package]] name = "sc-sync-state-rpc" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "jsonrpsee", "parity-scale-codec", @@ -10149,7 +10222,7 @@ dependencies = [ [[package]] name = "sc-sysinfo" version = "6.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "futures", "libc", @@ -10168,7 +10241,7 @@ dependencies = [ [[package]] name = "sc-telemetry" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "chrono", "futures", @@ -10187,7 +10260,7 @@ dependencies = [ [[package]] name = "sc-tracing" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "ansi_term", "atty", @@ -10218,18 +10291,18 @@ dependencies = [ [[package]] name = "sc-tracing-proc-macro" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "sc-transaction-pool" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "futures", @@ -10256,7 +10329,7 @@ dependencies = [ [[package]] name = "sc-transaction-pool-api" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "futures", @@ -10270,7 +10343,7 @@ dependencies = [ [[package]] name = "sc-utils" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-channel", "futures", @@ -10516,7 +10589,7 @@ checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -10530,15 +10603,24 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.85" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -10732,7 +10814,7 @@ checksum = "03b634d87b960ab1a38c4fe143b508576f075e7c978bfad18217645ebfdfa2ec" [[package]] name = "slot-range-helper" -version = "0.9.39" +version = "0.9.41" dependencies = [ "enumn", "parity-scale-codec", @@ -10809,13 +10891,15 @@ dependencies = [ [[package]] name = "sp-api" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "hash-db", "log", "parity-scale-codec", + "scale-info", "sp-api-proc-macro", "sp-core", + "sp-metadata-ir", "sp-runtime", "sp-state-machine", "sp-std", @@ -10827,7 +10911,7 @@ dependencies = [ [[package]] name = "sp-api-proc-macro" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "Inflector", "blake2", @@ -10835,13 +10919,13 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "sp-application-crypto" version = "7.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "scale-info", @@ -10854,7 +10938,7 @@ dependencies = [ [[package]] name = "sp-arithmetic" version = "6.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "integer-sqrt", "num-traits", @@ -10868,7 +10952,7 @@ dependencies = [ [[package]] name = "sp-authority-discovery" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "scale-info", @@ -10881,7 +10965,7 @@ dependencies = [ [[package]] name = "sp-block-builder" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "sp-api", @@ -10893,7 +10977,7 @@ dependencies = [ [[package]] name = "sp-blockchain" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "futures", "log", @@ -10911,7 +10995,7 @@ dependencies = [ [[package]] name = "sp-consensus" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "futures", @@ -10926,7 +11010,7 @@ dependencies = [ [[package]] name = "sp-consensus-aura" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "parity-scale-codec", @@ -10944,10 +11028,9 @@ dependencies = [ [[package]] name = "sp-consensus-babe" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", - "merlin", "parity-scale-codec", "scale-info", "serde", @@ -10955,7 +11038,6 @@ dependencies = [ "sp-application-crypto", "sp-consensus", "sp-consensus-slots", - "sp-consensus-vrf", "sp-core", "sp-inherents", "sp-keystore", @@ -10967,7 +11049,7 @@ dependencies = [ [[package]] name = "sp-consensus-beefy" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "lazy_static", "parity-scale-codec", @@ -10986,7 +11068,7 @@ dependencies = [ [[package]] name = "sp-consensus-grandpa" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "finality-grandpa", "log", @@ -11004,7 +11086,7 @@ dependencies = [ [[package]] name = "sp-consensus-slots" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "scale-info", @@ -11013,23 +11095,10 @@ dependencies = [ "sp-timestamp", ] -[[package]] -name = "sp-consensus-vrf" -version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" -dependencies = [ - "parity-scale-codec", - "scale-info", - "schnorrkel", - "sp-core", - "sp-runtime", - "sp-std", -] - [[package]] name = "sp-core" version = "7.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "bitflags", @@ -11072,7 +11141,7 @@ dependencies = [ [[package]] name = "sp-core-hashing" version = "5.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "blake2b_simd", "byteorder", @@ -11086,18 +11155,18 @@ dependencies = [ [[package]] name = "sp-core-hashing-proc-macro" version = "5.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "proc-macro2", "quote", "sp-core-hashing", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "sp-database" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "kvdb", "parking_lot 0.12.1", @@ -11106,17 +11175,17 @@ dependencies = [ [[package]] name = "sp-debug-derive" version = "5.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "sp-externalities" version = "0.13.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "environmental", "parity-scale-codec", @@ -11127,7 +11196,7 @@ dependencies = [ [[package]] name = "sp-inherents" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "impl-trait-for-tuples", @@ -11142,7 +11211,7 @@ dependencies = [ [[package]] name = "sp-io" version = "7.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "bytes", "ed25519", @@ -11168,7 +11237,7 @@ dependencies = [ [[package]] name = "sp-keyring" version = "7.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "lazy_static", "sp-core", @@ -11179,13 +11248,11 @@ dependencies = [ [[package]] name = "sp-keystore" version = "0.13.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "futures", - "merlin", "parity-scale-codec", "parking_lot 0.12.1", - "schnorrkel", "serde", "sp-core", "sp-externalities", @@ -11195,16 +11262,27 @@ dependencies = [ [[package]] name = "sp-maybe-compressed-blob" version = "4.1.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "thiserror", - "zstd", + "zstd 0.12.3+zstd.1.5.2", +] + +[[package]] +name = "sp-metadata-ir" +version = "0.1.0" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" +dependencies = [ + "frame-metadata", + "parity-scale-codec", + "scale-info", + "sp-std", ] [[package]] name = "sp-mmr-primitives" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "ckb-merkle-mountain-range", "log", @@ -11222,7 +11300,7 @@ dependencies = [ [[package]] name = "sp-npos-elections" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "scale-info", @@ -11236,7 +11314,7 @@ dependencies = [ [[package]] name = "sp-offchain" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "sp-api", "sp-core", @@ -11246,7 +11324,7 @@ dependencies = [ [[package]] name = "sp-panic-handler" version = "5.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "backtrace", "lazy_static", @@ -11256,7 +11334,7 @@ dependencies = [ [[package]] name = "sp-rpc" version = "6.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "rustc-hash", "serde", @@ -11266,7 +11344,7 @@ dependencies = [ [[package]] name = "sp-runtime" version = "7.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "either", "hash256-std-hasher", @@ -11288,7 +11366,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface" version = "7.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "bytes", "impl-trait-for-tuples", @@ -11306,19 +11384,19 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" version = "6.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "Inflector", "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "sp-session" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "scale-info", @@ -11332,7 +11410,7 @@ dependencies = [ [[package]] name = "sp-staking" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "scale-info", @@ -11344,7 +11422,7 @@ dependencies = [ [[package]] name = "sp-state-machine" version = "0.13.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "hash-db", "log", @@ -11364,12 +11442,12 @@ dependencies = [ [[package]] name = "sp-std" version = "5.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" [[package]] name = "sp-storage" version = "7.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "impl-serde", "parity-scale-codec", @@ -11382,7 +11460,7 @@ dependencies = [ [[package]] name = "sp-timestamp" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "futures-timer", @@ -11397,7 +11475,7 @@ dependencies = [ [[package]] name = "sp-tracing" version = "6.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "sp-std", @@ -11409,7 +11487,7 @@ dependencies = [ [[package]] name = "sp-transaction-pool" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "sp-api", "sp-runtime", @@ -11418,7 +11496,7 @@ dependencies = [ [[package]] name = "sp-transaction-storage-proof" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "log", @@ -11434,7 +11512,7 @@ dependencies = [ [[package]] name = "sp-trie" version = "7.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "ahash 0.8.2", "hash-db", @@ -11457,7 +11535,7 @@ dependencies = [ [[package]] name = "sp-version" version = "5.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "impl-serde", "parity-scale-codec", @@ -11474,18 +11552,18 @@ dependencies = [ [[package]] name = "sp-version-proc-macro" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "sp-wasm-interface" version = "7.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "anyhow", "impl-trait-for-tuples", @@ -11499,7 +11577,7 @@ dependencies = [ [[package]] name = "sp-weights" version = "4.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "parity-scale-codec", "scale-info", @@ -11560,7 +11638,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "staking-miner" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_cmd", "clap 4.0.15", @@ -11729,7 +11807,7 @@ dependencies = [ [[package]] name = "substrate-build-script-utils" version = "3.0.0" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "platforms 2.0.0", ] @@ -11737,7 +11815,7 @@ dependencies = [ [[package]] name = "substrate-frame-rpc-system" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "frame-system-rpc-runtime-api", "futures", @@ -11756,7 +11834,7 @@ dependencies = [ [[package]] name = "substrate-prometheus-endpoint" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "hyper", "log", @@ -11768,7 +11846,7 @@ dependencies = [ [[package]] name = "substrate-rpc-client" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "jsonrpsee", @@ -11781,7 +11859,7 @@ dependencies = [ [[package]] name = "substrate-state-trie-migration-rpc" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "jsonrpsee", "log", @@ -11800,7 +11878,7 @@ dependencies = [ [[package]] name = "substrate-test-client" version = "2.0.1" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "array-bytes", "async-trait", @@ -11826,7 +11904,7 @@ dependencies = [ [[package]] name = "substrate-test-utils" version = "4.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "futures", "substrate-test-utils-derive", @@ -11836,18 +11914,18 @@ dependencies = [ [[package]] name = "substrate-test-utils-derive" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "substrate-wasm-builder" version = "5.0.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "ansi_term", "build-helper", @@ -11856,7 +11934,7 @@ dependencies = [ "sp-maybe-compressed-blob", "strum", "tempfile", - "toml", + "toml 0.7.3", "walkdir", "wasm-opt", ] @@ -11921,9 +11999,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.13" +version = "2.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" dependencies = [ "proc-macro2", "quote", @@ -12006,7 +12084,7 @@ checksum = "13a4ec180a2de59b57434704ccfad967f789b12737738798fa08798cd5824c16" [[package]] name = "test-parachain-adder" -version = "0.9.39" +version = "0.9.41" dependencies = [ "dlmalloc", "parity-scale-codec", @@ -12019,7 +12097,7 @@ dependencies = [ [[package]] name = "test-parachain-adder-collator" -version = "0.9.39" +version = "0.9.41" dependencies = [ "clap 4.0.15", "futures", @@ -12027,7 +12105,7 @@ dependencies = [ "log", "parity-scale-codec", "polkadot-cli", - "polkadot-node-core-pvf", + "polkadot-node-core-pvf-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-parachain", @@ -12045,7 +12123,7 @@ dependencies = [ [[package]] name = "test-parachain-halt" -version = "0.9.39" +version = "0.9.41" dependencies = [ "rustversion", "substrate-wasm-builder", @@ -12053,7 +12131,7 @@ dependencies = [ [[package]] name = "test-parachain-undying" -version = "0.9.39" +version = "0.9.41" dependencies = [ "dlmalloc", "log", @@ -12067,7 +12145,7 @@ dependencies = [ [[package]] name = "test-parachain-undying-collator" -version = "0.9.39" +version = "0.9.41" dependencies = [ "clap 4.0.15", "futures", @@ -12075,7 +12153,7 @@ dependencies = [ "log", "parity-scale-codec", "polkadot-cli", - "polkadot-node-core-pvf", + "polkadot-node-core-pvf-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-parachain", @@ -12093,7 +12171,7 @@ dependencies = [ [[package]] name = "test-parachains" -version = "0.9.39" +version = "0.9.41" dependencies = [ "parity-scale-codec", "sp-core", @@ -12104,7 +12182,7 @@ dependencies = [ [[package]] name = "test-runtime-constants" -version = "0.9.39" +version = "0.9.41" dependencies = [ "frame-support", "polkadot-primitives", @@ -12138,7 +12216,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -12362,7 +12440,7 @@ dependencies = [ "futures-core", "pin-project-lite 0.2.9", "tokio", - "tokio-util 0.7.1", + "tokio-util", ] [[package]] @@ -12377,20 +12455,6 @@ dependencies = [ "tungstenite", ] -[[package]] -name = "tokio-util" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.2.9", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.1" @@ -12403,6 +12467,7 @@ dependencies = [ "futures-sink", "pin-project-lite 0.2.9", "tokio", + "tracing", ] [[package]] @@ -12414,11 +12479,26 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + [[package]] name = "toml_datetime" version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +dependencies = [ + "serde", +] [[package]] name = "toml_edit" @@ -12427,6 +12507,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" dependencies = [ "indexmap", + "serde", + "serde_spanned", "toml_datetime", "winnow", ] @@ -12518,7 +12600,7 @@ dependencies = [ [[package]] name = "tracing-gum" -version = "0.9.39" +version = "0.9.41" dependencies = [ "polkadot-node-jaeger", "polkadot-primitives", @@ -12528,14 +12610,14 @@ dependencies = [ [[package]] name = "tracing-gum-proc-macro" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", - "expander 0.0.6", + "expander 2.0.0", "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] @@ -12677,7 +12759,7 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "try-runtime-cli" version = "0.10.0-dev" -source = "git+https://github.com/paritytech/substrate?branch=master#7bbfe737a180e548ace7e819099dcb62cf48fa11" +source = "git+https://github.com/paritytech/substrate?branch=master#2fde3802da6256b5846ce33760685ca40008d00d" dependencies = [ "async-trait", "clap 4.0.15", @@ -12708,7 +12790,7 @@ dependencies = [ "sp-version", "sp-weights", "substrate-rpc-client", - "zstd", + "zstd 0.12.3+zstd.1.5.2", ] [[package]] @@ -12724,7 +12806,7 @@ dependencies = [ "serde_derive", "serde_json", "termcolor", - "toml", + "toml 0.5.11", ] [[package]] @@ -13230,9 +13312,9 @@ dependencies = [ "rustix 0.36.7", "serde", "sha2 0.10.2", - "toml", + "toml 0.5.11", "windows-sys 0.42.0", - "zstd", + "zstd 0.11.2+zstd.1.5.2", ] [[package]] @@ -13619,7 +13701,7 @@ dependencies = [ [[package]] name = "westend-runtime" -version = "0.9.39" +version = "0.9.41" dependencies = [ "bitvec", "frame-benchmarking", @@ -13631,7 +13713,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal", + "hex-literal 0.4.1", "log", "pallet-authority-discovery", "pallet-authorship", @@ -13716,7 +13798,7 @@ dependencies = [ [[package]] name = "westend-runtime-constants" -version = "0.9.39" +version = "0.9.41" dependencies = [ "frame-support", "polkadot-primitives", @@ -14025,12 +14107,12 @@ dependencies = [ [[package]] name = "xcm" -version = "0.9.39" +version = "0.9.41" dependencies = [ "bounded-collections", "derivative", "hex", - "hex-literal", + "hex-literal 0.4.1", "impl-trait-for-tuples", "log", "parity-scale-codec", @@ -14043,7 +14125,7 @@ dependencies = [ [[package]] name = "xcm-builder" -version = "0.9.39" +version = "0.9.41" dependencies = [ "assert_matches", "frame-support", @@ -14068,7 +14150,7 @@ dependencies = [ [[package]] name = "xcm-executor" -version = "0.9.39" +version = "0.9.41" dependencies = [ "environmental", "frame-benchmarking", @@ -14087,7 +14169,7 @@ dependencies = [ [[package]] name = "xcm-executor-integration-tests" -version = "0.9.39" +version = "0.9.41" dependencies = [ "frame-support", "frame-system", @@ -14107,17 +14189,17 @@ dependencies = [ [[package]] name = "xcm-procedural" -version = "0.9.39" +version = "0.9.41" dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] name = "xcm-simulator" -version = "0.9.39" +version = "0.9.41" dependencies = [ "frame-support", "parity-scale-codec", @@ -14133,7 +14215,7 @@ dependencies = [ [[package]] name = "xcm-simulator-example" -version = "0.9.39" +version = "0.9.41" dependencies = [ "frame-support", "frame-system", @@ -14159,7 +14241,7 @@ dependencies = [ [[package]] name = "xcm-simulator-fuzzer" -version = "0.9.39" +version = "0.9.41" dependencies = [ "arbitrary", "frame-support", @@ -14228,7 +14310,7 @@ dependencies = [ [[package]] name = "zombienet-backchannel" -version = "0.9.39" +version = "0.9.41" dependencies = [ "futures-util", "lazy_static", @@ -14249,7 +14331,16 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.12.3+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +dependencies = [ + "zstd-safe 6.0.5+zstd.1.5.4", ] [[package]] @@ -14262,12 +14353,23 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "6.0.5+zstd.1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" +dependencies = [ + "libc", + "zstd-sys", +] + [[package]] name = "zstd-sys" -version = "2.0.1+zstd.1.5.2" +version = "2.0.8+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" dependencies = [ "cc", "libc", + "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 5ec0a74d5cac..886b489a5024 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ version.workspace = true authors = ["Parity Technologies "] edition = "2021" repository = "https://github.com/paritytech/polkadot.git" -version = "0.9.39" +version = "0.9.41" [dependencies] color-eyre = { version = "0.6.1", default-features = false } @@ -24,7 +24,7 @@ tikv-jemallocator = "0.5.0" # Crates in our workspace, defined as dependencies so we can pass them feature flags. polkadot-cli = { path = "cli", features = [ "kusama-native", "westend-native", "rococo-native" ] } -polkadot-node-core-pvf = { path = "node/core/pvf" } +polkadot-node-core-pvf-worker = { path = "node/core/pvf/worker" } polkadot-overseer = { path = "node/overseer" } [dev-dependencies] @@ -80,6 +80,7 @@ members = [ "node/core/parachains-inherent", "node/core/provisioner", "node/core/pvf", + "node/core/pvf/worker", "node/core/pvf-checker", "node/core/runtime-api", "node/network/approval-distribution", @@ -206,7 +207,7 @@ try-runtime = [ "polkadot-cli/try-runtime" ] fast-runtime = [ "polkadot-cli/fast-runtime" ] runtime-metrics = [ "polkadot-cli/runtime-metrics" ] pyroscope = ["polkadot-cli/pyroscope"] -jemalloc-allocator = ["polkadot-node-core-pvf/jemalloc-allocator", "polkadot-overseer/jemalloc-allocator"] +jemalloc-allocator = ["polkadot-node-core-pvf-worker/jemalloc-allocator", "polkadot-overseer/jemalloc-allocator"] # Configuration for building a .deb package - for use with `cargo-deb` [package.metadata.deb] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 01247bbc996f..4d08ee18ed1b 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -22,12 +22,13 @@ pyro = { package = "pyroscope", version = "0.3.1", optional = true } service = { package = "polkadot-service", path = "../node/service", default-features = false, optional = true } polkadot-client = { path = "../node/client", optional = true } -polkadot-node-core-pvf = { path = "../node/core/pvf", optional = true } +polkadot-node-core-pvf-worker = { path = "../node/core/pvf/worker", optional = true } polkadot-performance-test = { path = "../node/test/performance-test", optional = true } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-keyring = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-maybe-compressed-blob = { git = "https://github.com/paritytech/substrate", branch = "master" } frame-benchmarking-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } try-runtime-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true } @@ -52,7 +53,7 @@ cli = [ "frame-benchmarking-cli", "try-runtime-cli", "polkadot-client", - "polkadot-node-core-pvf", + "polkadot-node-core-pvf-worker", ] runtime-benchmarks = [ "service/runtime-benchmarks", diff --git a/cli/src/command.rs b/cli/src/command.rs index 2f0bc9e2f856..c0e96de2a54b 100644 --- a/cli/src/command.rs +++ b/cli/src/command.rs @@ -494,7 +494,7 @@ pub fn run() -> Result<()> { #[cfg(not(target_os = "android"))] { - polkadot_node_core_pvf::prepare_worker_entrypoint( + polkadot_node_core_pvf_worker::prepare_worker_entrypoint( &cmd.socket_path, Some(&cmd.node_impl_version), ); @@ -516,7 +516,7 @@ pub fn run() -> Result<()> { #[cfg(not(target_os = "android"))] { - polkadot_node_core_pvf::execute_worker_entrypoint( + polkadot_node_core_pvf_worker::execute_worker_entrypoint( &cmd.socket_path, Some(&cmd.node_impl_version), ); diff --git a/cli/src/host_perf_check.rs b/cli/src/host_perf_check.rs index 1225c4708a3a..adfdebce6779 100644 --- a/cli/src/host_perf_check.rs +++ b/cli/src/host_perf_check.rs @@ -15,7 +15,6 @@ // along with Polkadot. If not, see . use log::info; -use polkadot_node_core_pvf::sp_maybe_compressed_blob; use polkadot_performance_test::{ measure_erasure_coding, measure_pvf_prepare, PerfCheckError, ERASURE_CODING_N_VALIDATORS, ERASURE_CODING_TIME_LIMIT, PVF_PREPARE_TIME_LIMIT, VALIDATION_CODE_BOMB_LIMIT, diff --git a/node/core/approval-voting/src/criteria.rs b/node/core/approval-voting/src/criteria.rs index e8690895217a..40a24e2dd937 100644 --- a/node/core/approval-voting/src/criteria.rs +++ b/node/core/approval-voting/src/criteria.rs @@ -356,10 +356,10 @@ fn compute_relay_vrf_modulo_assignments( // has been executed. let cert = AssignmentCert { kind: AssignmentCertKind::RelayVRFModulo { sample: rvm_sample }, - vrf: ( - approval_types::VRFOutput(vrf_in_out.to_output()), - approval_types::VRFProof(vrf_proof), - ), + vrf: approval_types::VrfSignature { + output: approval_types::VrfOutput(vrf_in_out.to_output()), + proof: approval_types::VrfProof(vrf_proof), + }, }; // All assignments of type RelayVRFModulo have tranche 0. @@ -393,10 +393,10 @@ fn compute_relay_vrf_delay_assignments( let cert = AssignmentCert { kind: AssignmentCertKind::RelayVRFDelay { core_index: core }, - vrf: ( - approval_types::VRFOutput(vrf_in_out.to_output()), - approval_types::VRFProof(vrf_proof), - ), + vrf: approval_types::VrfSignature { + output: approval_types::VrfOutput(vrf_in_out.to_output()), + proof: approval_types::VrfProof(vrf_proof), + }, }; let our_assignment = OurAssignment { cert, tranche, validator_index, triggered: false }; @@ -496,7 +496,7 @@ pub(crate) fn check_assignment_cert( return Err(InvalidAssignment(Reason::IsInBackingGroup)) } - let (vrf_output, vrf_proof) = &assignment.vrf; + let vrf_signature = &assignment.vrf; match assignment.kind { AssignmentCertKind::RelayVRFModulo { sample } => { if sample >= config.relay_vrf_modulo_samples { @@ -506,8 +506,8 @@ pub(crate) fn check_assignment_cert( let (vrf_in_out, _) = public .vrf_verify_extra( relay_vrf_modulo_transcript(relay_vrf_story, sample), - &vrf_output.0, - &vrf_proof.0, + &vrf_signature.output.0, + &vrf_signature.proof.0, assigned_core_transcript(claimed_core_index), ) .map_err(|_| InvalidAssignment(Reason::VRFModuloOutputMismatch))?; @@ -527,8 +527,8 @@ pub(crate) fn check_assignment_cert( let (vrf_in_out, _) = public .vrf_verify( relay_vrf_delay_transcript(relay_vrf_story, core_index), - &vrf_output.0, - &vrf_proof.0, + &vrf_signature.output.0, + &vrf_signature.proof.0, ) .map_err(|_| InvalidAssignment(Reason::VRFDelayOutputMismatch))?; @@ -552,7 +552,7 @@ fn is_in_backing_group( #[cfg(test)] mod tests { use super::*; - use polkadot_node_primitives::approval::{VRFOutput, VRFProof}; + use crate::import::tests::garbage_vrf_signature; use polkadot_primitives::{Hash, ASSIGNMENT_KEY_TYPE_ID}; use sp_application_crypto::sr25519; use sp_core::crypto::Pair as PairT; @@ -606,15 +606,6 @@ mod tests { .collect() } - // used for generating assignments where the validity of the VRF doesn't matter. - fn garbage_vrf() -> (VRFOutput, VRFProof) { - let key = Sr25519Keyring::Alice.pair(); - let key: &schnorrkel::Keypair = key.as_ref(); - - let (o, p, _) = key.vrf_sign(Transcript::new(b"test-garbage")); - (VRFOutput(o.to_output()), VRFProof(p)) - } - #[test] fn assignments_produced_for_non_backing() { let keystore = make_keystore(&[Sr25519Keyring::Alice]); @@ -825,7 +816,7 @@ mod tests { check_mutated_assignments(40, 10, 8, |m| { match m.cert.kind.clone() { AssignmentCertKind::RelayVRFDelay { .. } => { - m.cert.vrf = garbage_vrf(); + m.cert.vrf = garbage_vrf_signature(); Some(false) }, _ => None, // skip everything else. @@ -838,7 +829,7 @@ mod tests { check_mutated_assignments(200, 100, 25, |m| { match m.cert.kind.clone() { AssignmentCertKind::RelayVRFModulo { .. } => { - m.cert.vrf = garbage_vrf(); + m.cert.vrf = garbage_vrf_signature(); Some(false) }, _ => None, // skip everything else. diff --git a/node/core/approval-voting/src/import.rs b/node/core/approval-voting/src/import.rs index ebe300bc26ea..f3a571a7133d 100644 --- a/node/core/approval-voting/src/import.rs +++ b/node/core/approval-voting/src/import.rs @@ -612,8 +612,7 @@ pub(crate) mod tests { use crate::approval_db::v1::DbBackend; use ::test_helpers::{dummy_candidate_receipt, dummy_hash}; use assert_matches::assert_matches; - use merlin::Transcript; - use polkadot_node_primitives::approval::{VRFOutput, VRFProof}; + use polkadot_node_primitives::approval::{VrfSignature, VrfTranscript}; use polkadot_node_subsystem::messages::{AllMessages, ApprovalVotingMessage}; use polkadot_node_subsystem_test_helpers::make_subsystem_context; use polkadot_node_subsystem_util::database::Database; @@ -622,7 +621,7 @@ pub(crate) mod tests { digests::{CompatibleDigestItem, PreDigest, SecondaryVRFPreDigest}, AllowedSlots, BabeEpochConfiguration, Epoch as BabeEpoch, }; - use sp_core::testing::TaskExecutor; + use sp_core::{crypto::VrfSigner, testing::TaskExecutor}; use sp_keyring::sr25519::Keyring as Sr25519Keyring; pub(crate) use sp_runtime::{Digest, DigestItem}; use std::{pin::Pin, sync::Arc}; @@ -703,12 +702,9 @@ pub(crate) mod tests { } // used for generating assignments where the validity of the VRF doesn't matter. - pub(crate) fn garbage_vrf() -> (VRFOutput, VRFProof) { - let key = Sr25519Keyring::Alice.pair(); - let key: &schnorrkel::Keypair = key.as_ref(); - - let (o, p, _) = key.vrf_sign(Transcript::new(b"test-garbage")); - (VRFOutput(o.to_output()), VRFProof(p)) + pub(crate) fn garbage_vrf_signature() -> VrfSignature { + let transcript = VrfTranscript::new(b"test-garbage", &[]); + Sr25519Keyring::Alice.pair().vrf_sign(&transcript) } fn dummy_session_info(index: SessionIndex) -> SessionInfo { @@ -743,9 +739,9 @@ pub(crate) mod tests { let header = Header { digest: { let mut d = Digest::default(); - let (vrf_output, vrf_proof) = garbage_vrf(); + let vrf_signature = garbage_vrf_signature(); d.push(DigestItem::babe_pre_digest(PreDigest::SecondaryVRF( - SecondaryVRFPreDigest { authority_index: 0, slot, vrf_output, vrf_proof }, + SecondaryVRFPreDigest { authority_index: 0, slot, vrf_signature }, ))); d @@ -1042,9 +1038,9 @@ pub(crate) mod tests { let header = Header { digest: { let mut d = Digest::default(); - let (vrf_output, vrf_proof) = garbage_vrf(); + let vrf_signature = garbage_vrf_signature(); d.push(DigestItem::babe_pre_digest(PreDigest::SecondaryVRF( - SecondaryVRFPreDigest { authority_index: 0, slot, vrf_output, vrf_proof }, + SecondaryVRFPreDigest { authority_index: 0, slot, vrf_signature }, ))); d.push(ConsensusLog::ForceApprove(3).into()); @@ -1194,9 +1190,9 @@ pub(crate) mod tests { let header = Header { digest: { let mut d = Digest::default(); - let (vrf_output, vrf_proof) = garbage_vrf(); + let vrf_signature = garbage_vrf_signature(); d.push(DigestItem::babe_pre_digest(PreDigest::SecondaryVRF( - SecondaryVRFPreDigest { authority_index: 0, slot, vrf_output, vrf_proof }, + SecondaryVRFPreDigest { authority_index: 0, slot, vrf_signature }, ))); d diff --git a/node/core/approval-voting/src/tests.rs b/node/core/approval-voting/src/tests.rs index e9d599bdd91e..9ec0c89d4baa 100644 --- a/node/core/approval-voting/src/tests.rs +++ b/node/core/approval-voting/src/tests.rs @@ -19,7 +19,7 @@ use crate::tests::test_constants::TEST_CONFIG; use super::*; use polkadot_node_primitives::{ approval::{ - AssignmentCert, AssignmentCertKind, DelayTranche, VRFOutput, VRFProof, + AssignmentCert, AssignmentCertKind, DelayTranche, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, }, AvailableData, BlockData, PoV, @@ -56,8 +56,8 @@ use super::{ approval_db::v1::StoredBlockRange, backend::BackendWriteOp, import::tests::{ - garbage_vrf, AllowedSlots, BabeEpoch, BabeEpochConfiguration, CompatibleDigestItem, Digest, - DigestItem, PreDigest, SecondaryVRFPreDigest, + garbage_vrf_signature, AllowedSlots, BabeEpoch, BabeEpochConfiguration, + CompatibleDigestItem, Digest, DigestItem, PreDigest, SecondaryVRFPreDigest, }, }; @@ -392,7 +392,7 @@ fn garbage_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert { let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg)); let out = inout.to_output(); - AssignmentCert { kind, vrf: (VRFOutput(out), VRFProof(proof)) } + AssignmentCert { kind, vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) } } } fn sign_approval( @@ -721,9 +721,9 @@ impl ChainBuilder { fn make_header(parent_hash: Hash, slot: Slot, number: u32) -> Header { let digest = { let mut digest = Digest::default(); - let (vrf_output, vrf_proof) = garbage_vrf(); + let vrf_signature = garbage_vrf_signature(); digest.push(DigestItem::babe_pre_digest(PreDigest::SecondaryVRF( - SecondaryVRFPreDigest { authority_index: 0, slot, vrf_output, vrf_proof }, + SecondaryVRFPreDigest { authority_index: 0, slot, vrf_signature }, ))); digest }; diff --git a/node/core/candidate-validation/src/lib.rs b/node/core/candidate-validation/src/lib.rs index dce589e4c708..59c4aa562e74 100644 --- a/node/core/candidate-validation/src/lib.rs +++ b/node/core/candidate-validation/src/lib.rs @@ -691,6 +691,9 @@ trait ValidationBackend { /// Tries executing a PVF. Will retry once if an error is encountered that may have been /// transient. + /// + /// NOTE: Should retry only on errors that are a result of execution itself, and not of + /// preparation. async fn validate_candidate_with_retry( &mut self, raw_validation_code: Vec, @@ -698,31 +701,44 @@ trait ValidationBackend { params: ValidationParams, executor_params: ExecutorParams, ) -> Result { - // Construct the PVF a single time, since it is an expensive operation. Cloning it is cheap. let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepTimeoutKind::Lenient); + // Construct the PVF a single time, since it is an expensive operation. Cloning it is cheap. let pvf = PvfPrepData::from_code(raw_validation_code, executor_params, prep_timeout); let mut validation_result = self.validate_candidate(pvf.clone(), exec_timeout, params.encode()).await; - // If we get an AmbiguousWorkerDeath error, retry once after a brief delay, on the - // assumption that the conditions that caused this error may have been transient. Note that - // this error is only a result of execution itself and not of preparation. - if let Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)) = - validation_result - { - // Wait a brief delay before retrying. - futures_timer::Delay::new(PVF_EXECUTION_RETRY_DELAY).await; + // Allow limited retries for each kind of error. + let mut num_internal_retries_left = 1; + let mut num_awd_retries_left = 1; + loop { + match validation_result { + Err(ValidationError::InvalidCandidate( + WasmInvalidCandidate::AmbiguousWorkerDeath, + )) if num_awd_retries_left > 0 => num_awd_retries_left -= 1, + Err(ValidationError::InternalError(_)) if num_internal_retries_left > 0 => + num_internal_retries_left -= 1, + _ => break, + } + + // If we got a possibly transient error, retry once after a brief delay, on the assumption + // that the conditions that caused this error may have resolved on their own. + { + // Wait a brief delay before retrying. + futures_timer::Delay::new(PVF_EXECUTION_RETRY_DELAY).await; - gum::warn!( - target: LOG_TARGET, - ?pvf, - "Re-trying failed candidate validation due to AmbiguousWorkerDeath." - ); + gum::warn!( + target: LOG_TARGET, + ?pvf, + "Re-trying failed candidate validation due to possible transient error: {:?}", + validation_result + ); - // Encode the params again when re-trying. We expect the retry case to be relatively - // rare, and we want to avoid unconditionally cloning data. - validation_result = self.validate_candidate(pvf, exec_timeout, params.encode()).await; + // Encode the params again when re-trying. We expect the retry case to be relatively + // rare, and we want to avoid unconditionally cloning data. + validation_result = + self.validate_candidate(pvf.clone(), exec_timeout, params.encode()).await; + } } validation_result diff --git a/node/core/candidate-validation/src/tests.rs b/node/core/candidate-validation/src/tests.rs index 4f0ebf037e3a..5d1cc75b7437 100644 --- a/node/core/candidate-validation/src/tests.rs +++ b/node/core/candidate-validation/src/tests.rs @@ -672,6 +672,62 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { assert_matches!(v, ValidationResult::Invalid(InvalidCandidate::ExecutionError(_))); } +// Test that we retry on internal errors. +#[test] +fn candidate_validation_retry_internal_errors() { + let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; + + let pov = PoV { block_data: BlockData(vec![1; 32]) }; + let validation_code = ValidationCode(vec![2; 16]); + + let descriptor = make_valid_candidate_descriptor( + ParaId::from(1_u32), + dummy_hash(), + validation_data.hash(), + pov.hash(), + validation_code.hash(), + dummy_hash(), + dummy_hash(), + Sr25519Keyring::Alice, + ); + + let check = perform_basic_checks( + &descriptor, + validation_data.max_pov_size, + &pov, + &validation_code.hash(), + ); + assert!(check.is_ok()); + + let candidate_receipt = CandidateReceipt { descriptor, commitments_hash: Hash::zero() }; + + let pool = TaskExecutor::new(); + let (mut ctx, ctx_handle) = + test_helpers::make_subsystem_context::(pool.clone()); + let metrics = Metrics::default(); + + let v = test_with_executor_params(ctx_handle, || { + validate_candidate_exhaustive( + ctx.sender(), + MockValidateCandidateBackend::with_hardcoded_result_list(vec![ + Err(ValidationError::InternalError("foo".into())), + // Throw an AWD error, we should still retry again. + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + // Throw another internal error. + Err(ValidationError::InternalError("bar".into())), + ]), + validation_data, + validation_code, + candidate_receipt, + Arc::new(pov), + PvfExecTimeoutKind::Backing, + &metrics, + ) + }); + + assert_matches!(v, Err(ValidationFailed(s)) if s == "bar".to_string()); +} + #[test] fn candidate_validation_timeout_is_internal_error() { let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; diff --git a/node/core/pvf/Cargo.toml b/node/core/pvf/Cargo.toml index 6478edc44115..026930758b86 100644 --- a/node/core/pvf/Cargo.toml +++ b/node/core/pvf/Cargo.toml @@ -4,24 +4,15 @@ version.workspace = true authors.workspace = true edition.workspace = true -[[bin]] -name = "puppet_worker" -path = "bin/puppet_worker.rs" - [dependencies] always-assert = "0.1" -assert_matches = "1.4.0" -cpu-time = "1.0.0" futures = "0.3.21" futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../../gum" } libc = "0.2.139" pin-project = "1.0.9" rand = "0.8.5" -rayon = "1.5.1" slotmap = "1.0" -tempfile = "3.3.0" -tikv-jemalloc-ctl = { version = "0.5.0", optional = true } tokio = { version = "1.24.2", features = ["fs", "process"] } parity-scale-codec = { version = "3.4.0", default-features = false, features = ["derive"] } @@ -30,13 +21,8 @@ polkadot-parachain = { path = "../../../parachain" } polkadot-core-primitives = { path = "../../../core-primitives" } polkadot-node-metrics = { path = "../../metrics" } polkadot-node-primitives = { path = "../../primitives" } - polkadot-primitives = { path = "../../../primitives" } -sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master" } -sc-executor-wasmtime = { git = "https://github.com/paritytech/substrate", branch = "master" } -sc-executor-common = { git = "https://github.com/paritytech/substrate", branch = "master" } -sp-externalities = { git = "https://github.com/paritytech/substrate", branch = "master" } -sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" } + sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-wasm-interface = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-maybe-compressed-blob = { git = "https://github.com/paritytech/substrate", branch = "master" } @@ -45,14 +31,7 @@ sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master [build-dependencies] substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } -[target.'cfg(target_os = "linux")'.dependencies] -tikv-jemalloc-ctl = "0.5.0" - [dev-dependencies] -adder = { package = "test-parachain-adder", path = "../../../parachain/test-parachains/adder" } -halt = { package = "test-parachain-halt", path = "../../../parachain/test-parachains/halt" } +assert_matches = "1.4.0" hex-literal = "0.3.4" tempfile = "3.3.0" - -[features] -jemalloc-allocator = ["dep:tikv-jemalloc-ctl"] diff --git a/node/core/pvf/src/artifacts.rs b/node/core/pvf/src/artifacts.rs index 0c6146ec8a82..d5a660cc3aa5 100644 --- a/node/core/pvf/src/artifacts.rs +++ b/node/core/pvf/src/artifacts.rs @@ -14,6 +14,47 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +//! PVF artifacts (final compiled code blobs). +//! +//! # Lifecycle of an artifact +//! +//! 1. During node start-up, the artifacts cache is cleaned up. This means that all local artifacts +//! stored on-disk are cleared, and we start with an empty [`Artifacts`] table. +//! +//! 2. In order to be executed, a PVF should be prepared first. This means that artifacts should +//! have an [`ArtifactState::Prepared`] entry for that artifact in the table. If not, the +//! preparation process kicks in. The execution request is stashed until after the preparation is +//! done, and the artifact state in the host is set to [`ArtifactState::Preparing`]. Preparation +//! goes through the preparation queue and the pool. +//! +//! 1. If the artifact is already being processed, we add another execution request to the +//! existing preparation job, without starting a new one. +//! +//! 2. Note that if the state is [`ArtifactState::FailedToProcess`], we usually do not retry +//! preparation, though we may under certain conditions. +//! +//! 3. The pool gets an available worker and instructs it to work on the given PVF. The worker +//! starts compilation. When the worker finishes successfully, it writes the serialized artifact +//! into a temporary file and notifies the host that it's done. The host atomically moves +//! (renames) the temporary file to the destination filename of the artifact. +//! +//! 4. If the worker concluded successfully or returned an error, then the pool notifies the queue. +//! In both cases, the queue reports to the host that the result is ready. +//! +//! 5. The host will react by changing the artifact state to either [`ArtifactState::Prepared`] or +//! [`ArtifactState::FailedToProcess`] for the PVF in question. On success, the +//! `last_time_needed` will be set to the current time. It will also dispatch the pending +//! execution requests. +//! +//! 6. On success, the execution request will come through the execution queue and ultimately be +//! processed by an execution worker. When this worker receives the request, it will read the +//! requested artifact. If it doesn't exist it reports an internal error. A request for execution +//! will bump the `last_time_needed` to the current time. +//! +//! 7. There is a separate process for pruning the prepared artifacts whose `last_time_needed` is +//! older by a predefined parameter. This process is run very rarely (say, once a day). Once the +//! artifact is expired it is removed from disk eagerly atomically. + use crate::{error::PrepareError, host::PrepareResultSender, prepare::PrepareStats}; use always_assert::always; use polkadot_parachain::primitives::ValidationCodeHash; @@ -24,9 +65,11 @@ use std::{ time::{Duration, SystemTime}, }; +/// Contains the bytes for a successfully compiled artifact. pub struct CompiledArtifact(Vec); impl CompiledArtifact { + /// Creates a `CompiledArtifact`. pub fn new(code: Vec) -> Self { Self(code) } diff --git a/node/core/pvf/src/error.rs b/node/core/pvf/src/error.rs index 662fcc22cd31..21f23d515fdd 100644 --- a/node/core/pvf/src/error.rs +++ b/node/core/pvf/src/error.rs @@ -16,7 +16,7 @@ use crate::prepare::PrepareStats; use parity_scale_codec::{Decode, Encode}; -use std::{any::Any, fmt}; +use std::fmt; /// Result of PVF preparation performed by the validation host. Contains stats about the preparation if /// successful @@ -126,17 +126,3 @@ impl From for ValidationError { } } } - -/// Attempt to convert an opaque panic payload to a string. -/// -/// This is a best effort, and is not guaranteed to provide the most accurate value. -pub(crate) fn stringify_panic_payload(payload: Box) -> String { - match payload.downcast::<&'static str>() { - Ok(msg) => msg.to_string(), - Err(payload) => match payload.downcast::() { - Ok(msg) => *msg, - // At least we tried... - Err(_) => "unknown panic payload".to_string(), - }, - } -} diff --git a/node/core/pvf/src/execute/mod.rs b/node/core/pvf/src/execute/mod.rs index e863b4e24e54..8e3b17d71569 100644 --- a/node/core/pvf/src/execute/mod.rs +++ b/node/core/pvf/src/execute/mod.rs @@ -18,10 +18,10 @@ //! //! The validation host [runs the queue][`start`] communicating with it by sending [`ToQueue`] //! messages. The queue will spawn workers in new processes. Those processes should jump to -//! [`worker_entrypoint`]. +//! `polkadot_node_core_pvf_worker::execute_worker_entrypoint`. mod queue; -mod worker; +mod worker_intf; -pub use queue::{start, ToQueue}; -pub use worker::{worker_entrypoint, Response as ExecuteResponse}; +pub use queue::{start, PendingExecutionRequest, ToQueue}; +pub use worker_intf::{Handshake as ExecuteHandshake, Response as ExecuteResponse}; diff --git a/node/core/pvf/src/execute/queue.rs b/node/core/pvf/src/execute/queue.rs index 3e977cfa4c5d..5b3e21cee079 100644 --- a/node/core/pvf/src/execute/queue.rs +++ b/node/core/pvf/src/execute/queue.rs @@ -16,7 +16,7 @@ //! A queue that handles requests for PVF execution. -use super::worker::Outcome; +use super::worker_intf::Outcome; use crate::{ artifacts::{ArtifactId, ArtifactPathId}, host::ResultSender, @@ -50,13 +50,17 @@ slotmap::new_key_type! { struct Worker; } #[derive(Debug)] pub enum ToQueue { - Enqueue { - artifact: ArtifactPathId, - exec_timeout: Duration, - params: Vec, - executor_params: ExecutorParams, - result_tx: ResultSender, - }, + Enqueue { artifact: ArtifactPathId, pending_execution_request: PendingExecutionRequest }, +} + +/// An execution request that should execute the PVF (known in the context) and send the results +/// to the given result sender. +#[derive(Debug)] +pub struct PendingExecutionRequest { + pub exec_timeout: Duration, + pub params: Vec, + pub executor_params: ExecutorParams, + pub result_tx: ResultSender, } struct ExecuteJob { @@ -259,7 +263,9 @@ async fn purge_dead(metrics: &Metrics, workers: &mut Workers) { } fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) { - let ToQueue::Enqueue { artifact, exec_timeout, params, executor_params, result_tx } = to_queue; + let ToQueue::Enqueue { artifact, pending_execution_request } = to_queue; + let PendingExecutionRequest { exec_timeout, params, executor_params, result_tx } = + pending_execution_request; gum::debug!( target: LOG_TARGET, validation_code_hash = ?artifact.id.code_hash, @@ -410,7 +416,8 @@ async fn spawn_worker_task( use futures_timer::Delay; loop { - match super::worker::spawn(&program_path, job.executor_params.clone(), spawn_timeout).await + match super::worker_intf::spawn(&program_path, job.executor_params.clone(), spawn_timeout) + .await { Ok((idle, handle)) => break QueueEvent::Spawn(idle, handle, job), Err(err) => { @@ -454,9 +461,13 @@ fn assign(queue: &mut Queue, worker: Worker, job: ExecuteJob) { queue.mux.push( async move { let _timer = execution_timer; - let outcome = - super::worker::start_work(idle, job.artifact.clone(), job.exec_timeout, job.params) - .await; + let outcome = super::worker_intf::start_work( + idle, + job.artifact.clone(), + job.exec_timeout, + job.params, + ) + .await; QueueEvent::StartWork(worker, outcome, job.artifact.id, job.result_tx) } .boxed(), diff --git a/node/core/pvf/src/execute/worker.rs b/node/core/pvf/src/execute/worker_intf.rs similarity index 56% rename from node/core/pvf/src/execute/worker.rs rename to node/core/pvf/src/execute/worker_intf.rs index c30ebceae693..bc467cf90de6 100644 --- a/node/core/pvf/src/execute/worker.rs +++ b/node/core/pvf/src/execute/worker_intf.rs @@ -14,28 +14,23 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +//! Host interface to the execute worker. + use crate::{ artifacts::ArtifactPathId, - executor_intf::Executor, worker_common::{ - bytes_to_path, cpu_time_monitor_loop, framed_recv, framed_send, path_to_bytes, - spawn_with_program_path, worker_event_loop, IdleWorker, SpawnErr, WorkerHandle, - JOB_TIMEOUT_WALL_CLOCK_FACTOR, + framed_recv, framed_send, path_to_bytes, spawn_with_program_path, IdleWorker, SpawnErr, + WorkerHandle, JOB_TIMEOUT_WALL_CLOCK_FACTOR, }, LOG_TARGET, }; -use cpu_time::ProcessTime; -use futures::{pin_mut, select_biased, FutureExt}; +use futures::FutureExt; use futures_timer::Delay; use parity_scale_codec::{Decode, Encode}; use polkadot_parachain::primitives::ValidationResult; use polkadot_primitives::ExecutorParams; -use std::{ - path::{Path, PathBuf}, - sync::{mpsc::channel, Arc}, - time::Duration, -}; +use std::{path::Path, time::Duration}; use tokio::{io, net::UnixStream}; /// Spawns a new worker with the given program path that acts as the worker and the spawn timeout. @@ -185,17 +180,6 @@ async fn send_handshake(stream: &mut UnixStream, handshake: Handshake) -> io::Re framed_send(stream, &handshake.encode()).await } -async fn recv_handshake(stream: &mut UnixStream) -> io::Result { - let handshake_enc = framed_recv(stream).await?; - let handshake = Handshake::decode(&mut &handshake_enc[..]).map_err(|_| { - io::Error::new( - io::ErrorKind::Other, - "execute pvf recv_handshake: failed to decode Handshake".to_owned(), - ) - })?; - Ok(handshake) -} - async fn send_request( stream: &mut UnixStream, artifact_path: &Path, @@ -207,29 +191,6 @@ async fn send_request( framed_send(stream, &execution_timeout.encode()).await } -async fn recv_request(stream: &mut UnixStream) -> io::Result<(PathBuf, Vec, Duration)> { - let artifact_path = framed_recv(stream).await?; - let artifact_path = bytes_to_path(&artifact_path).ok_or_else(|| { - io::Error::new( - io::ErrorKind::Other, - "execute pvf recv_request: non utf-8 artifact path".to_string(), - ) - })?; - let params = framed_recv(stream).await?; - let execution_timeout = framed_recv(stream).await?; - let execution_timeout = Duration::decode(&mut &execution_timeout[..]).map_err(|_| { - io::Error::new( - io::ErrorKind::Other, - "execute pvf recv_request: failed to decode duration".to_string(), - ) - })?; - Ok((artifact_path, params, execution_timeout)) -} - -async fn send_response(stream: &mut UnixStream, response: Response) -> io::Result<()> { - framed_send(stream, &response.encode()).await -} - async fn recv_response(stream: &mut UnixStream) -> io::Result { let response_bytes = framed_recv(stream).await?; Response::decode(&mut &response_bytes[..]).map_err(|e| { @@ -240,28 +201,43 @@ async fn recv_response(stream: &mut UnixStream) -> io::Result { }) } +/// The payload of the one-time handshake that is done when a worker process is created. Carries +/// data from the host to the worker. #[derive(Encode, Decode)] -struct Handshake { - executor_params: ExecutorParams, +pub struct Handshake { + /// The executor parameters. + pub executor_params: ExecutorParams, } +/// The response from an execution job on the worker. #[derive(Encode, Decode)] pub enum Response { - Ok { result_descriptor: ValidationResult, duration: Duration }, + /// The job completed successfully. + Ok { + /// The result of parachain validation. + result_descriptor: ValidationResult, + /// The amount of CPU time taken by the job. + duration: Duration, + }, + /// The candidate is invalid. InvalidCandidate(String), + /// The job timed out. TimedOut, + /// Some internal error occurred. Should only be used for errors independent of the candidate. InternalError(String), } impl Response { - fn format_invalid(ctx: &'static str, msg: &str) -> Self { + /// Creates an invalid response from a context `ctx` and a message `msg` (which can be empty). + pub fn format_invalid(ctx: &'static str, msg: &str) -> Self { if msg.is_empty() { Self::InvalidCandidate(ctx.to_string()) } else { Self::InvalidCandidate(format!("{}: {}", ctx, msg)) } } - fn format_internal(ctx: &'static str, msg: &str) -> Self { + /// Creates an internal response from a context `ctx` and a message `msg` (which can be empty). + pub fn format_internal(ctx: &'static str, msg: &str) -> Self { if msg.is_empty() { Self::InternalError(ctx.to_string()) } else { @@ -269,103 +245,3 @@ impl Response { } } } - -/// The entrypoint that the spawned execute worker should start with. The `socket_path` specifies -/// the path to the socket used to communicate with the host. The `node_version`, if `Some`, -/// is checked against the worker version. A mismatch results in immediate worker termination. -/// `None` is used for tests and in other situations when version check is not necessary. -pub fn worker_entrypoint(socket_path: &str, node_version: Option<&str>) { - worker_event_loop("execute", socket_path, node_version, |rt_handle, mut stream| async move { - let worker_pid = std::process::id(); - - let handshake = recv_handshake(&mut stream).await?; - let executor = Arc::new(Executor::new(handshake.executor_params).map_err(|e| { - io::Error::new(io::ErrorKind::Other, format!("cannot create executor: {}", e)) - })?); - - loop { - let (artifact_path, params, execution_timeout) = recv_request(&mut stream).await?; - gum::debug!( - target: LOG_TARGET, - %worker_pid, - "worker: validating artifact {}", - artifact_path.display(), - ); - - // Used to signal to the cpu time monitor thread that it can finish. - let (finished_tx, finished_rx) = channel::<()>(); - let cpu_time_start = ProcessTime::now(); - - // Spawn a new thread that runs the CPU time monitor. - let cpu_time_monitor_fut = rt_handle - .spawn_blocking(move || { - cpu_time_monitor_loop(cpu_time_start, execution_timeout, finished_rx) - }) - .fuse(); - let executor_2 = executor.clone(); - let execute_fut = rt_handle - .spawn_blocking(move || { - validate_using_artifact(&artifact_path, ¶ms, executor_2, cpu_time_start) - }) - .fuse(); - - pin_mut!(cpu_time_monitor_fut); - pin_mut!(execute_fut); - - let response = select_biased! { - // If this future is not selected, the join handle is dropped and the thread will - // finish in the background. - cpu_time_monitor_res = cpu_time_monitor_fut => { - match cpu_time_monitor_res { - Ok(Some(cpu_time_elapsed)) => { - // Log if we exceed the timeout and the other thread hasn't finished. - gum::warn!( - target: LOG_TARGET, - %worker_pid, - "execute job took {}ms cpu time, exceeded execute timeout {}ms", - cpu_time_elapsed.as_millis(), - execution_timeout.as_millis(), - ); - Response::TimedOut - }, - Ok(None) => Response::InternalError("error communicating over finished channel".into()), - Err(e) => Response::format_internal("cpu time monitor thread error", &e.to_string()), - } - }, - execute_res = execute_fut => { - let _ = finished_tx.send(()); - execute_res.unwrap_or_else(|e| Response::format_internal("execute thread error", &e.to_string())) - }, - }; - - send_response(&mut stream, response).await?; - } - }); -} - -fn validate_using_artifact( - artifact_path: &Path, - params: &[u8], - executor: Arc, - cpu_time_start: ProcessTime, -) -> Response { - let descriptor_bytes = match unsafe { - // SAFETY: this should be safe since the compiled artifact passed here comes from the - // file created by the prepare workers. These files are obtained by calling - // [`executor_intf::prepare`]. - executor.execute(artifact_path.as_ref(), params) - } { - Err(err) => return Response::format_invalid("execute", &err), - Ok(d) => d, - }; - - let duration = cpu_time_start.elapsed(); - - let result_descriptor = match ValidationResult::decode(&mut &descriptor_bytes[..]) { - Err(err) => - return Response::format_invalid("validation result decoding failed", &err.to_string()), - Ok(r) => r, - }; - - Response::Ok { result_descriptor, duration } -} diff --git a/node/core/pvf/src/host.rs b/node/core/pvf/src/host.rs index 31a0266145f8..bfc775a32dee 100644 --- a/node/core/pvf/src/host.rs +++ b/node/core/pvf/src/host.rs @@ -23,7 +23,7 @@ use crate::{ artifacts::{ArtifactId, ArtifactPathId, ArtifactState, Artifacts}, error::PrepareError, - execute, + execute::{self, PendingExecutionRequest}, metrics::Metrics, prepare, PrepareResult, Priority, PvfPrepData, ValidationError, LOG_TARGET, }; @@ -33,7 +33,6 @@ use futures::{ Future, FutureExt, SinkExt, StreamExt, }; use polkadot_parachain::primitives::ValidationResult; -use polkadot_primitives::ExecutorParams; use std::{ collections::HashMap, path::{Path, PathBuf}, @@ -249,36 +248,14 @@ pub fn start(config: Config, metrics: Metrics) -> (ValidationHost, impl Future, - executor_params: ExecutorParams, - result_tx: ResultSender, -} - /// A mapping from an artifact ID which is in preparation state to the list of pending execution /// requests that should be executed once the artifact's preparation is finished. #[derive(Default)] struct AwaitingPrepare(HashMap>); impl AwaitingPrepare { - fn add( - &mut self, - artifact_id: ArtifactId, - exec_timeout: Duration, - params: Vec, - executor_params: ExecutorParams, - result_tx: ResultSender, - ) { - self.0.entry(artifact_id).or_default().push(PendingExecutionRequest { - exec_timeout, - params, - executor_params, - result_tx, - }); + fn add(&mut self, artifact_id: ArtifactId, pending_execution_request: PendingExecutionRequest) { + self.0.entry(artifact_id).or_default().push(pending_execution_request); } fn take(&mut self, artifact_id: &ArtifactId) -> Vec { @@ -475,9 +452,11 @@ async fn handle_precheck_pvf( /// This will try to prepare the PVF, if a prepared artifact does not already exist. If there is already a /// preparation job, we coalesce the two preparation jobs. /// +/// If the prepare job succeeded previously, we will enqueue an execute job right away. +/// /// If the prepare job failed previously, we may retry it under certain conditions. /// -/// When preparing for execution, we use a more lenient timeout ([`EXECUTE_PREPARATION_TIMEOUT`]) +/// When preparing for execution, we use a more lenient timeout ([`LENIENT_PREPARATION_TIMEOUT`]) /// than when prechecking. async fn handle_execute_pvf( cache_path: &Path, @@ -489,32 +468,63 @@ async fn handle_execute_pvf( ) -> Result<(), Fatal> { let ExecutePvfInputs { pvf, exec_timeout, params, priority, result_tx } = inputs; let artifact_id = pvf.as_artifact_id(); + let executor_params = (*pvf.executor_params()).clone(); if let Some(state) = artifacts.artifact_state_mut(&artifact_id) { match state { ArtifactState::Prepared { last_time_needed, .. } => { - *last_time_needed = SystemTime::now(); + let file_metadata = std::fs::metadata(artifact_id.path(cache_path)); + + if file_metadata.is_ok() { + *last_time_needed = SystemTime::now(); + + // This artifact has already been prepared, send it to the execute queue. + send_execute( + execute_queue, + execute::ToQueue::Enqueue { + artifact: ArtifactPathId::new(artifact_id, cache_path), + pending_execution_request: PendingExecutionRequest { + exec_timeout, + params, + executor_params, + result_tx, + }, + }, + ) + .await?; + } else { + gum::warn!( + target: LOG_TARGET, + ?pvf, + ?artifact_id, + "handle_execute_pvf: Re-queuing PVF preparation for prepared artifact with missing file." + ); - // This artifact has already been prepared, send it to the execute queue. - send_execute( - execute_queue, - execute::ToQueue::Enqueue { - artifact: ArtifactPathId::new(artifact_id, cache_path), - exec_timeout, - params, - executor_params: (*pvf.executor_params()).clone(), - result_tx, - }, - ) - .await?; + // The artifact has been prepared previously but the file is missing, prepare it again. + *state = ArtifactState::Preparing { + waiting_for_response: Vec::new(), + num_failures: 0, + }; + enqueue_prepare_for_execute( + prepare_queue, + awaiting_prepare, + pvf, + priority, + artifact_id, + PendingExecutionRequest { + exec_timeout, + params, + executor_params, + result_tx, + }, + ) + .await?; + } }, ArtifactState::Preparing { .. } => { awaiting_prepare.add( artifact_id, - exec_timeout, - params, - (*pvf.executor_params()).clone(), - result_tx, + PendingExecutionRequest { exec_timeout, params, executor_params, result_tx }, ); }, ArtifactState::FailedToProcess { last_time_failed, num_failures, error } => { @@ -535,19 +545,20 @@ async fn handle_execute_pvf( waiting_for_response: Vec::new(), num_failures: *num_failures, }; - let executor_params = (*pvf.executor_params()).clone(); - send_prepare(prepare_queue, prepare::ToQueue::Enqueue { priority, pvf }) - .await?; - - // Add an execution request that will wait to run after this prepare job has - // finished. - awaiting_prepare.add( + enqueue_prepare_for_execute( + prepare_queue, + awaiting_prepare, + pvf, + priority, artifact_id, - exec_timeout, - params, - executor_params, - result_tx, - ); + PendingExecutionRequest { + exec_timeout, + params, + executor_params, + result_tx, + }, + ) + .await?; } else { let _ = result_tx.send(Err(ValidationError::from(error.clone()))); } @@ -556,12 +567,16 @@ async fn handle_execute_pvf( } else { // Artifact is unknown: register it and enqueue a job with the corresponding priority and // PVF. - let executor_params = (*pvf.executor_params()).clone(); artifacts.insert_preparing(artifact_id.clone(), Vec::new()); - send_prepare(prepare_queue, prepare::ToQueue::Enqueue { priority, pvf }).await?; - - // Add an execution request that will wait to run after this prepare job has finished. - awaiting_prepare.add(artifact_id, exec_timeout, params, executor_params, result_tx); + enqueue_prepare_for_execute( + prepare_queue, + awaiting_prepare, + pvf, + priority, + artifact_id, + PendingExecutionRequest { exec_timeout, params, executor_params, result_tx }, + ) + .await?; } Ok(()) @@ -700,10 +715,12 @@ async fn handle_prepare_done( execute_queue, execute::ToQueue::Enqueue { artifact: ArtifactPathId::new(artifact_id.clone(), cache_path), - exec_timeout, - params, - executor_params, - result_tx, + pending_execution_request: PendingExecutionRequest { + exec_timeout, + params, + executor_params, + result_tx, + }, }, ) .await?; @@ -745,6 +762,24 @@ async fn send_execute( execute_queue.send(to_queue).await.map_err(|_| Fatal) } +/// Sends a job to the preparation queue, and adds an execution request that will wait to run after +/// this prepare job has finished. +async fn enqueue_prepare_for_execute( + prepare_queue: &mut mpsc::Sender, + awaiting_prepare: &mut AwaitingPrepare, + pvf: PvfPrepData, + priority: Priority, + artifact_id: ArtifactId, + pending_execution_request: PendingExecutionRequest, +) -> Result<(), Fatal> { + send_prepare(prepare_queue, prepare::ToQueue::Enqueue { priority, pvf }).await?; + + // Add an execution request that will wait to run after this prepare job has finished. + awaiting_prepare.add(artifact_id, pending_execution_request); + + Ok(()) +} + async fn handle_cleanup_pulse( cache_path: &Path, sweeper_tx: &mut mpsc::Sender, @@ -1125,11 +1160,11 @@ pub(crate) mod tests { .unwrap(); let result_tx_pvf_1_1 = assert_matches!( test.poll_and_recv_to_execute_queue().await, - execute::ToQueue::Enqueue { result_tx, .. } => result_tx + execute::ToQueue::Enqueue { pending_execution_request: PendingExecutionRequest { result_tx, .. }, .. } => result_tx ); let result_tx_pvf_1_2 = assert_matches!( test.poll_and_recv_to_execute_queue().await, - execute::ToQueue::Enqueue { result_tx, .. } => result_tx + execute::ToQueue::Enqueue { pending_execution_request: PendingExecutionRequest { result_tx, .. }, .. } => result_tx ); test.from_prepare_queue_tx @@ -1141,7 +1176,7 @@ pub(crate) mod tests { .unwrap(); let result_tx_pvf_2 = assert_matches!( test.poll_and_recv_to_execute_queue().await, - execute::ToQueue::Enqueue { result_tx, .. } => result_tx + execute::ToQueue::Enqueue { pending_execution_request: PendingExecutionRequest { result_tx, .. }, .. } => result_tx ); result_tx_pvf_1_1 @@ -1456,7 +1491,7 @@ pub(crate) mod tests { // Preparation should have been retried and succeeded this time. let result_tx_3 = assert_matches!( test.poll_and_recv_to_execute_queue().await, - execute::ToQueue::Enqueue { result_tx, .. } => result_tx + execute::ToQueue::Enqueue { pending_execution_request: PendingExecutionRequest { result_tx, .. }, .. } => result_tx ); // Send an error for the execution here, just so we can check the result receiver is still diff --git a/node/core/pvf/src/lib.rs b/node/core/pvf/src/lib.rs index 8b6f2ddb262c..cdaee3341402 100644 --- a/node/core/pvf/src/lib.rs +++ b/node/core/pvf/src/lib.rs @@ -29,11 +29,11 @@ //! //! Then using the handle the client can send three types of requests: //! -//! (a) PVF pre-checking. This takes the PVF [code][`Pvf`] and tries to prepare it (verify and +//! (a) PVF pre-checking. This takes the `Pvf` code and tries to prepare it (verify and //! compile) in order to pre-check its validity. //! //! (b) PVF execution. This accepts the PVF [`params`][`polkadot_parachain::primitives::ValidationParams`] -//! and the PVF [code][`Pvf`], prepares (verifies and compiles) the code, and then executes PVF +//! and the `Pvf` code, prepares (verifies and compiles) the code, and then executes PVF //! with the `params`. //! //! (c) Heads up. This request allows to signal that the given PVF may be needed soon and that it @@ -72,14 +72,12 @@ //! ## Artifacts //! //! An artifact is the final product of preparation. If the preparation succeeded, then the artifact -//! will contain the compiled code usable for quick execution by a worker later on. -//! -//! If the preparation failed, then the worker will still write the artifact with the error message. -//! We save the artifact with the error so that we don't try to prepare the artifacts that are broken -//! repeatedly. +//! will contain the compiled code usable for quick execution by a worker later on. If the +//! preparation failed, then no artifact is created. //! //! The artifact is saved on disk and is also tracked by an in memory table. This in memory table -//! doesn't contain the artifact contents though, only a flag that the given artifact is compiled. +//! doesn't contain the artifact contents though, only a flag for the state of the given artifact +//! and some associated data. If the artifact failed to process, this also includes the error. //! //! A pruning task will run at a fixed interval of time. This task will remove all artifacts that //! weren't used or received a heads up signal for a while. @@ -93,7 +91,6 @@ mod artifacts; mod error; mod execute; -mod executor_intf; mod host; mod metrics; mod prepare; @@ -101,27 +98,22 @@ mod priority; mod pvf; mod worker_common; -#[doc(hidden)] -pub mod testing; - -#[doc(hidden)] -pub use sp_tracing; - +pub use artifacts::CompiledArtifact; pub use error::{InvalidCandidate, PrepareError, PrepareResult, ValidationError}; -pub use prepare::PrepareStats; +pub use execute::{ExecuteHandshake, ExecuteResponse}; +#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] +pub use prepare::MemoryAllocationStats; +pub use prepare::{MemoryStats, PrepareStats}; pub use priority::Priority; pub use pvf::PvfPrepData; pub use host::{start, Config, ValidationHost}; pub use metrics::Metrics; -pub use worker_common::JOB_TIMEOUT_WALL_CLOCK_FACTOR; - -pub use execute::worker_entrypoint as execute_worker_entrypoint; -pub use prepare::worker_entrypoint as prepare_worker_entrypoint; - -pub use executor_intf::{prepare, prevalidate}; - -pub use sc_executor_common; -pub use sp_maybe_compressed_blob; +pub use worker_common::{framed_recv, framed_send, JOB_TIMEOUT_WALL_CLOCK_FACTOR}; const LOG_TARGET: &str = "parachain::pvf"; + +#[doc(hidden)] +pub mod testing { + pub use crate::worker_common::{spawn_with_program_path, SpawnErr}; +} diff --git a/node/core/pvf/src/prepare/mod.rs b/node/core/pvf/src/prepare/mod.rs index d8d036a82238..de40c48464c4 100644 --- a/node/core/pvf/src/prepare/mod.rs +++ b/node/core/pvf/src/prepare/mod.rs @@ -20,23 +20,44 @@ //! (by running [`start_pool`]). //! //! The pool will spawn workers in new processes and those should execute pass control to -//! [`worker_entrypoint`]. +//! `polkadot_node_core_pvf_worker::prepare_worker_entrypoint`. -mod memory_stats; mod pool; mod queue; -mod worker; +mod worker_intf; -pub use memory_stats::MemoryStats; pub use pool::start as start_pool; pub use queue::{start as start_queue, FromQueue, ToQueue}; -pub use worker::worker_entrypoint; use parity_scale_codec::{Decode, Encode}; /// Preparation statistics, including the CPU time and memory taken. #[derive(Debug, Clone, Default, Encode, Decode)] pub struct PrepareStats { - cpu_time_elapsed: std::time::Duration, - memory_stats: MemoryStats, + /// The CPU time that elapsed for the preparation job. + pub cpu_time_elapsed: std::time::Duration, + /// The observed memory statistics for the preparation job. + pub memory_stats: MemoryStats, +} + +/// Helper struct to contain all the memory stats, including `MemoryAllocationStats` and, if +/// supported by the OS, `ru_maxrss`. +#[derive(Clone, Debug, Default, Encode, Decode)] +pub struct MemoryStats { + /// Memory stats from `tikv_jemalloc_ctl`. + #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] + pub memory_tracker_stats: Option, + /// `ru_maxrss` from `getrusage`. `None` if an error occurred. + #[cfg(target_os = "linux")] + pub max_rss: Option, +} + +/// Statistics of collected memory metrics. +#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] +#[derive(Clone, Debug, Default, Encode, Decode)] +pub struct MemoryAllocationStats { + /// Total resident memory, in bytes. + pub resident: u64, + /// Total allocated memory, in bytes. + pub allocated: u64, } diff --git a/node/core/pvf/src/prepare/pool.rs b/node/core/pvf/src/prepare/pool.rs index f8435a40348d..d151f097805e 100644 --- a/node/core/pvf/src/prepare/pool.rs +++ b/node/core/pvf/src/prepare/pool.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use super::worker::{self, Outcome}; +use super::worker_intf::{self, Outcome}; use crate::{ error::{PrepareError, PrepareResult}, metrics::Metrics, @@ -250,7 +250,7 @@ async fn spawn_worker_task(program_path: PathBuf, spawn_timeout: Duration) -> Po use futures_timer::Delay; loop { - match worker::spawn(&program_path, spawn_timeout).await { + match worker_intf::spawn(&program_path, spawn_timeout).await { Ok((idle, handle)) => break PoolEvent::Spawn(idle, handle), Err(err) => { gum::warn!(target: LOG_TARGET, "failed to spawn a prepare worker: {:?}", err); @@ -271,7 +271,7 @@ async fn start_work_task( artifact_path: PathBuf, _preparation_timer: Option, ) -> PoolEvent { - let outcome = worker::start_work(&metrics, idle, pvf, &cache_path, artifact_path).await; + let outcome = worker_intf::start_work(&metrics, idle, pvf, &cache_path, artifact_path).await; PoolEvent::StartWork(worker, outcome) } diff --git a/node/core/pvf/src/prepare/queue.rs b/node/core/pvf/src/prepare/queue.rs index 20ee95a435b2..f84d5ab0e56e 100644 --- a/node/core/pvf/src/prepare/queue.rs +++ b/node/core/pvf/src/prepare/queue.rs @@ -226,7 +226,7 @@ async fn handle_enqueue( target: LOG_TARGET, validation_code_hash = ?pvf.code_hash(), ?priority, - preparation_timeout = ?pvf.prep_timeout, + preparation_timeout = ?pvf.prep_timeout(), "PVF is enqueued for preparation.", ); queue.metrics.prepare_enqueued(); diff --git a/node/core/pvf/src/prepare/worker.rs b/node/core/pvf/src/prepare/worker_intf.rs similarity index 55% rename from node/core/pvf/src/prepare/worker.rs rename to node/core/pvf/src/prepare/worker_intf.rs index 3b2ae211e6ca..daf94aadc672 100644 --- a/node/core/pvf/src/prepare/worker.rs +++ b/node/core/pvf/src/prepare/worker_intf.rs @@ -14,33 +14,24 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -#[cfg(target_os = "linux")] -use super::memory_stats::max_rss_stat::{extract_max_rss_stat, get_max_rss_thread}; -#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] -use super::memory_stats::memory_tracker::{get_memory_tracker_loop_stats, memory_tracker_loop}; -use super::memory_stats::MemoryStats; +//! Host interface to the prepare worker. + use crate::{ - artifacts::CompiledArtifact, error::{PrepareError, PrepareResult}, metrics::Metrics, prepare::PrepareStats, pvf::PvfPrepData, worker_common::{ - bytes_to_path, cpu_time_monitor_loop, framed_recv, framed_send, path_to_bytes, - spawn_with_program_path, tmpfile_in, worker_event_loop, IdleWorker, SpawnErr, WorkerHandle, - JOB_TIMEOUT_WALL_CLOCK_FACTOR, + framed_recv, framed_send, path_to_bytes, spawn_with_program_path, tmpfile_in, IdleWorker, + SpawnErr, WorkerHandle, JOB_TIMEOUT_WALL_CLOCK_FACTOR, }, LOG_TARGET, }; -use cpu_time::ProcessTime; -use futures::{pin_mut, select_biased, FutureExt}; use parity_scale_codec::{Decode, Encode}; use sp_core::hexdisplay::HexDisplay; use std::{ - panic, path::{Path, PathBuf}, - sync::mpsc::channel, time::Duration, }; use tokio::{io, net::UnixStream}; @@ -104,7 +95,7 @@ pub async fn start_work( ); with_tmp_file(stream, pid, cache_path, |tmp_file, mut stream| async move { - let preparation_timeout = pvf.prep_timeout; + let preparation_timeout = pvf.prep_timeout(); if let Err(err) = send_request(&mut stream, pvf, &tmp_file).await { gum::warn!( target: LOG_TARGET, @@ -285,28 +276,6 @@ async fn send_request( Ok(()) } -async fn recv_request(stream: &mut UnixStream) -> io::Result<(PvfPrepData, PathBuf)> { - let pvf = framed_recv(stream).await?; - let pvf = PvfPrepData::decode(&mut &pvf[..]).map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("prepare pvf recv_request: failed to decode PvfPrepData: {}", e), - ) - })?; - let tmp_file = framed_recv(stream).await?; - let tmp_file = bytes_to_path(&tmp_file).ok_or_else(|| { - io::Error::new( - io::ErrorKind::Other, - "prepare pvf recv_request: non utf-8 artifact path".to_string(), - ) - })?; - Ok((pvf, tmp_file)) -} - -async fn send_response(stream: &mut UnixStream, result: PrepareResult) -> io::Result<()> { - framed_send(stream, &result.encode()).await -} - async fn recv_response(stream: &mut UnixStream, pid: u32) -> io::Result { let result = framed_recv(stream).await?; let result = PrepareResult::decode(&mut &result[..]).map_err(|e| { @@ -325,158 +294,3 @@ async fn recv_response(stream: &mut UnixStream, pid: u32) -> io::Result) { - worker_event_loop("prepare", socket_path, node_version, |rt_handle, mut stream| async move { - let worker_pid = std::process::id(); - - loop { - let (pvf, dest) = recv_request(&mut stream).await?; - gum::debug!( - target: LOG_TARGET, - %worker_pid, - "worker: preparing artifact", - ); - - let cpu_time_start = ProcessTime::now(); - let preparation_timeout = pvf.prep_timeout; - - // Run the memory tracker. - #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] - let (memory_tracker_tx, memory_tracker_rx) = channel::<()>(); - #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] - let memory_tracker_fut = rt_handle.spawn_blocking(move || memory_tracker_loop(memory_tracker_rx)); - - // Spawn a new thread that runs the CPU time monitor. - let (cpu_time_monitor_tx, cpu_time_monitor_rx) = channel::<()>(); - let cpu_time_monitor_fut = rt_handle - .spawn_blocking(move || { - cpu_time_monitor_loop(cpu_time_start, preparation_timeout, cpu_time_monitor_rx) - }) - .fuse(); - // Spawn another thread for preparation. - let prepare_fut = rt_handle - .spawn_blocking(move || { - let result = prepare_artifact(pvf); - - // Get the `ru_maxrss` stat. If supported, call getrusage for the thread. - #[cfg(target_os = "linux")] - let result = result.map(|artifact| (artifact, get_max_rss_thread())); - - result - }) - .fuse(); - - pin_mut!(cpu_time_monitor_fut); - pin_mut!(prepare_fut); - - let result = select_biased! { - // If this future is not selected, the join handle is dropped and the thread will - // finish in the background. - join_res = cpu_time_monitor_fut => { - match join_res { - Ok(Some(cpu_time_elapsed)) => { - // Log if we exceed the timeout and the other thread hasn't finished. - gum::warn!( - target: LOG_TARGET, - %worker_pid, - "prepare job took {}ms cpu time, exceeded prepare timeout {}ms", - cpu_time_elapsed.as_millis(), - preparation_timeout.as_millis(), - ); - Err(PrepareError::TimedOut) - }, - Ok(None) => Err(PrepareError::IoErr("error communicating over finished channel".into())), - Err(err) => Err(PrepareError::IoErr(err.to_string())), - } - }, - prepare_res = prepare_fut => { - let cpu_time_elapsed = cpu_time_start.elapsed(); - let _ = cpu_time_monitor_tx.send(()); - - match prepare_res.unwrap_or_else(|err| Err(PrepareError::IoErr(err.to_string()))) { - Err(err) => { - // Serialized error will be written into the socket. - Err(err) - }, - Ok(ok) => { - // Stop the memory stats worker and get its observed memory stats. - #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] - let memory_tracker_stats = - get_memory_tracker_loop_stats(memory_tracker_fut, memory_tracker_tx, worker_pid).await; - #[cfg(target_os = "linux")] - let (ok, max_rss) = ok; - let memory_stats = MemoryStats { - #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] - memory_tracker_stats, - #[cfg(target_os = "linux")] - max_rss: extract_max_rss_stat(max_rss, worker_pid), - }; - - // Write the serialized artifact into a temp file. - // - // PVF host only keeps artifacts statuses in its memory, successfully - // compiled code gets stored on the disk (and consequently deserialized - // by execute-workers). The prepare worker is only required to send `Ok` - // to the pool to indicate the success. - - gum::debug!( - target: LOG_TARGET, - %worker_pid, - "worker: writing artifact to {}", - dest.display(), - ); - tokio::fs::write(&dest, &ok).await?; - - Ok(PrepareStats{cpu_time_elapsed, memory_stats}) - }, - } - }, - }; - - send_response(&mut stream, result).await?; - } - }); -} - -fn prepare_artifact(pvf: PvfPrepData) -> Result { - panic::catch_unwind(|| { - let blob = match crate::executor_intf::prevalidate(&pvf.code()) { - Err(err) => return Err(PrepareError::Prevalidation(format!("{:?}", err))), - Ok(b) => b, - }; - - match crate::executor_intf::prepare(blob, &pvf.executor_params()) { - Ok(compiled_artifact) => Ok(CompiledArtifact::new(compiled_artifact)), - Err(err) => Err(PrepareError::Preparation(format!("{:?}", err))), - } - }) - .map_err(|panic_payload| { - PrepareError::Panic(crate::error::stringify_panic_payload(panic_payload)) - }) - .and_then(|inner_result| inner_result) -} diff --git a/node/core/pvf/src/pvf.rs b/node/core/pvf/src/pvf.rs index ad2dc5fcd918..c134cacb4acf 100644 --- a/node/core/pvf/src/pvf.rs +++ b/node/core/pvf/src/pvf.rs @@ -36,13 +36,13 @@ use crate::host::tests::TEST_PREPARATION_TIMEOUT; #[derive(Clone, Encode, Decode)] pub struct PvfPrepData { /// Wasm code (uncompressed) - pub(crate) code: Arc>, + code: Arc>, /// Wasm code hash - pub(crate) code_hash: ValidationCodeHash, + code_hash: ValidationCodeHash, /// Executor environment parameters for the session for which artifact is prepared - pub(crate) executor_params: Arc, + executor_params: Arc, /// Preparation timeout - pub(crate) prep_timeout: Duration, + prep_timeout: Duration, } impl PvfPrepData { @@ -69,15 +69,20 @@ impl PvfPrepData { } /// Returns PVF code - pub(crate) fn code(&self) -> Arc> { + pub fn code(&self) -> Arc> { self.code.clone() } /// Returns executor params - pub(crate) fn executor_params(&self) -> Arc { + pub fn executor_params(&self) -> Arc { self.executor_params.clone() } + /// Returns preparation timeout. + pub fn prep_timeout(&self) -> Duration { + self.prep_timeout + } + /// Creates a structure for tests #[cfg(test)] pub(crate) fn from_discriminator_and_timeout(num: u32, timeout: Duration) -> Self { diff --git a/node/core/pvf/src/worker_common.rs b/node/core/pvf/src/worker_common.rs index 3caee34a5d0f..33144616601d 100644 --- a/node/core/pvf/src/worker_common.rs +++ b/node/core/pvf/src/worker_common.rs @@ -17,8 +17,7 @@ //! Common logic for implementation of worker processes. use crate::LOG_TARGET; -use cpu_time::ProcessTime; -use futures::{never::Never, FutureExt as _}; +use futures::FutureExt as _; use futures_timer::Delay; use pin_project::pin_project; use rand::Rng; @@ -26,7 +25,6 @@ use std::{ fmt, mem, path::{Path, PathBuf}, pin::Pin, - sync::mpsc::{Receiver, RecvTimeoutError}, task::{Context, Poll}, time::Duration, }; @@ -34,17 +32,12 @@ use tokio::{ io::{self, AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _, ReadBuf}, net::{UnixListener, UnixStream}, process, - runtime::{Handle, Runtime}, }; /// A multiple of the job timeout (in CPU time) for which we are willing to wait on the host (in /// wall clock time). This is lenient because CPU time may go slower than wall clock time. pub const JOB_TIMEOUT_WALL_CLOCK_FACTOR: u32 = 4; -/// Some allowed overhead that we account for in the "CPU time monitor" thread's sleeps, on the -/// child process. -pub const JOB_TIMEOUT_OVERHEAD: Duration = Duration::from_millis(50); - /// This is publicly exposed only for integration tests. #[doc(hidden)] pub async fn spawn_with_program_path( @@ -171,92 +164,6 @@ pub async fn tmpfile(prefix: &str) -> io::Result { tmpfile_in(prefix, &temp_dir).await } -pub fn worker_event_loop( - debug_id: &'static str, - socket_path: &str, - node_version: Option<&str>, - mut event_loop: F, -) where - F: FnMut(Handle, UnixStream) -> Fut, - Fut: futures::Future>, -{ - let worker_pid = std::process::id(); - gum::debug!(target: LOG_TARGET, %worker_pid, "starting pvf worker ({})", debug_id); - - // Check for a mismatch between the node and worker versions. - if let Some(version) = node_version { - if version != env!("SUBSTRATE_CLI_IMPL_VERSION") { - gum::error!( - target: LOG_TARGET, - %worker_pid, - "Node and worker version mismatch, node needs restarting, forcing shutdown", - ); - kill_parent_node_in_emergency(); - let err: io::Result = - Err(io::Error::new(io::ErrorKind::Unsupported, "Version mismatch")); - gum::debug!(target: LOG_TARGET, %worker_pid, "quitting pvf worker({}): {:?}", debug_id, err); - return - } - } - - // Run the main worker loop. - let rt = Runtime::new().expect("Creates tokio runtime. If this panics the worker will die and the host will detect that and deal with it."); - let handle = rt.handle(); - let err = rt - .block_on(async move { - let stream = UnixStream::connect(socket_path).await?; - let _ = tokio::fs::remove_file(socket_path).await; - - let result = event_loop(handle.clone(), stream).await; - - result - }) - // It's never `Ok` because it's `Ok(Never)`. - .unwrap_err(); - - gum::debug!(target: LOG_TARGET, %worker_pid, "quitting pvf worker ({}): {:?}", debug_id, err); - - // We don't want tokio to wait for the tasks to finish. We want to bring down the worker as fast - // as possible and not wait for stalled validation to finish. This isn't strictly necessary now, - // but may be in the future. - rt.shutdown_background(); -} - -/// Loop that runs in the CPU time monitor thread on prepare and execute jobs. Continuously wakes up -/// and then either blocks for the remaining CPU time, or returns if we exceed the CPU timeout. -/// -/// Returning `Some` indicates that we should send a `TimedOut` error to the host. Will return -/// `None` if the other thread finishes first, without us timing out. -/// -/// NOTE: Sending a `TimedOut` error to the host will cause the worker, whether preparation or -/// execution, to be killed by the host. We do not kill the process here because it would interfere -/// with the proper handling of this error. -pub fn cpu_time_monitor_loop( - cpu_time_start: ProcessTime, - timeout: Duration, - finished_rx: Receiver<()>, -) -> Option { - loop { - let cpu_time_elapsed = cpu_time_start.elapsed(); - - // Treat the timeout as CPU time, which is less subject to variance due to load. - if cpu_time_elapsed <= timeout { - // Sleep for the remaining CPU time, plus a bit to account for overhead. Note that the sleep - // is wall clock time. The CPU clock may be slower than the wall clock. - let sleep_interval = timeout.saturating_sub(cpu_time_elapsed) + JOB_TIMEOUT_OVERHEAD; - match finished_rx.recv_timeout(sleep_interval) { - // Received finish signal. - Ok(()) => return None, - // Timed out, restart loop. - Err(RecvTimeoutError::Timeout) => continue, - Err(RecvTimeoutError::Disconnected) => return None, - } - } - - return Some(cpu_time_elapsed) - } -} - /// A struct that represents an idle worker. /// /// This struct is supposed to be used as a token that is passed by move into a subroutine that @@ -405,12 +312,7 @@ pub fn path_to_bytes(path: &Path) -> &[u8] { path.to_str().expect("non-UTF-8 path").as_bytes() } -/// Interprets the given bytes as a path. Returns `None` if the given bytes do not constitute a -/// a proper utf-8 string. -pub fn bytes_to_path(bytes: &[u8]) -> Option { - std::str::from_utf8(bytes).ok().map(PathBuf::from) -} - +/// Write some data prefixed by its length into `w`. pub async fn framed_send(w: &mut (impl AsyncWrite + Unpin), buf: &[u8]) -> io::Result<()> { let len_buf = buf.len().to_le_bytes(); w.write_all(&len_buf).await?; @@ -418,6 +320,7 @@ pub async fn framed_send(w: &mut (impl AsyncWrite + Unpin), buf: &[u8]) -> io::R Ok(()) } +/// Read some data prefixed by its length from `r`. pub async fn framed_recv(r: &mut (impl AsyncRead + Unpin)) -> io::Result> { let mut len_buf = [0u8; mem::size_of::()]; r.read_exact(&mut len_buf).await?; @@ -426,20 +329,3 @@ pub async fn framed_recv(r: &mut (impl AsyncRead + Unpin)) -> io::Result r.read_exact(&mut buf).await?; Ok(buf) } - -/// In case of node and worker version mismatch (as a result of in-place upgrade), send `SIGTERM` -/// to the node to tear it down and prevent it from raising disputes on valid candidates. Node -/// restart should be handled by the node owner. As node exits, unix sockets opened to workers -/// get closed by the OS and other workers receive error on socket read and also exit. Preparation -/// jobs are written to the temporary files that are renamed to real artifacts on the node side, so -/// no leftover artifacts are possible. -fn kill_parent_node_in_emergency() { - unsafe { - // SAFETY: `getpid()` never fails but may return "no-parent" (0) or "parent-init" (1) in - // some corner cases, which is checked. `kill()` never fails. - let ppid = libc::getppid(); - if ppid > 1 { - libc::kill(ppid, libc::SIGTERM); - } - } -} diff --git a/node/core/pvf/worker/Cargo.toml b/node/core/pvf/worker/Cargo.toml new file mode 100644 index 000000000000..260c6217eb67 --- /dev/null +++ b/node/core/pvf/worker/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "polkadot-node-core-pvf-worker" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[[bin]] +name = "puppet_worker" +path = "bin/puppet_worker.rs" + +[dependencies] +assert_matches = "1.4.0" +cpu-time = "1.0.0" +futures = "0.3.21" +gum = { package = "tracing-gum", path = "../../../gum" } +libc = "0.2.139" +rayon = "1.5.1" +tempfile = "3.3.0" +tikv-jemalloc-ctl = { version = "0.5.0", optional = true } +tokio = "1.24.2" + +parity-scale-codec = { version = "3.4.0", default-features = false, features = ["derive"] } + +polkadot-node-core-pvf = { path = ".." } +polkadot-parachain = { path = "../../../../parachain" } +polkadot-primitives = { path = "../../../../primitives" } + +sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master" } +sc-executor-common = { git = "https://github.com/paritytech/substrate", branch = "master" } +sc-executor-wasmtime = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-externalities = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-maybe-compressed-blob = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } + +[target.'cfg(target_os = "linux")'.dependencies] +tikv-jemalloc-ctl = "0.5.0" + +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } + +[dev-dependencies] +adder = { package = "test-parachain-adder", path = "../../../../parachain/test-parachains/adder" } +halt = { package = "test-parachain-halt", path = "../../../../parachain/test-parachains/halt" } +tempfile = "3.3.0" + +[features] +jemalloc-allocator = ["dep:tikv-jemalloc-ctl"] diff --git a/node/core/pvf/bin/puppet_worker.rs b/node/core/pvf/worker/bin/puppet_worker.rs similarity index 92% rename from node/core/pvf/bin/puppet_worker.rs rename to node/core/pvf/worker/bin/puppet_worker.rs index 7f93519d8454..ddd81971292b 100644 --- a/node/core/pvf/bin/puppet_worker.rs +++ b/node/core/pvf/worker/bin/puppet_worker.rs @@ -14,4 +14,4 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -polkadot_node_core_pvf::decl_puppet_worker_main!(); +polkadot_node_core_pvf_worker::decl_puppet_worker_main!(); diff --git a/node/core/pvf/worker/build.rs b/node/core/pvf/worker/build.rs new file mode 100644 index 000000000000..40e9f832586e --- /dev/null +++ b/node/core/pvf/worker/build.rs @@ -0,0 +1,19 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_cargo_keys(); +} diff --git a/node/core/pvf/worker/src/common.rs b/node/core/pvf/worker/src/common.rs new file mode 100644 index 000000000000..84bc88701d62 --- /dev/null +++ b/node/core/pvf/worker/src/common.rs @@ -0,0 +1,142 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use crate::LOG_TARGET; +use cpu_time::ProcessTime; +use futures::never::Never; +use std::{ + path::PathBuf, + sync::mpsc::{Receiver, RecvTimeoutError}, + time::Duration, +}; +use tokio::{ + io, + net::UnixStream, + runtime::{Handle, Runtime}, +}; + +/// Some allowed overhead that we account for in the "CPU time monitor" thread's sleeps, on the +/// child process. +pub const JOB_TIMEOUT_OVERHEAD: Duration = Duration::from_millis(50); + +/// Interprets the given bytes as a path. Returns `None` if the given bytes do not constitute a +/// a proper utf-8 string. +pub fn bytes_to_path(bytes: &[u8]) -> Option { + std::str::from_utf8(bytes).ok().map(PathBuf::from) +} + +pub fn worker_event_loop( + debug_id: &'static str, + socket_path: &str, + node_version: Option<&str>, + mut event_loop: F, +) where + F: FnMut(Handle, UnixStream) -> Fut, + Fut: futures::Future>, +{ + let worker_pid = std::process::id(); + gum::debug!(target: LOG_TARGET, %worker_pid, "starting pvf worker ({})", debug_id); + + // Check for a mismatch between the node and worker versions. + if let Some(version) = node_version { + if version != env!("SUBSTRATE_CLI_IMPL_VERSION") { + gum::error!( + target: LOG_TARGET, + %worker_pid, + "Node and worker version mismatch, node needs restarting, forcing shutdown", + ); + kill_parent_node_in_emergency(); + let err: io::Result = + Err(io::Error::new(io::ErrorKind::Unsupported, "Version mismatch")); + gum::debug!(target: LOG_TARGET, %worker_pid, "quitting pvf worker({}): {:?}", debug_id, err); + return + } + } + + // Run the main worker loop. + let rt = Runtime::new().expect("Creates tokio runtime. If this panics the worker will die and the host will detect that and deal with it."); + let handle = rt.handle(); + let err = rt + .block_on(async move { + let stream = UnixStream::connect(socket_path).await?; + let _ = tokio::fs::remove_file(socket_path).await; + + let result = event_loop(handle.clone(), stream).await; + + result + }) + // It's never `Ok` because it's `Ok(Never)`. + .unwrap_err(); + + gum::debug!(target: LOG_TARGET, %worker_pid, "quitting pvf worker ({}): {:?}", debug_id, err); + + // We don't want tokio to wait for the tasks to finish. We want to bring down the worker as fast + // as possible and not wait for stalled validation to finish. This isn't strictly necessary now, + // but may be in the future. + rt.shutdown_background(); +} + +/// Loop that runs in the CPU time monitor thread on prepare and execute jobs. Continuously wakes up +/// and then either blocks for the remaining CPU time, or returns if we exceed the CPU timeout. +/// +/// Returning `Some` indicates that we should send a `TimedOut` error to the host. Will return +/// `None` if the other thread finishes first, without us timing out. +/// +/// NOTE: Sending a `TimedOut` error to the host will cause the worker, whether preparation or +/// execution, to be killed by the host. We do not kill the process here because it would interfere +/// with the proper handling of this error. +pub fn cpu_time_monitor_loop( + cpu_time_start: ProcessTime, + timeout: Duration, + finished_rx: Receiver<()>, +) -> Option { + loop { + let cpu_time_elapsed = cpu_time_start.elapsed(); + + // Treat the timeout as CPU time, which is less subject to variance due to load. + if cpu_time_elapsed <= timeout { + // Sleep for the remaining CPU time, plus a bit to account for overhead. Note that the sleep + // is wall clock time. The CPU clock may be slower than the wall clock. + let sleep_interval = timeout.saturating_sub(cpu_time_elapsed) + JOB_TIMEOUT_OVERHEAD; + match finished_rx.recv_timeout(sleep_interval) { + // Received finish signal. + Ok(()) => return None, + // Timed out, restart loop. + Err(RecvTimeoutError::Timeout) => continue, + Err(RecvTimeoutError::Disconnected) => return None, + } + } + + return Some(cpu_time_elapsed) + } +} + +/// In case of node and worker version mismatch (as a result of in-place upgrade), send `SIGTERM` +/// to the node to tear it down and prevent it from raising disputes on valid candidates. Node +/// restart should be handled by the node owner. As node exits, unix sockets opened to workers +/// get closed by the OS and other workers receive error on socket read and also exit. Preparation +/// jobs are written to the temporary files that are renamed to real artifacts on the node side, so +/// no leftover artifacts are possible. +fn kill_parent_node_in_emergency() { + unsafe { + // SAFETY: `getpid()` never fails but may return "no-parent" (0) or "parent-init" (1) in + // some corner cases, which is checked. `kill()` never fails. + let ppid = libc::getppid(); + if ppid > 1 { + libc::kill(ppid, libc::SIGTERM); + } + } +} diff --git a/node/core/pvf/worker/src/execute.rs b/node/core/pvf/worker/src/execute.rs new file mode 100644 index 000000000000..9f6ff164a2b6 --- /dev/null +++ b/node/core/pvf/worker/src/execute.rs @@ -0,0 +1,175 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use crate::{ + common::{bytes_to_path, cpu_time_monitor_loop, worker_event_loop}, + executor_intf::Executor, + LOG_TARGET, +}; +use cpu_time::ProcessTime; +use futures::{pin_mut, select_biased, FutureExt}; +use parity_scale_codec::{Decode, Encode}; +use polkadot_node_core_pvf::{ + framed_recv, framed_send, ExecuteHandshake as Handshake, ExecuteResponse as Response, +}; +use polkadot_parachain::primitives::ValidationResult; +use std::{ + path::{Path, PathBuf}, + sync::{mpsc::channel, Arc}, + time::Duration, +}; +use tokio::{io, net::UnixStream}; + +async fn recv_handshake(stream: &mut UnixStream) -> io::Result { + let handshake_enc = framed_recv(stream).await?; + let handshake = Handshake::decode(&mut &handshake_enc[..]).map_err(|_| { + io::Error::new( + io::ErrorKind::Other, + "execute pvf recv_handshake: failed to decode Handshake".to_owned(), + ) + })?; + Ok(handshake) +} + +async fn recv_request(stream: &mut UnixStream) -> io::Result<(PathBuf, Vec, Duration)> { + let artifact_path = framed_recv(stream).await?; + let artifact_path = bytes_to_path(&artifact_path).ok_or_else(|| { + io::Error::new( + io::ErrorKind::Other, + "execute pvf recv_request: non utf-8 artifact path".to_string(), + ) + })?; + let params = framed_recv(stream).await?; + let execution_timeout = framed_recv(stream).await?; + let execution_timeout = Duration::decode(&mut &execution_timeout[..]).map_err(|_| { + io::Error::new( + io::ErrorKind::Other, + "execute pvf recv_request: failed to decode duration".to_string(), + ) + })?; + Ok((artifact_path, params, execution_timeout)) +} + +async fn send_response(stream: &mut UnixStream, response: Response) -> io::Result<()> { + framed_send(stream, &response.encode()).await +} + +/// The entrypoint that the spawned execute worker should start with. The `socket_path` specifies +/// the path to the socket used to communicate with the host. The `node_version`, if `Some`, +/// is checked against the worker version. A mismatch results in immediate worker termination. +/// `None` is used for tests and in other situations when version check is not necessary. +pub fn worker_entrypoint(socket_path: &str, node_version: Option<&str>) { + worker_event_loop("execute", socket_path, node_version, |rt_handle, mut stream| async move { + let worker_pid = std::process::id(); + + let handshake = recv_handshake(&mut stream).await?; + let executor = Arc::new(Executor::new(handshake.executor_params).map_err(|e| { + io::Error::new(io::ErrorKind::Other, format!("cannot create executor: {}", e)) + })?); + + loop { + let (artifact_path, params, execution_timeout) = recv_request(&mut stream).await?; + gum::debug!( + target: LOG_TARGET, + %worker_pid, + "worker: validating artifact {}", + artifact_path.display(), + ); + + // Used to signal to the cpu time monitor thread that it can finish. + let (finished_tx, finished_rx) = channel::<()>(); + let cpu_time_start = ProcessTime::now(); + + // Spawn a new thread that runs the CPU time monitor. + let cpu_time_monitor_fut = rt_handle + .spawn_blocking(move || { + cpu_time_monitor_loop(cpu_time_start, execution_timeout, finished_rx) + }) + .fuse(); + let executor_2 = executor.clone(); + let execute_fut = rt_handle + .spawn_blocking(move || { + validate_using_artifact(&artifact_path, ¶ms, executor_2, cpu_time_start) + }) + .fuse(); + + pin_mut!(cpu_time_monitor_fut); + pin_mut!(execute_fut); + + let response = select_biased! { + // If this future is not selected, the join handle is dropped and the thread will + // finish in the background. + cpu_time_monitor_res = cpu_time_monitor_fut => { + match cpu_time_monitor_res { + Ok(Some(cpu_time_elapsed)) => { + // Log if we exceed the timeout and the other thread hasn't finished. + gum::warn!( + target: LOG_TARGET, + %worker_pid, + "execute job took {}ms cpu time, exceeded execute timeout {}ms", + cpu_time_elapsed.as_millis(), + execution_timeout.as_millis(), + ); + Response::TimedOut + }, + Ok(None) => Response::InternalError("error communicating over finished channel".into()), + Err(e) => Response::format_internal("cpu time monitor thread error", &e.to_string()), + } + }, + execute_res = execute_fut => { + let _ = finished_tx.send(()); + execute_res.unwrap_or_else(|e| Response::format_internal("execute thread error", &e.to_string())) + }, + }; + + send_response(&mut stream, response).await?; + } + }); +} + +fn validate_using_artifact( + artifact_path: &Path, + params: &[u8], + executor: Arc, + cpu_time_start: ProcessTime, +) -> Response { + // Check here if the file exists, because the error from Substrate is not match-able. + // TODO: Re-evaluate after . + let file_metadata = std::fs::metadata(artifact_path); + if let Err(err) = file_metadata { + return Response::format_internal("execute: could not find or open file", &err.to_string()) + } + + let descriptor_bytes = match unsafe { + // SAFETY: this should be safe since the compiled artifact passed here comes from the + // file created by the prepare workers. These files are obtained by calling + // [`executor_intf::prepare`]. + executor.execute(artifact_path.as_ref(), params) + } { + Err(err) => return Response::format_invalid("execute", &err), + Ok(d) => d, + }; + + let duration = cpu_time_start.elapsed(); + + let result_descriptor = match ValidationResult::decode(&mut &descriptor_bytes[..]) { + Err(err) => + return Response::format_invalid("validation result decoding failed", &err.to_string()), + Ok(r) => r, + }; + + Response::Ok { result_descriptor, duration } +} diff --git a/node/core/pvf/src/executor_intf.rs b/node/core/pvf/worker/src/executor_intf.rs similarity index 100% rename from node/core/pvf/src/executor_intf.rs rename to node/core/pvf/worker/src/executor_intf.rs diff --git a/node/core/pvf/worker/src/lib.rs b/node/core/pvf/worker/src/lib.rs new file mode 100644 index 000000000000..456362cf8f57 --- /dev/null +++ b/node/core/pvf/worker/src/lib.rs @@ -0,0 +1,73 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +mod common; +mod execute; +mod executor_intf; +mod memory_stats; +mod prepare; + +#[doc(hidden)] +pub mod testing; + +#[doc(hidden)] +pub use sp_tracing; + +pub use execute::worker_entrypoint as execute_worker_entrypoint; +pub use prepare::worker_entrypoint as prepare_worker_entrypoint; + +pub use executor_intf::{prepare, prevalidate}; + +// NOTE: Initializing logging in e.g. tests will not have an effect in the workers, as they are +// separate spawned processes. Run with e.g. `RUST_LOG=parachain::pvf-worker=trace`. +const LOG_TARGET: &str = "parachain::pvf-worker"; + +/// Use this macro to declare a `fn main() {}` that will create an executable that can be used for +/// spawning the desired worker. +#[macro_export(local_inner_macros)] +macro_rules! decl_worker_main { + ($command:tt) => { + fn main() { + $crate::sp_tracing::try_init_simple(); + + let args = std::env::args().collect::>(); + + let mut version = None; + let mut socket_path: &str = ""; + + for i in 1..args.len() { + match args[i].as_ref() { + "--socket-path" => socket_path = args[i + 1].as_str(), + "--node-version" => version = Some(args[i + 1].as_str()), + _ => (), + } + } + + decl_worker_main_command!($command, socket_path, version) + } + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! decl_worker_main_command { + (prepare, $socket_path:expr, $version: expr) => { + $crate::prepare_worker_entrypoint(&$socket_path, $version) + }; + (execute, $socket_path:expr, $version: expr) => { + $crate::execute_worker_entrypoint(&$socket_path, $version) + }; +} diff --git a/node/core/pvf/src/prepare/memory_stats.rs b/node/core/pvf/worker/src/memory_stats.rs similarity index 82% rename from node/core/pvf/src/prepare/memory_stats.rs rename to node/core/pvf/worker/src/memory_stats.rs index 069ef46caba4..945c849eb1db 100644 --- a/node/core/pvf/src/prepare/memory_stats.rs +++ b/node/core/pvf/worker/src/memory_stats.rs @@ -27,38 +27,14 @@ //! for more //! background. -use parity_scale_codec::{Decode, Encode}; - -/// Helper struct to contain all the memory stats, including [`MemoryAllocationStats`] and, if -/// supported by the OS, `ru_maxrss`. -#[derive(Clone, Debug, Default, Encode, Decode)] -pub struct MemoryStats { - /// Memory stats from `tikv_jemalloc_ctl`. - #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] - pub memory_tracker_stats: Option, - /// `ru_maxrss` from `getrusage`. A string error since `io::Error` is not `Encode`able. - #[cfg(target_os = "linux")] - pub max_rss: Option, -} - -/// Statistics of collected memory metrics. -#[non_exhaustive] -#[derive(Clone, Debug, Default, Encode, Decode)] -pub struct MemoryAllocationStats { - /// Total resident memory, in bytes. - pub resident: u64, - /// Total allocated memory, in bytes. - pub allocated: u64, -} - /// Module for the memory tracker. The memory tracker runs in its own thread, where it polls memory /// usage at an interval. /// /// NOTE: Requires jemalloc enabled. #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] pub mod memory_tracker { - use super::*; use crate::LOG_TARGET; + use polkadot_node_core_pvf::MemoryAllocationStats; use std::{ sync::mpsc::{Receiver, RecvTimeoutError, Sender}, time::Duration, @@ -197,33 +173,21 @@ pub mod memory_tracker { #[cfg(target_os = "linux")] pub mod max_rss_stat { use crate::LOG_TARGET; - use libc::{getrusage, rusage, timeval, RUSAGE_THREAD}; + use core::mem::MaybeUninit; + use libc::{getrusage, rusage, RUSAGE_THREAD}; use std::io; /// Get the rusage stats for the current thread. fn getrusage_thread() -> io::Result { - let mut result = rusage { - ru_utime: timeval { tv_sec: 0, tv_usec: 0 }, - ru_stime: timeval { tv_sec: 0, tv_usec: 0 }, - ru_maxrss: 0, - ru_ixrss: 0, - ru_idrss: 0, - ru_isrss: 0, - ru_minflt: 0, - ru_majflt: 0, - ru_nswap: 0, - ru_inblock: 0, - ru_oublock: 0, - ru_msgsnd: 0, - ru_msgrcv: 0, - ru_nsignals: 0, - ru_nvcsw: 0, - ru_nivcsw: 0, - }; - if unsafe { getrusage(RUSAGE_THREAD, &mut result) } == -1 { + let mut result: MaybeUninit = MaybeUninit::zeroed(); + + // SAFETY: `result` is a valid pointer, so calling this is safe. + if unsafe { getrusage(RUSAGE_THREAD, result.as_mut_ptr()) } == -1 { return Err(io::Error::last_os_error()) } - Ok(result) + + // SAFETY: `result` was successfully initialized by `getrusage`. + unsafe { Ok(result.assume_init()) } } /// Gets the `ru_maxrss` for the current thread. diff --git a/node/core/pvf/worker/src/prepare.rs b/node/core/pvf/worker/src/prepare.rs new file mode 100644 index 000000000000..3cec7439f8df --- /dev/null +++ b/node/core/pvf/worker/src/prepare.rs @@ -0,0 +1,222 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +#[cfg(target_os = "linux")] +use crate::memory_stats::max_rss_stat::{extract_max_rss_stat, get_max_rss_thread}; +#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] +use crate::memory_stats::memory_tracker::{get_memory_tracker_loop_stats, memory_tracker_loop}; +use crate::{ + common::{bytes_to_path, cpu_time_monitor_loop, worker_event_loop}, + prepare, prevalidate, LOG_TARGET, +}; +use cpu_time::ProcessTime; +use futures::{pin_mut, select_biased, FutureExt}; +use parity_scale_codec::{Decode, Encode}; +use polkadot_node_core_pvf::{ + framed_recv, framed_send, CompiledArtifact, MemoryStats, PrepareError, PrepareResult, + PrepareStats, PvfPrepData, +}; +use std::{any::Any, panic, path::PathBuf, sync::mpsc::channel}; +use tokio::{io, net::UnixStream}; + +async fn recv_request(stream: &mut UnixStream) -> io::Result<(PvfPrepData, PathBuf)> { + let pvf = framed_recv(stream).await?; + let pvf = PvfPrepData::decode(&mut &pvf[..]).map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("prepare pvf recv_request: failed to decode PvfPrepData: {}", e), + ) + })?; + let tmp_file = framed_recv(stream).await?; + let tmp_file = bytes_to_path(&tmp_file).ok_or_else(|| { + io::Error::new( + io::ErrorKind::Other, + "prepare pvf recv_request: non utf-8 artifact path".to_string(), + ) + })?; + Ok((pvf, tmp_file)) +} + +async fn send_response(stream: &mut UnixStream, result: PrepareResult) -> io::Result<()> { + framed_send(stream, &result.encode()).await +} + +/// The entrypoint that the spawned prepare worker should start with. The `socket_path` specifies +/// the path to the socket used to communicate with the host. The `node_version`, if `Some`, +/// is checked against the worker version. A mismatch results in immediate worker termination. +/// `None` is used for tests and in other situations when version check is not necessary. +/// +/// # Flow +/// +/// This runs the following in a loop: +/// +/// 1. Get the code and parameters for preparation from the host. +/// +/// 2. Start a memory tracker in a separate thread. +/// +/// 3. Start the CPU time monitor loop and the actual preparation in two separate threads. +/// +/// 4. Select on the two threads created in step 3. If the CPU timeout was hit, the CPU time monitor +/// thread will trigger first. +/// +/// 5. Stop the memory tracker and get the stats. +/// +/// 6. If compilation succeeded, write the compiled artifact into a temporary file. +/// +/// 7. Send the result of preparation back to the host. If any error occurred in the above steps, we +/// send that in the `PrepareResult`. +pub fn worker_entrypoint(socket_path: &str, node_version: Option<&str>) { + worker_event_loop("prepare", socket_path, node_version, |rt_handle, mut stream| async move { + let worker_pid = std::process::id(); + + loop { + let (pvf, dest) = recv_request(&mut stream).await?; + gum::debug!( + target: LOG_TARGET, + %worker_pid, + "worker: preparing artifact", + ); + + let cpu_time_start = ProcessTime::now(); + let preparation_timeout = pvf.prep_timeout(); + + // Run the memory tracker. + #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] + let (memory_tracker_tx, memory_tracker_rx) = channel::<()>(); + #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] + let memory_tracker_fut = rt_handle.spawn_blocking(move || memory_tracker_loop(memory_tracker_rx)); + + // Spawn a new thread that runs the CPU time monitor. + let (cpu_time_monitor_tx, cpu_time_monitor_rx) = channel::<()>(); + let cpu_time_monitor_fut = rt_handle + .spawn_blocking(move || { + cpu_time_monitor_loop(cpu_time_start, preparation_timeout, cpu_time_monitor_rx) + }) + .fuse(); + // Spawn another thread for preparation. + let prepare_fut = rt_handle + .spawn_blocking(move || { + let result = prepare_artifact(pvf); + + // Get the `ru_maxrss` stat. If supported, call getrusage for the thread. + #[cfg(target_os = "linux")] + let result = result.map(|artifact| (artifact, get_max_rss_thread())); + + result + }) + .fuse(); + + pin_mut!(cpu_time_monitor_fut); + pin_mut!(prepare_fut); + + let result = select_biased! { + // If this future is not selected, the join handle is dropped and the thread will + // finish in the background. + join_res = cpu_time_monitor_fut => { + match join_res { + Ok(Some(cpu_time_elapsed)) => { + // Log if we exceed the timeout and the other thread hasn't finished. + gum::warn!( + target: LOG_TARGET, + %worker_pid, + "prepare job took {}ms cpu time, exceeded prepare timeout {}ms", + cpu_time_elapsed.as_millis(), + preparation_timeout.as_millis(), + ); + Err(PrepareError::TimedOut) + }, + Ok(None) => Err(PrepareError::IoErr("error communicating over finished channel".into())), + Err(err) => Err(PrepareError::IoErr(err.to_string())), + } + }, + prepare_res = prepare_fut => { + let cpu_time_elapsed = cpu_time_start.elapsed(); + let _ = cpu_time_monitor_tx.send(()); + + match prepare_res.unwrap_or_else(|err| Err(PrepareError::IoErr(err.to_string()))) { + Err(err) => { + // Serialized error will be written into the socket. + Err(err) + }, + Ok(ok) => { + // Stop the memory stats worker and get its observed memory stats. + #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] + let memory_tracker_stats = + get_memory_tracker_loop_stats(memory_tracker_fut, memory_tracker_tx, worker_pid).await; + #[cfg(target_os = "linux")] + let (ok, max_rss) = ok; + let memory_stats = MemoryStats { + #[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))] + memory_tracker_stats, + #[cfg(target_os = "linux")] + max_rss: extract_max_rss_stat(max_rss, worker_pid), + }; + + // Write the serialized artifact into a temp file. + // + // PVF host only keeps artifacts statuses in its memory, successfully + // compiled code gets stored on the disk (and consequently deserialized + // by execute-workers). The prepare worker is only required to send `Ok` + // to the pool to indicate the success. + + gum::debug!( + target: LOG_TARGET, + %worker_pid, + "worker: writing artifact to {}", + dest.display(), + ); + tokio::fs::write(&dest, &ok).await?; + + Ok(PrepareStats{cpu_time_elapsed, memory_stats}) + }, + } + }, + }; + + send_response(&mut stream, result).await?; + } + }); +} + +fn prepare_artifact(pvf: PvfPrepData) -> Result { + panic::catch_unwind(|| { + let blob = match prevalidate(&pvf.code()) { + Err(err) => return Err(PrepareError::Prevalidation(format!("{:?}", err))), + Ok(b) => b, + }; + + match prepare(blob, &pvf.executor_params()) { + Ok(compiled_artifact) => Ok(CompiledArtifact::new(compiled_artifact)), + Err(err) => Err(PrepareError::Preparation(format!("{:?}", err))), + } + }) + .map_err(|panic_payload| PrepareError::Panic(stringify_panic_payload(panic_payload))) + .and_then(|inner_result| inner_result) +} + +/// Attempt to convert an opaque panic payload to a string. +/// +/// This is a best effort, and is not guaranteed to provide the most accurate value. +fn stringify_panic_payload(payload: Box) -> String { + match payload.downcast::<&'static str>() { + Ok(msg) => msg.to_string(), + Err(payload) => match payload.downcast::() { + Ok(msg) => *msg, + // At least we tried... + Err(_) => "unknown panic payload".to_string(), + }, + } +} diff --git a/node/core/pvf/src/testing.rs b/node/core/pvf/worker/src/testing.rs similarity index 96% rename from node/core/pvf/src/testing.rs rename to node/core/pvf/worker/src/testing.rs index fb1b406cdad6..d09b68bf8b33 100644 --- a/node/core/pvf/src/testing.rs +++ b/node/core/pvf/worker/src/testing.rs @@ -21,10 +21,6 @@ use polkadot_primitives::ExecutorParams; -pub mod worker_common { - pub use crate::worker_common::{spawn_with_program_path, SpawnErr}; -} - /// A function that emulates the stitches together behaviors of the preparation and the execution /// worker in a single synchronous function. pub fn validate_candidate( diff --git a/node/core/pvf/tests/it/adder.rs b/node/core/pvf/worker/tests/it/adder.rs similarity index 100% rename from node/core/pvf/tests/it/adder.rs rename to node/core/pvf/worker/tests/it/adder.rs diff --git a/node/core/pvf/tests/it/main.rs b/node/core/pvf/worker/tests/it/main.rs similarity index 82% rename from node/core/pvf/tests/it/main.rs rename to node/core/pvf/worker/tests/it/main.rs index ecd885ab642e..91c93e35ccd8 100644 --- a/node/core/pvf/tests/it/main.rs +++ b/node/core/pvf/worker/tests/it/main.rs @@ -33,7 +33,7 @@ const TEST_EXECUTION_TIMEOUT: Duration = Duration::from_secs(3); const TEST_PREPARATION_TIMEOUT: Duration = Duration::from_secs(3); struct TestHost { - _cache_dir: tempfile::TempDir, + cache_dir: tempfile::TempDir, host: Mutex, } @@ -52,7 +52,7 @@ impl TestHost { f(&mut config); let (host, task) = start(config, Metrics::default()); let _ = tokio::task::spawn(task); - Self { _cache_dir: cache_dir, host: Mutex::new(host) } + Self { cache_dir, host: Mutex::new(host) } } async fn validate_candidate( @@ -240,3 +240,58 @@ async fn execute_queue_doesnt_stall_with_varying_executor_params() { max_duration.as_millis() ); } + +// Test that deleting a prepared artifact does not lead to a dispute when we try to execute it. +#[tokio::test] +async fn deleting_prepared_artifact_does_not_dispute() { + let host = TestHost::new(); + let cache_dir = host.cache_dir.path().clone(); + + let result = host + .validate_candidate( + halt::wasm_binary_unwrap(), + ValidationParams { + block_data: BlockData(Vec::new()), + parent_head: Default::default(), + relay_parent_number: 1, + relay_parent_storage_root: Default::default(), + }, + Default::default(), + ) + .await; + + match result { + Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) => {}, + r => panic!("{:?}", r), + } + + // Delete the prepared artifact. + { + // Get the artifact path (asserting it exists). + let mut cache_dir: Vec<_> = std::fs::read_dir(cache_dir).unwrap().collect(); + assert_eq!(cache_dir.len(), 1); + let artifact_path = cache_dir.pop().unwrap().unwrap(); + + // Delete the artifact. + std::fs::remove_file(artifact_path.path()).unwrap(); + } + + // Try to validate again, artifact should get recreated. + let result = host + .validate_candidate( + halt::wasm_binary_unwrap(), + ValidationParams { + block_data: BlockData(Vec::new()), + parent_head: Default::default(), + relay_parent_number: 1, + relay_parent_storage_root: Default::default(), + }, + Default::default(), + ) + .await; + + match result { + Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) => {}, + r => panic!("{:?}", r), + } +} diff --git a/node/core/pvf/tests/it/worker_common.rs b/node/core/pvf/worker/tests/it/worker_common.rs similarity index 94% rename from node/core/pvf/tests/it/worker_common.rs rename to node/core/pvf/worker/tests/it/worker_common.rs index 3a17efc8df5c..439ac8538c95 100644 --- a/node/core/pvf/tests/it/worker_common.rs +++ b/node/core/pvf/worker/tests/it/worker_common.rs @@ -15,7 +15,7 @@ // along with Polkadot. If not, see . use crate::PUPPET_EXE; -use polkadot_node_core_pvf::testing::worker_common::{spawn_with_program_path, SpawnErr}; +use polkadot_node_core_pvf::testing::{spawn_with_program_path, SpawnErr}; use std::time::Duration; // Test spawning a program that immediately exits with a failure code. diff --git a/node/gum/proc-macro/Cargo.toml b/node/gum/proc-macro/Cargo.toml index fccd4d218121..3fac1a7b928f 100644 --- a/node/gum/proc-macro/Cargo.toml +++ b/node/gum/proc-macro/Cargo.toml @@ -12,11 +12,11 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -syn = { version = "1.0.95", features = ["full", "extra-traits"] } -quote = "1.0.20" -proc-macro2 = "1.0.43" +syn = { version = "2.0.15", features = ["full", "extra-traits"] } +quote = "1.0.26" +proc-macro2 = "1.0.56" proc-macro-crate = "1.1.3" -expander = "0.0.6" +expander = "2.0.0" [dev-dependencies] assert_matches = "1.5.0" diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index 97752af8d71d..a36822b041a3 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -20,9 +20,9 @@ polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-core-dispute-coordinator = { path = "../core/dispute-coordinator" } polkadot-node-core-candidate-validation = { path = "../core/candidate-validation" } polkadot-node-core-backing = { path = "../core/backing" } +polkadot-node-core-pvf-worker = { path = "../core/pvf/worker" } polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } -polkadot-node-core-pvf = { path = "../core/pvf" } color-eyre = { version = "0.6.1", default-features = false } assert_matches = "1.5" async-trait = "0.1.57" diff --git a/node/malus/src/malus.rs b/node/malus/src/malus.rs index f202996aca1e..36cf0cca06bf 100644 --- a/node/malus/src/malus.rs +++ b/node/malus/src/malus.rs @@ -97,7 +97,10 @@ impl MalusCli { #[cfg(not(target_os = "android"))] { - polkadot_node_core_pvf::prepare_worker_entrypoint(&cmd.socket_path, None); + polkadot_node_core_pvf_worker::prepare_worker_entrypoint( + &cmd.socket_path, + None, + ); } }, NemesisVariant::PvfExecuteWorker(cmd) => { @@ -108,7 +111,10 @@ impl MalusCli { #[cfg(not(target_os = "android"))] { - polkadot_node_core_pvf::execute_worker_entrypoint(&cmd.socket_path, None); + polkadot_node_core_pvf_worker::execute_worker_entrypoint( + &cmd.socket_path, + None, + ); } }, } diff --git a/node/network/approval-distribution/src/lib.rs b/node/network/approval-distribution/src/lib.rs index 22c5ed8b66e2..c8ad21ad406e 100644 --- a/node/network/approval-distribution/src/lib.rs +++ b/node/network/approval-distribution/src/lib.rs @@ -124,12 +124,12 @@ struct AggressionConfig { } impl AggressionConfig { - /// Returns `true` if block is not too old depending on the aggression level - fn is_age_relevant(&self, block_age: BlockNumber) -> bool { + /// Returns `true` if lag is past threshold depending on the aggression level + fn should_trigger_aggression(&self, approval_checking_lag: BlockNumber) -> bool { if let Some(t) = self.l1_threshold { - block_age >= t + approval_checking_lag >= t } else if let Some(t) = self.resend_unfinalized_period { - block_age > 0 && block_age % t == 0 + approval_checking_lag > 0 && approval_checking_lag % t == 0 } else { false } @@ -184,6 +184,9 @@ struct State { /// HashMap from active leaves to spans spans: HashMap, + + /// Current approval checking finality lag. + approval_checking_lag: BlockNumber, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1425,19 +1428,29 @@ impl State { resend: Resend, metrics: &Metrics, ) { - let min_age = self.blocks_by_number.iter().next().map(|(num, _)| num); - let max_age = self.blocks_by_number.iter().rev().next().map(|(num, _)| num); let config = self.aggression_config.clone(); - let (min_age, max_age) = match (min_age, max_age) { - (Some(min), Some(max)) => (min, max), + if !self.aggression_config.should_trigger_aggression(self.approval_checking_lag) { + gum::trace!( + target: LOG_TARGET, + approval_checking_lag = self.approval_checking_lag, + "Aggression not enabled", + ); + return + } + + let max_age = self.blocks_by_number.iter().rev().next().map(|(num, _)| num); + + let max_age = match max_age { + Some(max) => *max, _ => return, // empty. }; - let diff = max_age - min_age; - if !self.aggression_config.is_age_relevant(diff) { - return - } + // Since we have the approval checking lag, we need to set the `min_age` accordingly to + // enable aggresion for the oldest block that is not approved. + let min_age = max_age.saturating_sub(self.approval_checking_lag); + + gum::debug!(target: LOG_TARGET, min_age, max_age, "Aggression enabled",); adjust_required_routing_and_propagate( ctx, @@ -1476,20 +1489,21 @@ impl State { // its descendants from being finalized. Waste minimal bandwidth // this way. Also, disputes might prevent finality - again, nothing // to waste bandwidth on newer blocks for. - &block_entry.number == min_age + block_entry.number == min_age }, |required_routing, local, _| { // It's a bit surprising not to have a topology at this age. if *required_routing == RequiredRouting::PendingTopology { gum::debug!( target: LOG_TARGET, - age = ?diff, + lag = ?self.approval_checking_lag, "Encountered old block pending gossip topology", ); return } - if config.l1_threshold.as_ref().map_or(false, |t| &diff >= t) { + if config.l1_threshold.as_ref().map_or(false, |t| &self.approval_checking_lag >= t) + { // Message originator sends to everyone. if local && *required_routing != RequiredRouting::All { metrics.on_aggression_l1(); @@ -1497,7 +1511,8 @@ impl State { } } - if config.l2_threshold.as_ref().map_or(false, |t| &diff >= t) { + if config.l2_threshold.as_ref().map_or(false, |t| &self.approval_checking_lag >= t) + { // Message originator sends to everyone. Everyone else sends to XY. if !local && *required_routing != RequiredRouting::GridXY { metrics.on_aggression_l2(); @@ -1764,6 +1779,10 @@ impl ApprovalDistribution { ); } }, + ApprovalDistributionMessage::ApprovalCheckingLagUpdate(lag) => { + gum::debug!(target: LOG_TARGET, lag, "Received `ApprovalCheckingLagUpdate`"); + state.approval_checking_lag = lag; + }, } } } diff --git a/node/network/approval-distribution/src/tests.rs b/node/network/approval-distribution/src/tests.rs index 31fe0a421d38..f4b507b18fd6 100644 --- a/node/network/approval-distribution/src/tests.rs +++ b/node/network/approval-distribution/src/tests.rs @@ -24,7 +24,7 @@ use polkadot_node_network_protocol::{ view, ObservedRole, }; use polkadot_node_primitives::approval::{ - AssignmentCertKind, VRFOutput, VRFProof, RELAY_VRF_MODULO_CONTEXT, + AssignmentCertKind, VrfOutput, VrfProof, VrfSignature, RELAY_VRF_MODULO_CONTEXT, }; use polkadot_node_subsystem::messages::{network_bridge_event, AllMessages, ApprovalCheckError}; use polkadot_node_subsystem_test_helpers as test_helpers; @@ -265,7 +265,7 @@ fn fake_assignment_cert(block_hash: Hash, validator: ValidatorIndex) -> Indirect validator, cert: AssignmentCert { kind: AssignmentCertKind::RelayVRFModulo { sample: 1 }, - vrf: (VRFOutput(out), VRFProof(proof)), + vrf: VrfSignature { output: VrfOutput(out), proof: VrfProof(proof) }, }, } } @@ -1817,6 +1817,9 @@ fn originator_aggression_l1() { session: 1, }; + let msg = ApprovalDistributionMessage::ApprovalCheckingLagUpdate(level + 1); + overseer_send(overseer, msg).await; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; @@ -2071,6 +2074,9 @@ fn non_originator_aggression_l2() { session: 1, }; + let msg = ApprovalDistributionMessage::ApprovalCheckingLagUpdate(level + 1); + overseer_send(overseer, msg).await; + let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; @@ -2097,6 +2103,10 @@ fn non_originator_aggression_l2() { session: 1, }; + let msg = ApprovalDistributionMessage::ApprovalCheckingLagUpdate( + aggression_l1_threshold + level + 1, + ); + overseer_send(overseer, msg).await; let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; @@ -2241,6 +2251,8 @@ fn resends_messages_periodically() { session: 1, }; + let msg = ApprovalDistributionMessage::ApprovalCheckingLagUpdate(2); + overseer_send(overseer, msg).await; let msg = ApprovalDistributionMessage::NewBlocks(vec![meta]); overseer_send(overseer, msg).await; diff --git a/node/primitives/Cargo.toml b/node/primitives/Cargo.toml index 196230014908..ce77a9fc2ecd 100644 --- a/node/primitives/Cargo.toml +++ b/node/primitives/Cargo.toml @@ -12,7 +12,6 @@ polkadot-primitives = { path = "../../primitives" } parity-scale-codec = { version = "3.4.0", default-features = false, features = ["derive"] } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master" } -sp-consensus-vrf = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-maybe-compressed-blob = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/primitives/src/approval.rs b/node/primitives/src/approval.rs index 67a88bf60020..1693fa68df0b 100644 --- a/node/primitives/src/approval.rs +++ b/node/primitives/src/approval.rs @@ -16,8 +16,7 @@ //! Types relevant for approval. -pub use sp_consensus_babe::Slot; -pub use sp_consensus_vrf::schnorrkel::{Randomness, VRFOutput, VRFProof}; +pub use sp_consensus_babe::{Randomness, Slot, VrfOutput, VrfProof, VrfSignature, VrfTranscript}; use parity_scale_codec::{Decode, Encode}; use polkadot_primitives::{ @@ -82,8 +81,8 @@ pub enum AssignmentCertKind { pub struct AssignmentCert { /// The criterion which is claimed to be met by this cert. pub kind: AssignmentCertKind, - /// The VRF showing the criterion is met. - pub vrf: (VRFOutput, VRFProof), + /// The VRF signature showing the criterion is met. + pub vrf: VrfSignature, } /// An assignment criterion which refers to the candidate under which the assignment is @@ -144,7 +143,7 @@ pub enum ApprovalError { /// An unsafe VRF output. Provide BABE Epoch info to create a `RelayVRFStory`. pub struct UnsafeVRFOutput { - vrf_output: VRFOutput, + vrf_output: VrfOutput, slot: Slot, authority_index: u32, } @@ -170,12 +169,12 @@ impl UnsafeVRFOutput { let pubkey = schnorrkel::PublicKey::from_bytes(author.as_slice()) .map_err(ApprovalError::SchnorrkelSignature)?; - let transcript = babe_primitives::make_transcript(randomness, self.slot, epoch_index); + let transcript = sp_consensus_babe::make_transcript(randomness, self.slot, epoch_index); let inout = self .vrf_output .0 - .attach_input_hash(&pubkey, transcript) + .attach_input_hash(&pubkey, transcript.0) .map_err(ApprovalError::SchnorrkelSignature)?; Ok(RelayVRFStory(inout.make_bytes(RELAY_VRF_STORY_CONTEXT))) } @@ -187,21 +186,18 @@ impl UnsafeVRFOutput { /// the digest has type `SecondaryPlain`, which Substrate nodes do /// not produce or accept anymore. pub fn babe_unsafe_vrf_info(header: &Header) -> Option { - use babe_primitives::digests::{CompatibleDigestItem, PreDigest}; + use babe_primitives::digests::CompatibleDigestItem; for digest in &header.digest.logs { if let Some(pre) = digest.as_babe_pre_digest() { let slot = pre.slot(); let authority_index = pre.authority_index(); - // exhaustive match to defend against upstream variant changes. - let vrf_output = match pre { - PreDigest::Primary(primary) => primary.vrf_output, - PreDigest::SecondaryVRF(secondary) => secondary.vrf_output, - PreDigest::SecondaryPlain(_) => return None, - }; - - return Some(UnsafeVRFOutput { vrf_output, slot, authority_index }) + return pre.vrf_signature().map(|sig| UnsafeVRFOutput { + vrf_output: sig.output.clone(), + slot, + authority_index, + }) } } diff --git a/node/service/Cargo.toml b/node/service/Cargo.toml index abcea3cc033c..0fbb550753d1 100644 --- a/node/service/Cargo.toml +++ b/node/service/Cargo.toml @@ -68,13 +68,13 @@ frame-benchmarking-cli = { git = "https://github.com/paritytech/substrate", bran # External Crates futures = "0.3.21" -hex-literal = "0.3.4" +hex-literal = "0.4.1" gum = { package = "tracing-gum", path = "../gum/" } serde = { version = "1.0.137", features = ["derive"] } -serde_json = "1.0.81" +serde_json = "1.0.96" thiserror = "1.0.31" kvdb = "0.13.0" -kvdb-rocksdb = { version = "0.17.0", optional = true } +kvdb-rocksdb = { version = "0.18.0", optional = true } parity-db = { version = "0.4.6", optional = true } async-trait = "0.1.57" diff --git a/node/service/chain-specs/kusama.json b/node/service/chain-specs/kusama.json index b917a1fb12d0..be7cf64923de 100644 --- a/node/service/chain-specs/kusama.json +++ b/node/service/chain-specs/kusama.json @@ -36,7 +36,9 @@ "/dns/boot-kusama.metaspan.io/tcp/23015/ws/p2p/12D3KooWE1tq9ZL9AAxMiUBBqy1ENmh5pwfWabnoBPMo8gFPXhn6", "/dns/boot-kusama.metaspan.io/tcp/23016/wss/p2p/12D3KooWE1tq9ZL9AAxMiUBBqy1ENmh5pwfWabnoBPMo8gFPXhn6", "/dns/kusama-bootnode.turboflakes.io/tcp/30305/p2p/12D3KooWR6cMhCYRhbJdqYZfzWZT6bcck3unpRLk8GBQGmHBgPwu", - "/dns/kusama-bootnode.turboflakes.io/tcp/30405/wss/p2p/12D3KooWR6cMhCYRhbJdqYZfzWZT6bcck3unpRLk8GBQGmHBgPwu" + "/dns/kusama-bootnode.turboflakes.io/tcp/30405/wss/p2p/12D3KooWR6cMhCYRhbJdqYZfzWZT6bcck3unpRLk8GBQGmHBgPwu", + "/dns/kusama-boot-ng.dwellir.com/tcp/443/wss/p2p/12D3KooWLswepVYVdCNduvWRTyNTaDMXEBcmvJdZ9Bhw3u2Jhad2", + "/dns/kusama-boot-ng.dwellir.com/tcp/30334/p2p/12D3KooWLswepVYVdCNduvWRTyNTaDMXEBcmvJdZ9Bhw3u2Jhad2" ], "telemetryEndpoints": [ [ diff --git a/node/service/chain-specs/polkadot.json b/node/service/chain-specs/polkadot.json index cc7d326625b9..ed7050b5ff49 100644 --- a/node/service/chain-specs/polkadot.json +++ b/node/service/chain-specs/polkadot.json @@ -36,7 +36,9 @@ "/dns/boot-polkadot.metaspan.io/tcp/13015/ws/p2p/12D3KooWRjHFApinuqSBjoaDjQHvxwubQSpEVy5hrgC9Smvh92WF", "/dns/boot-polkadot.metaspan.io/tcp/13016/wss/p2p/12D3KooWRjHFApinuqSBjoaDjQHvxwubQSpEVy5hrgC9Smvh92WF", "/dns/polkadot-bootnode.turboflakes.io/tcp/30300/p2p/12D3KooWHJBMZgt7ymAdTRtadPcGXpJw79vBGe8z53r9JMkZW7Ha", - "/dns/polkadot-bootnode.turboflakes.io/tcp/30400/wss/p2p/12D3KooWHJBMZgt7ymAdTRtadPcGXpJw79vBGe8z53r9JMkZW7Ha" + "/dns/polkadot-bootnode.turboflakes.io/tcp/30400/wss/p2p/12D3KooWHJBMZgt7ymAdTRtadPcGXpJw79vBGe8z53r9JMkZW7Ha", + "/dns/polkadot-boot-ng.dwellir.com/tcp/443/wss/p2p/12D3KooWFFqjBKoSdQniRpw1Y8W6kkV7takWv1DU2ZMkaA81PYVq", + "/dns/polkadot-boot-ng.dwellir.com/tcp/30336/p2p/12D3KooWFFqjBKoSdQniRpw1Y8W6kkV7takWv1DU2ZMkaA81PYVq" ], "telemetryEndpoints": [ [ diff --git a/node/service/chain-specs/rococo.json b/node/service/chain-specs/rococo.json index ba1ad29cdd82..43dc959b5767 100644 --- a/node/service/chain-specs/rococo.json +++ b/node/service/chain-specs/rococo.json @@ -3,14 +3,22 @@ "id": "rococo_v2_2", "chainType": "Live", "bootNodes": [ - "/ip4/34.90.151.124/tcp/30333/p2p/12D3KooWF7BUbG5ErMZ47ZdarRwtpZamgcZqxwpnFzkhjc1spHnP", - "/ip4/34.90.137.14/tcp/30333/p2p/12D3KooWLcpkpvjr5ccgtUdTSYtNDjEdsDcPNrt2Rb7yXuAf7bUE", - "/ip4/35.204.67.254/tcp/30333/p2p/12D3KooWGjEEDmNbBkXLM1uKMseK9iYD3osKA4JGdGKMZDCusjd6", - "/ip4/34.90.121.39/tcp/30333/p2p/12D3KooWBhkZQydNHDR3XSehnrfj1KNFCdpwgDrYpX54FrUR1FRS", - "/ip4/34.91.145.35/tcp/30333/p2p/12D3KooWBuLAMevZexnFKCgTyoz3AnHQn98D9cfe1Mg3kPoCjkwf", - "/ip4/34.91.77.80/tcp/30333/p2p/12D3KooWA5BAM71y9NtV5NH6EjANgYKRZ8jNLJ5z8GJ5RPdjt63n", - "/ip4/34.91.84.25/tcp/30333/p2p/12D3KooWSV4VqhBHZKKBsZKmVU462qRW9PmXTSuYvuajt1P93djA", - "/ip4/34.91.97.19/tcp/30333/p2p/12D3KooWD6wC88atMMyVeP6ZKg9sK7QmUL8x8m1RxMW8rhv2vWyg" + "/dns/rococo-bootnode-0.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWGikJMBmRiG5ofCqn8aijCijgfmZR5H9f53yUF3srm6Nm", + "/dns/rococo-bootnode-1.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWLDfH9mHRCidrd5NfQjp7rRMUcJSEUwSvEKyu7xU2cG3d", + "/dns/rococo-bootnode-2.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWSikgbrcWjVgSed7r1uXk4TeAieDnHKtrPDVZBu5XkQha", + "/dns/rococo-bootnode-3.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWPeKuW1BBPv4pNr8xqEv7jqy7rQnS3oq9U7xTCvj9qt2k", + "/dns/rococo-bootnode-4.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWNy7K8TNaP2Whcp3tsjBVUg2HcKMUvAArsimjvd1g31w4", + "/dns/rococo-bootnode-5.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWAVV9DZfvJp2brvs5zcQDTBFxNmEFJKy2dsvezWL4Bmy8", + "/dns/rococo-bootnode-6.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWM3hvXvaShyp7drQCavFHuwobkYdnCp2uHU5iRRAQwsw2", + "/dns/rococo-bootnode-7.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWSbGtxfWCwn1tdmfZYESbmxzbTG2LKwKUrioDaZBcdMY4", + "/dns/rococo-bootnode-0.polkadot.io/tcp/443/wss/p2p/12D3KooWGikJMBmRiG5ofCqn8aijCijgfmZR5H9f53yUF3srm6Nm", + "/dns/rococo-bootnode-1.polkadot.io/tcp/443/wss/p2p/12D3KooWLDfH9mHRCidrd5NfQjp7rRMUcJSEUwSvEKyu7xU2cG3d", + "/dns/rococo-bootnode-2.polkadot.io/tcp/443/wss/p2p/12D3KooWSikgbrcWjVgSed7r1uXk4TeAieDnHKtrPDVZBu5XkQha", + "/dns/rococo-bootnode-3.polkadot.io/tcp/443/wss/p2p/12D3KooWPeKuW1BBPv4pNr8xqEv7jqy7rQnS3oq9U7xTCvj9qt2k", + "/dns/rococo-bootnode-4.polkadot.io/tcp/443/wss/p2p/12D3KooWNy7K8TNaP2Whcp3tsjBVUg2HcKMUvAArsimjvd1g31w4", + "/dns/rococo-bootnode-5.polkadot.io/tcp/443/wss/p2p/12D3KooWAVV9DZfvJp2brvs5zcQDTBFxNmEFJKy2dsvezWL4Bmy8", + "/dns/rococo-bootnode-6.polkadot.io/tcp/443/wss/p2p/12D3KooWM3hvXvaShyp7drQCavFHuwobkYdnCp2uHU5iRRAQwsw2", + "/dns/rococo-bootnode-7.polkadot.io/tcp/443/wss/p2p/12D3KooWSbGtxfWCwn1tdmfZYESbmxzbTG2LKwKUrioDaZBcdMY4" ], "telemetryEndpoints": [ [ @@ -208,4 +216,4 @@ "childrenDefault": {} } } -} \ No newline at end of file +} diff --git a/node/service/chain-specs/westend.json b/node/service/chain-specs/westend.json index 43c7333f8a26..299a1d13418b 100644 --- a/node/service/chain-specs/westend.json +++ b/node/service/chain-specs/westend.json @@ -27,7 +27,9 @@ "/dns/boot-westend.metaspan.io/tcp/33015/ws/p2p/12D3KooWNTau7iG4G9cUJSwwt2QJP1W88pUf2SgqsHjRU2RL8pfa", "/dns/boot-westend.metaspan.io/tcp/33016/wss/p2p/12D3KooWNTau7iG4G9cUJSwwt2QJP1W88pUf2SgqsHjRU2RL8pfa", "/dns/westend-bootnode.turboflakes.io/tcp/30310/p2p/12D3KooWJvPDCZmReU46ghpCMJCPVUvUCav4WQdKtXQhZgJdH6tZ", - "/dns/westend-bootnode.turboflakes.io/tcp/30410/wss/p2p/12D3KooWJvPDCZmReU46ghpCMJCPVUvUCav4WQdKtXQhZgJdH6tZ" + "/dns/westend-bootnode.turboflakes.io/tcp/30410/wss/p2p/12D3KooWJvPDCZmReU46ghpCMJCPVUvUCav4WQdKtXQhZgJdH6tZ", + "/dns/westend-boot-ng.dwellir.com/tcp/443/wss/p2p/12D3KooWJifoDhCL3swAKt7MWhFb7wLRFD9oG33AL3nAathmU24x", + "/dns/westend-boot-ng.dwellir.com/tcp/30335/p2p/12D3KooWJifoDhCL3swAKt7MWhFb7wLRFD9oG33AL3nAathmU24x" ], "telemetryEndpoints": [ [ diff --git a/node/service/src/lib.rs b/node/service/src/lib.rs index 86bae65a268b..d25b0e1a0767 100644 --- a/node/service/src/lib.rs +++ b/node/service/src/lib.rs @@ -104,8 +104,10 @@ pub use polkadot_client::{ pub use polkadot_primitives::{Block, BlockId, BlockNumber, CollatorPair, Hash, Id as ParaId}; pub use sc_client_api::{Backend, CallExecutor, ExecutionStrategy}; pub use sc_consensus::{BlockImport, LongestChain}; -use sc_executor::NativeElseWasmExecutor; pub use sc_executor::NativeExecutionDispatch; +use sc_executor::{ + HeapAllocStrategy, NativeElseWasmExecutor, WasmExecutor, DEFAULT_HEAP_ALLOC_STRATEGY, +}; pub use service::{ config::{DatabaseSource, PrometheusConfig}, ChainSpec, Configuration, Error as SubstrateServiceError, PruningMode, Role, RuntimeGenesis, @@ -404,12 +406,19 @@ where }) .transpose()?; - let executor = NativeElseWasmExecutor::::new( - config.wasm_method, - config.default_heap_pages, - config.max_runtime_instances, - config.runtime_cache_size, - ); + let heap_pages = config + .default_heap_pages + .map_or(DEFAULT_HEAP_ALLOC_STRATEGY, |h| HeapAllocStrategy::Static { extra_pages: h as _ }); + + let wasm = WasmExecutor::builder() + .with_execution_method(config.wasm_method) + .with_onchain_heap_alloc_strategy(heap_pages) + .with_offchain_heap_alloc_strategy(heap_pages) + .with_max_runtime_instances(config.max_runtime_instances) + .with_runtime_cache_size(config.runtime_cache_size) + .build(); + + let executor = NativeElseWasmExecutor::::new_with_wasm_executor(wasm); let (client, backend, keystore_container, task_manager) = service::new_full_parts::( @@ -522,7 +531,7 @@ where babe::block_import(babe_config.clone(), beefy_block_import, client.clone())?; let slot_duration = babe_link.config().slot_duration(); - let import_queue = babe::import_queue( + let (import_queue, babe_worker_handle) = babe::import_queue( babe_link.clone(), block_import.clone(), Some(Box::new(justification_import)), @@ -552,9 +561,6 @@ where Some(shared_authority_set.clone()), ); - let shared_epoch_changes = babe_link.epoch_changes().clone(); - let slot_duration = babe_config.slot_duration(); - let import_setup = (block_import, grandpa_link, babe_link, beefy_voter_links); let rpc_setup = shared_voter_state.clone(); @@ -576,8 +582,7 @@ where chain_spec: chain_spec.cloned_box(), deny_unsafe, babe: polkadot_rpc::BabeDeps { - babe_config: babe_config.clone(), - shared_epoch_changes: shared_epoch_changes.clone(), + babe_worker_handle: babe_worker_handle.clone(), keystore: keystore.clone(), }, grandpa: polkadot_rpc::GrandpaDeps { @@ -763,6 +768,7 @@ where basics.backend.clone(), overseer_handle.clone(), metrics, + Some(basics.task_manager.spawn_handle()), ) } else { SelectRelayChain::new_longest_chain(basics.backend.clone()) diff --git a/node/service/src/relay_chain_selection.rs b/node/service/src/relay_chain_selection.rs index 18d6866e049e..afc0ce320610 100644 --- a/node/service/src/relay_chain_selection.rs +++ b/node/service/src/relay_chain_selection.rs @@ -40,14 +40,16 @@ use consensus_common::{Error as ConsensusError, SelectChain}; use futures::channel::oneshot; use polkadot_node_primitives::MAX_FINALITY_LAG as PRIMITIVES_MAX_FINALITY_LAG; use polkadot_node_subsystem::messages::{ - ApprovalVotingMessage, ChainSelectionMessage, DisputeCoordinatorMessage, - HighestApprovedAncestorBlock, + ApprovalDistributionMessage, ApprovalVotingMessage, ChainSelectionMessage, + DisputeCoordinatorMessage, HighestApprovedAncestorBlock, }; use polkadot_node_subsystem_util::metrics::{self, prometheus}; use polkadot_overseer::{AllMessages, Handle}; use polkadot_primitives::{Block as PolkadotBlock, BlockNumber, Hash, Header as PolkadotHeader}; use std::sync::Arc; +pub use service::SpawnTaskHandle; + /// The maximum amount of unfinalized blocks we are willing to allow due to approval checking /// or disputes. /// @@ -162,13 +164,21 @@ where /// Create a new [`SelectRelayChain`] wrapping the given chain backend /// and a handle to the overseer. - pub fn new_with_overseer(backend: Arc, overseer: Handle, metrics: Metrics) -> Self { + pub fn new_with_overseer( + backend: Arc, + overseer: Handle, + metrics: Metrics, + spawn_handle: Option, + ) -> Self { gum::debug!(target: LOG_TARGET, "Using dispute aware relay-chain selection algorithm",); SelectRelayChain { longest_chain: sc_consensus::LongestChain::new(backend.clone()), selection: IsDisputesAwareWithOverseer::Yes(SelectRelayChainInner::new( - backend, overseer, metrics, + backend, + overseer, + metrics, + spawn_handle, )), } } @@ -219,6 +229,7 @@ pub struct SelectRelayChainInner { backend: Arc, overseer: OH, metrics: Metrics, + spawn_handle: Option, } impl SelectRelayChainInner @@ -228,8 +239,13 @@ where { /// Create a new [`SelectRelayChainInner`] wrapping the given chain backend /// and a handle to the overseer. - pub fn new(backend: Arc, overseer: OH, metrics: Metrics) -> Self { - SelectRelayChainInner { backend, overseer, metrics } + pub fn new( + backend: Arc, + overseer: OH, + metrics: Metrics, + spawn_handle: Option, + ) -> Self { + SelectRelayChainInner { backend, overseer, metrics, spawn_handle } } fn block_header(&self, hash: Hash) -> Result { @@ -267,6 +283,7 @@ where backend: self.backend.clone(), overseer: self.overseer.clone(), metrics: self.metrics.clone(), + spawn_handle: self.spawn_handle.clone(), } } } @@ -307,7 +324,7 @@ impl OverseerHandleT for Handle { impl SelectRelayChainInner where B: HeaderProviderProvider, - OH: OverseerHandleT, + OH: OverseerHandleT + 'static, { /// Get all leaves of the chain, i.e. block hashes that are suitable to /// build upon and have no suitable children. @@ -455,6 +472,26 @@ where let lag = initial_leaf_number.saturating_sub(subchain_number); self.metrics.note_approval_checking_finality_lag(lag); + // Messages sent to `approval-distrbution` are known to have high `ToF`, we need to spawn a task for sending + // the message to not block here and delay finality. + if let Some(spawn_handle) = &self.spawn_handle { + let mut overseer_handle = self.overseer.clone(); + let lag_update_task = async move { + overseer_handle + .send_msg( + ApprovalDistributionMessage::ApprovalCheckingLagUpdate(lag), + std::any::type_name::(), + ) + .await; + }; + + spawn_handle.spawn( + "approval-checking-lag-update", + Some("relay-chain-selection"), + Box::pin(lag_update_task), + ); + } + let (lag, subchain_head) = { // Prevent sending flawed data to the dispute-coordinator. if Some(subchain_block_descriptions.len() as _) != diff --git a/node/service/src/tests.rs b/node/service/src/tests.rs index d28391a42e51..758581e1e682 100644 --- a/node/service/src/tests.rs +++ b/node/service/src/tests.rs @@ -17,15 +17,16 @@ use super::{relay_chain_selection::*, *}; use futures::channel::oneshot::Receiver; -use polkadot_node_primitives::approval::{VRFOutput, VRFProof}; +use polkadot_node_primitives::approval::VrfSignature; use polkadot_node_subsystem::messages::{AllMessages, BlockDescription}; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_test_client::Sr25519Keyring; use sp_consensus_babe::{ digests::{CompatibleDigestItem, PreDigest, SecondaryVRFPreDigest}, - Transcript, + VrfTranscript, }; +use sp_core::{crypto::VrfSigner, testing::TaskExecutor}; use sp_runtime::{testing::*, DigestItem}; use std::{ collections::{BTreeMap, HashMap, HashSet}, @@ -74,7 +75,7 @@ fn test_harness>( .filter_level(log::LevelFilter::Trace) .try_init(); - let pool = sp_core::testing::TaskExecutor::new(); + let pool = TaskExecutor::new(); let (mut context, virtual_overseer) = test_helpers::make_subsystem_context(pool); let (finality_target_tx, finality_target_rx) = oneshot::channel::>(); @@ -83,6 +84,7 @@ fn test_harness>( Arc::new(case_vars.chain.clone()), context.sender().clone(), Default::default(), + None, ); let target_hash = case_vars.target_block.clone(); @@ -99,7 +101,6 @@ fn test_harness>( futures::pin_mut!(test_fut); futures::pin_mut!(selection_process); - futures::executor::block_on(future::join( async move { let _overseer = test_fut.await; @@ -129,12 +130,9 @@ async fn overseer_recv_with_timeout( const TIMEOUT: Duration = Duration::from_millis(2000); // used for generating assignments where the validity of the VRF doesn't matter. -fn garbage_vrf() -> (VRFOutput, VRFProof) { - let key = Sr25519Keyring::Alice.pair(); - let key = key.as_ref(); - - let (o, p, _) = key.vrf_sign(Transcript::new(b"test-garbage")); - (VRFOutput(o.to_output()), VRFProof(p)) +fn garbage_vrf_signature() -> VrfSignature { + let transcript = VrfTranscript::new(b"test-garbage", &[]); + Sr25519Keyring::Alice.pair().vrf_sign(&transcript) } /// Representation of a local representation @@ -316,13 +314,12 @@ impl ChainBuilder { fn make_header(parent_hash: Hash, number: u32) -> Header { let digest = { let mut digest = Digest::default(); - let (vrf_output, vrf_proof) = garbage_vrf(); + let vrf_signature = garbage_vrf_signature(); digest.push(DigestItem::babe_pre_digest(PreDigest::SecondaryVRF( SecondaryVRFPreDigest { authority_index: 0, slot: 1.into(), // slot, unused - vrf_output, - vrf_proof, + vrf_signature, }, ))); digest diff --git a/node/subsystem-types/src/messages.rs b/node/subsystem-types/src/messages.rs index ada7d9d8f09d..ee187c9f41a4 100644 --- a/node/subsystem-types/src/messages.rs +++ b/node/subsystem-types/src/messages.rs @@ -851,6 +851,8 @@ pub enum ApprovalDistributionMessage { HashSet<(Hash, CandidateIndex)>, oneshot::Sender>, ), + /// Approval checking lag update measured in blocks. + ApprovalCheckingLagUpdate(BlockNumber), } /// Message to the Gossip Support subsystem. diff --git a/node/test/performance-test/Cargo.toml b/node/test/performance-test/Cargo.toml index b2b2ef8148f2..70f072c03ae1 100644 --- a/node/test/performance-test/Cargo.toml +++ b/node/test/performance-test/Cargo.toml @@ -6,15 +6,18 @@ edition.workspace = true [dependencies] thiserror = "1.0.31" -quote = "1.0.20" +quote = "1.0.26" env_logger = "0.9" log = "0.4" -polkadot-node-core-pvf = { path = "../../core/pvf" } +polkadot-node-core-pvf-worker = { path = "../../core/pvf/worker" } polkadot-erasure-coding = { path = "../../../erasure-coding" } polkadot-node-primitives = { path = "../../primitives" } polkadot-primitives = { path = "../../../primitives" } +sc-executor-common = { git = "https://github.com/paritytech/substrate", branch = "master" } +sp-maybe-compressed-blob = { git = "https://github.com/paritytech/substrate", branch = "master" } + kusama-runtime = { path = "../../../runtime/kusama" } [[bin]] diff --git a/node/test/performance-test/src/gen_ref_constants.rs b/node/test/performance-test/src/gen_ref_constants.rs index 0f06af1580ef..ba10ed215552 100644 --- a/node/test/performance-test/src/gen_ref_constants.rs +++ b/node/test/performance-test/src/gen_ref_constants.rs @@ -31,7 +31,6 @@ fn main() -> Result<(), PerfCheckError> { #[cfg(build_type = "release")] mod run { - use polkadot_node_core_pvf::sp_maybe_compressed_blob; use polkadot_node_primitives::VALIDATION_CODE_BOMB_LIMIT; use polkadot_performance_test::{ measure_erasure_coding, measure_pvf_prepare, PerfCheckError, ERASURE_CODING_N_VALIDATORS, diff --git a/node/test/performance-test/src/lib.rs b/node/test/performance-test/src/lib.rs index e426cc4e5142..1afa43cc62ba 100644 --- a/node/test/performance-test/src/lib.rs +++ b/node/test/performance-test/src/lib.rs @@ -17,7 +17,6 @@ //! A Polkadot performance tests utilities. use polkadot_erasure_coding::{obtain_chunks, reconstruct}; -use polkadot_node_core_pvf::{sc_executor_common, sp_maybe_compressed_blob}; use polkadot_primitives::ExecutorParams; use std::time::{Duration, Instant}; @@ -66,8 +65,9 @@ pub fn measure_pvf_prepare(wasm_code: &[u8]) -> Result .or(Err(PerfCheckError::CodeDecompressionFailed))?; // Recreate the pipeline from the pvf prepare worker. - let blob = polkadot_node_core_pvf::prevalidate(code.as_ref()).map_err(PerfCheckError::from)?; - polkadot_node_core_pvf::prepare(blob, &ExecutorParams::default()) + let blob = + polkadot_node_core_pvf_worker::prevalidate(code.as_ref()).map_err(PerfCheckError::from)?; + polkadot_node_core_pvf_worker::prepare(blob, &ExecutorParams::default()) .map_err(PerfCheckError::from)?; Ok(start.elapsed()) diff --git a/node/test/service/Cargo.toml b/node/test/service/Cargo.toml index 590d4056d8c7..8bf8ba6976bf 100644 --- a/node/test/service/Cargo.toml +++ b/node/test/service/Cargo.toml @@ -58,7 +58,7 @@ substrate-test-client = { git = "https://github.com/paritytech/substrate", branc [dev-dependencies] pallet-balances = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } -serde_json = "1.0.81" +serde_json = "1.0.96" substrate-test-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } tokio = { version = "1.24.2", features = ["macros"] } diff --git a/parachain/test-parachains/adder/collator/Cargo.toml b/parachain/test-parachains/adder/collator/Cargo.toml index 7fe4aefc688d..ee20cb0b0d17 100644 --- a/parachain/test-parachains/adder/collator/Cargo.toml +++ b/parachain/test-parachains/adder/collator/Cargo.toml @@ -34,7 +34,7 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master # This one is tricky. Even though it is not used directly by the collator, we still need it for the # `puppet_worker` binary, which is required for the integration test. However, this shouldn't be # a big problem since it is used transitively anyway. -polkadot-node-core-pvf = { path = "../../../../node/core/pvf" } +polkadot-node-core-pvf-worker = { path = "../../../../node/core/pvf/worker" } [dev-dependencies] polkadot-parachain = { path = "../../.." } diff --git a/parachain/test-parachains/adder/collator/bin/puppet_worker.rs b/parachain/test-parachains/adder/collator/bin/puppet_worker.rs index 7f93519d8454..ddd81971292b 100644 --- a/parachain/test-parachains/adder/collator/bin/puppet_worker.rs +++ b/parachain/test-parachains/adder/collator/bin/puppet_worker.rs @@ -14,4 +14,4 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -polkadot_node_core_pvf::decl_puppet_worker_main!(); +polkadot_node_core_pvf_worker::decl_puppet_worker_main!(); diff --git a/parachain/test-parachains/adder/collator/src/lib.rs b/parachain/test-parachains/adder/collator/src/lib.rs index 02a4598f9e47..4b2b9248de22 100644 --- a/parachain/test-parachains/adder/collator/src/lib.rs +++ b/parachain/test-parachains/adder/collator/src/lib.rs @@ -272,7 +272,7 @@ mod tests { } fn validate_collation(collator: &Collator, parent_head: HeadData, collation: Collation) { - use polkadot_node_core_pvf::testing::validate_candidate; + use polkadot_node_core_pvf_worker::testing::validate_candidate; let block_data = match collation.proof_of_validity { MaybeCompressedPoV::Raw(pov) => pov.block_data, diff --git a/parachain/test-parachains/undying/collator/Cargo.toml b/parachain/test-parachains/undying/collator/Cargo.toml index 2b9d80401f5d..1b2ccf3be0ca 100644 --- a/parachain/test-parachains/undying/collator/Cargo.toml +++ b/parachain/test-parachains/undying/collator/Cargo.toml @@ -34,7 +34,7 @@ sc-service = { git = "https://github.com/paritytech/substrate", branch = "master # This one is tricky. Even though it is not used directly by the collator, we still need it for the # `puppet_worker` binary, which is required for the integration test. However, this shouldn't be # a big problem since it is used transitively anyway. -polkadot-node-core-pvf = { path = "../../../../node/core/pvf" } +polkadot-node-core-pvf-worker = { path = "../../../../node/core/pvf/worker" } [dev-dependencies] polkadot-parachain = { path = "../../.." } diff --git a/parachain/test-parachains/undying/collator/bin/puppet_worker.rs b/parachain/test-parachains/undying/collator/bin/puppet_worker.rs index 7f93519d8454..ddd81971292b 100644 --- a/parachain/test-parachains/undying/collator/bin/puppet_worker.rs +++ b/parachain/test-parachains/undying/collator/bin/puppet_worker.rs @@ -14,4 +14,4 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -polkadot_node_core_pvf::decl_puppet_worker_main!(); +polkadot_node_core_pvf_worker::decl_puppet_worker_main!(); diff --git a/parachain/test-parachains/undying/collator/src/lib.rs b/parachain/test-parachains/undying/collator/src/lib.rs index 838590fa16f5..dcaf9b63296d 100644 --- a/parachain/test-parachains/undying/collator/src/lib.rs +++ b/parachain/test-parachains/undying/collator/src/lib.rs @@ -354,7 +354,7 @@ mod tests { } fn validate_collation(collator: &Collator, parent_head: HeadData, collation: Collation) { - use polkadot_node_core_pvf::testing::validate_candidate; + use polkadot_node_core_pvf_worker::testing::validate_candidate; let block_data = match collation.proof_of_validity { MaybeCompressedPoV::Raw(pov) => pov.block_data, diff --git a/primitives/Cargo.toml b/primitives/Cargo.toml index 9e459fb1773f..6ce968230724 100644 --- a/primitives/Cargo.toml +++ b/primitives/Cargo.toml @@ -6,7 +6,7 @@ edition.workspace = true [dependencies] bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } -hex-literal = "0.3.4" +hex-literal = "0.4.1" parity-scale-codec = { version = "3.4.0", default-features = false, features = ["bit-vec", "derive"] } scale-info = { version = "2.5.0", default-features = false, features = ["bit-vec", "derive"] } serde = { version = "1.0.137", optional = true, features = ["derive"] } diff --git a/primitives/src/v4/executor_params.rs b/primitives/src/v4/executor_params.rs index badaf3b44a66..0cdeb44704b2 100644 --- a/primitives/src/v4/executor_params.rs +++ b/primitives/src/v4/executor_params.rs @@ -29,6 +29,7 @@ use sp_std::{ops::Deref, time::Duration, vec, vec::Vec}; /// The different executor parameters for changing the execution environment semantics. #[derive(Clone, Debug, Encode, Decode, PartialEq, Eq, TypeInfo)] +#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] pub enum ExecutorParam { /// Maximum number of memory pages (64KiB bytes per page) the executor can allocate. #[codec(index = 1)] @@ -93,6 +94,7 @@ impl sp_std::fmt::LowerHex for ExecutorParamsHash { // new entries and removing old ones. At the moment, there's no mandatory parameters defined. If // they show up, they must be clearly documented as mandatory ones. #[derive(Clone, Debug, Encode, Decode, PartialEq, Eq, TypeInfo)] +#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] pub struct ExecutorParams(Vec); impl ExecutorParams { diff --git a/primitives/src/v4/mod.rs b/primitives/src/v4/mod.rs index 6b8434cbbe05..56fe856d1504 100644 --- a/primitives/src/v4/mod.rs +++ b/primitives/src/v4/mod.rs @@ -1708,6 +1708,7 @@ impl PvfCheckStatement { /// Type discriminator for PVF preparation timeouts #[derive(Encode, Decode, TypeInfo, Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] pub enum PvfPrepTimeoutKind { /// For prechecking requests, the time period after which the preparation worker is considered /// unresponsive and will be killed. @@ -1721,6 +1722,7 @@ pub enum PvfPrepTimeoutKind { /// Type discriminator for PVF execution timeouts #[derive(Encode, Decode, TypeInfo, Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] pub enum PvfExecTimeoutKind { /// The amount of time to spend on execution during backing. Backing, diff --git a/roadmap/implementers-guide/src/node/utility/candidate-validation.md b/roadmap/implementers-guide/src/node/utility/candidate-validation.md index 6e7a5f3d0c8f..bc1d11d8f664 100644 --- a/roadmap/implementers-guide/src/node/utility/candidate-validation.md +++ b/roadmap/implementers-guide/src/node/utility/candidate-validation.md @@ -23,7 +23,7 @@ Upon receiving a validation request, the first thing the candidate validation su * The [`CandidateDescriptor`](../../types/candidate.md#candidatedescriptor). * The [`ValidationData`](../../types/candidate.md#validationdata). * The [`PoV`](../../types/availability.md#proofofvalidity). - + The second category is for PVF pre-checking. This is primarly used by the [PVF pre-checker](pvf-prechecker.md) subsystem. ### Determining Parameters @@ -67,8 +67,20 @@ or time out). We will only retry preparation if another request comes in after resolved. We will retry up to 5 times. If the actual **execution** of the artifact fails, we will retry once if it was -an ambiguous error after a brief delay, to allow any potential transient -conditions to clear. +a possibly transient error, to allow the conditions that led to the error to +hopefully resolve. We use a more brief delay here (1 second as opposed to 15 +minutes for preparation (see above)), because a successful execution must happen +in a short amount of time. + +We currently know of at least two specific cases that will lead to a retried +execution request: + +1. **OOM:** The host might have been temporarily low on memory due to other + processes running on the same machine. **NOTE:** This case will lead to + voting against the candidate (and possibly a dispute) if the retry is still + not successful. +2. **Artifact missing:** The prepared artifact might have been deleted due to + operator error or some bug in the system. We will re-create it on retry. #### Preparation timeouts diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index e60afdc5acea..c08e28799c54 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -23,7 +23,6 @@ use std::sync::Arc; use jsonrpsee::RpcModule; use polkadot_primitives::{AccountId, Balance, Block, BlockNumber, Hash, Nonce}; use sc_client_api::AuxStore; -use sc_consensus_babe::{BabeConfiguration, Epoch}; use sc_consensus_beefy::communication::notification::{ BeefyBestBlockStream, BeefyVersionedFinalityProofStream, }; @@ -42,10 +41,8 @@ pub type RpcExtension = RpcModule<()>; /// Extra dependencies for BABE. pub struct BabeDeps { - /// BABE protocol config. - pub babe_config: BabeConfiguration, - /// BABE pending epoch changes. - pub shared_epoch_changes: sc_consensus_epochs::SharedEpochChanges, + /// A handle to the BABE worker for issuing requests. + pub babe_worker_handle: sc_consensus_babe::BabeWorkerHandle, /// The keystore that manages the keys of the node. pub keystore: KeystorePtr, } @@ -129,7 +126,7 @@ where let mut io = RpcModule::new(()); let FullDeps { client, pool, select_chain, chain_spec, deny_unsafe, babe, grandpa, beefy } = deps; - let BabeDeps { keystore, babe_config, shared_epoch_changes } = babe; + let BabeDeps { babe_worker_handle, keystore } = babe; let GrandpaDeps { shared_voter_state, shared_authority_set, @@ -143,15 +140,8 @@ where io.merge(TransactionPayment::new(client.clone()).into_rpc())?; io.merge(Mmr::new(client.clone()).into_rpc())?; io.merge( - Babe::new( - client.clone(), - shared_epoch_changes.clone(), - keystore, - babe_config, - select_chain, - deny_unsafe, - ) - .into_rpc(), + Babe::new(client.clone(), babe_worker_handle.clone(), keystore, select_chain, deny_unsafe) + .into_rpc(), )?; io.merge( Grandpa::new( @@ -164,7 +154,7 @@ where .into_rpc(), )?; io.merge( - SyncState::new(chain_spec, client, shared_authority_set, shared_epoch_changes)?.into_rpc(), + SyncState::new(chain_spec, client, shared_authority_set, babe_worker_handle)?.into_rpc(), )?; io.merge( diff --git a/runtime/common/Cargo.toml b/runtime/common/Cargo.toml index c065c09bd9e8..8cb7aa7f9808 100644 --- a/runtime/common/Cargo.toml +++ b/runtime/common/Cargo.toml @@ -51,12 +51,12 @@ slot-range-helper = { path = "slot_range_helper", default-features = false } xcm = { path = "../../xcm", default-features = false } [dev-dependencies] -hex-literal = "0.3.4" +hex-literal = "0.4.1" frame-support-test = { git = "https://github.com/paritytech/substrate", branch = "master" } pallet-babe = { git = "https://github.com/paritytech/substrate", branch = "master" } pallet-treasury = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } -serde_json = "1.0.81" +serde_json = "1.0.96" libsecp256k1 = "0.7.0" test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../primitives/test-helpers" } diff --git a/runtime/common/src/xcm_sender.rs b/runtime/common/src/xcm_sender.rs index 03120ccf704a..3654c44010e9 100644 --- a/runtime/common/src/xcm_sender.rs +++ b/runtime/common/src/xcm_sender.rs @@ -21,8 +21,9 @@ use parity_scale_codec::Encode; use primitives::Id as ParaId; use runtime_parachains::{ configuration::{self, HostConfiguration}, - dmp, + dmp, FeeTracker, }; +use sp_runtime::FixedPointNumber; use sp_std::{marker::PhantomData, prelude::*}; use xcm::prelude::*; use SendError::*; @@ -47,6 +48,24 @@ impl> PriceForParachainDelivery for ConstantPrice { } } +/// Implementation of `PriceForParachainDelivery` which returns an exponentially increasing price. +/// The `A` type parameter is used to denote the asset ID that will be used for paying the delivery +/// fee. +/// +/// The formula for the fee is based on the sum of a base fee plus a message length fee, multiplied +/// by a specified factor. In mathematical form, it is `F * (B + encoded_msg_len * M)`. +pub struct ExponentialPrice(sp_std::marker::PhantomData<(A, B, M, F)>); +impl, B: Get, M: Get, F: FeeTracker> PriceForParachainDelivery + for ExponentialPrice +{ + fn price_for_parachain_delivery(para: ParaId, msg: &Xcm<()>) -> MultiAssets { + let msg_fee = (msg.encoded_size() as u128).saturating_mul(M::get()); + let fee_sum = B::get().saturating_add(msg_fee); + let amount = F::get_fee_factor(para).saturating_mul_int(fee_sum); + (A::get(), amount).into() + } +} + /// XCM sender for relay chain. It only sends downward message. pub struct ChildParachainRouter(PhantomData<(T, W, P)>); @@ -88,3 +107,61 @@ impl FixedU128 { + FixedU128::from_rational(101, 100) + } + } + + type TestExponentialPrice = + ExponentialPrice; + + #[test] + fn exponential_price_correct_price_calculation() { + let id: ParaId = 123.into(); + let b: u128 = BaseDeliveryFee::get(); + let m: u128 = TransactionByteFee::get(); + + // F * (B + msg_length * M) + // message_length = 1 + let result: u128 = TestFeeTracker::get_fee_factor(id.clone()).saturating_mul_int(b + m); + assert_eq!( + TestExponentialPrice::price_for_parachain_delivery(id.clone(), &Xcm(vec![])), + (FeeAssetId::get(), result).into() + ); + + // message size = 2 + let result: u128 = + TestFeeTracker::get_fee_factor(id.clone()).saturating_mul_int(b + (2 * m)); + assert_eq!( + TestExponentialPrice::price_for_parachain_delivery(id.clone(), &Xcm(vec![ClearOrigin])), + (FeeAssetId::get(), result).into() + ); + + // message size = 4 + let result: u128 = + TestFeeTracker::get_fee_factor(id.clone()).saturating_mul_int(b + (4 * m)); + assert_eq!( + TestExponentialPrice::price_for_parachain_delivery( + id.clone(), + &Xcm(vec![SetAppendix(Xcm(vec![ClearOrigin]))]) + ), + (FeeAssetId::get(), result).into() + ); + } +} diff --git a/runtime/kusama/Cargo.toml b/runtime/kusama/Cargo.toml index a5af282b628e..4e082eb5acd7 100644 --- a/runtime/kusama/Cargo.toml +++ b/runtime/kusama/Cargo.toml @@ -24,7 +24,7 @@ sp-api = { git = "https://github.com/paritytech/substrate", branch = "master", d inherents = { package = "sp-inherents", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } offchain-primitives = { package = "sp-offchain", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-std = { package = "sp-std", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } -sp-arithmetic = { package = "sp-arithmetic", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } +sp-arithmetic = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-io = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-mmr-primitives = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } @@ -92,7 +92,7 @@ pallet-session-benchmarking = { git = "https://github.com/paritytech/substrate", pallet-nomination-pools-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true } frame-system-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true } pallet-election-provider-support-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true } -hex-literal = { version = "0.3.4", optional = true } +hex-literal = { version = "0.4.1", optional = true } runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false } runtime-parachains = { package = "polkadot-runtime-parachains", path = "../parachains", default-features = false } @@ -103,12 +103,12 @@ xcm-executor = { package = "xcm-executor", path = "../../xcm/xcm-executor", defa xcm-builder = { package = "xcm-builder", path = "../../xcm/xcm-builder", default-features = false } [dev-dependencies] -hex-literal = "0.3.4" +hex-literal = "0.4.1" tiny-keccak = { version = "2.0.2", features = ["keccak"] } keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate", branch = "master" } sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" } separator = "0.4.1" -serde_json = "1.0.81" +serde_json = "1.0.96" remote-externalities = { git = "https://github.com/paritytech/substrate", branch = "master", package = "frame-remote-externalities" } tokio = { version = "1.24.2", features = ["macros"] } sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } @@ -128,6 +128,7 @@ std = [ "parity-scale-codec/std", "scale-info/std", "inherents/std", + "sp-arithmetic/std", "sp-core/std", "sp-api/std", "tx-pool-api/std", diff --git a/runtime/kusama/src/lib.rs b/runtime/kusama/src/lib.rs index 7d52c946926c..79a5ce75f7ae 100644 --- a/runtime/kusama/src/lib.rs +++ b/runtime/kusama/src/lib.rs @@ -126,13 +126,13 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("kusama"), impl_name: create_runtime_str!("parity-kusama"), authoring_version: 2, - spec_version: 9390, + spec_version: 9410, impl_version: 0, #[cfg(not(feature = "disable-runtime-api"))] apis: RUNTIME_API_VERSIONS, #[cfg(feature = "disable-runtime-api")] apis: sp_version::create_apis_vec![[]], - transaction_version: 20, + transaction_version: 21, state_version: 0, }; @@ -303,7 +303,7 @@ impl pallet_balances::Config for Runtime { type WeightInfo = weights::pallet_balances::WeightInfo; type FreezeIdentifier = (); type MaxFreezes = (); - type HoldIdentifier = HoldReason; + type HoldIdentifier = RuntimeHoldReason; type MaxHolds = ConstU32<1>; } @@ -929,7 +929,7 @@ parameter_types! { Decode, RuntimeDebug, MaxEncodedLen, - scale_info::TypeInfo, + TypeInfo, )] pub enum ProxyType { Any, @@ -1258,27 +1258,7 @@ parameter_types! { pub const ThawThrottle: (Perquintill, BlockNumber) = (Perquintill::from_percent(25), 5); pub storage NisTarget: Perquintill = Perquintill::zero(); pub const NisPalletId: PalletId = PalletId(*b"py/nis "); - pub const NisHoldReason: HoldReason = HoldReason::Nis(HoldReasonNis::NftReceipt); -} - -/// A reason for placing a hold on funds. -#[derive( - Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Encode, Decode, MaxEncodedLen, Debug, TypeInfo, -)] -pub enum HoldReason { - /// Some reason of the NIS pallet. - #[codec(index = 38)] - Nis(HoldReasonNis), -} - -/// A reason for the NIS pallet placing a hold on funds. -#[derive( - Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Encode, Decode, MaxEncodedLen, Debug, TypeInfo, -)] -pub enum HoldReasonNis { - /// The NIS Pallet has reserved it for a non-fungible receipt. - #[codec(index = 0)] - NftReceipt = 0, + pub const NisHoldReason: RuntimeHoldReason = RuntimeHoldReason::Nis(pallet_nis::HoldReason::NftReceipt); } impl pallet_nis::Config for Runtime { @@ -1408,7 +1388,7 @@ construct_runtime! { ElectionProviderMultiPhase: pallet_election_provider_multi_phase::{Pallet, Call, Storage, Event, ValidateUnsigned} = 37, // NIS pallet. - Nis: pallet_nis::{Pallet, Call, Storage, Event} = 38, + Nis: pallet_nis::{Pallet, Call, Storage, Event, HoldReason} = 38, // pub type NisCounterpartInstance = pallet_balances::Instance2; NisCounterpartBalances: pallet_balances:: = 45, @@ -1430,7 +1410,7 @@ construct_runtime! { ParaScheduler: parachains_scheduler::{Pallet, Storage} = 55, Paras: parachains_paras::{Pallet, Call, Storage, Event, Config, ValidateUnsigned} = 56, Initializer: parachains_initializer::{Pallet, Call, Storage} = 57, - Dmp: parachains_dmp::{Pallet, Call, Storage} = 58, + Dmp: parachains_dmp::{Pallet, Storage} = 58, Ump: parachains_ump::{Pallet, Call, Storage, Event} = 59, Hrmp: parachains_hrmp::{Pallet, Call, Storage, Event, Config} = 60, ParaSessionInfo: parachains_session_info::{Pallet, Storage} = 61, @@ -1488,8 +1468,8 @@ pub type Migrations = ( Runtime, NominationPoolsMigrationV4OldPallet, >, - // Unreleased - add new migrations here: pallet_nomination_pools::migration::v5::MigrateToV5, + // Unreleased - add new migrations here: parachains_configuration::migration::v5::MigrateToV5, ); @@ -2155,7 +2135,7 @@ sp_api::impl_runtime_apis! { } #[cfg(test)] -mod tests_fess { +mod fees_tests { use super::*; use sp_runtime::assert_eq_error_rate; diff --git a/runtime/kusama/src/tests.rs b/runtime/kusama/src/tests.rs index c1bcf76ca8ee..b2dd5759f3a9 100644 --- a/runtime/kusama/src/tests.rs +++ b/runtime/kusama/src/tests.rs @@ -25,6 +25,11 @@ use runtime_common::MinimumMultiplier; use separator::Separatable; use sp_runtime::FixedPointNumber; +#[test] +fn nis_hold_reason_encoding_is_correct() { + assert_eq!(NisHoldReason::get().encode(), [38, 0]); +} + #[test] fn remove_keys_weight_is_sensible() { use runtime_common::crowdloan::WeightInfo; diff --git a/runtime/kusama/src/weights/runtime_parachains_configuration.rs b/runtime/kusama/src/weights/runtime_parachains_configuration.rs index b5a194796bf2..d5344bc8ea26 100644 --- a/runtime/kusama/src/weights/runtime_parachains_configuration.rs +++ b/runtime/kusama/src/weights/runtime_parachains_configuration.rs @@ -16,24 +16,26 @@ //! Autogenerated weights for `runtime_parachains::configuration` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-03-16, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2023-04-07, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `bm5`, CPU: `Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz` +//! HOSTNAME: `bm3`, CPU: `Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("kusama-dev"), DB CACHE: 1024 // Executed Command: -// ./target/production/polkadot +// target/production/polkadot // benchmark // pallet -// --chain=kusama-dev // --steps=50 // --repeat=20 -// --pallet=runtime_parachains::configuration // --extrinsic=* // --execution=wasm // --wasm-execution=compiled +// --heap-pages=4096 +// --json-file=/var/lib/gitlab-runner/builds/zyw4fam_/0/parity/mirrors/polkadot/.git/.artifacts/bench.json +// --pallet=runtime_parachains::configuration +// --chain=kusama-dev // --header=./file_header.txt -// --output=./runtime/kusama/src/weights/runtime_parachains_configuration.rs +// --output=./runtime/kusama/src/weights/ #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -55,8 +57,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `90` // Estimated: `4725` - // Minimum execution time: 8_970_000 picoseconds. - Weight::from_parts(9_143_000, 0) + // Minimum execution time: 9_394_000 picoseconds. + Weight::from_parts(9_686_000, 0) .saturating_add(Weight::from_parts(0, 4725)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -71,8 +73,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `90` // Estimated: `4725` - // Minimum execution time: 9_192_000 picoseconds. - Weight::from_parts(9_475_000, 0) + // Minimum execution time: 9_379_000 picoseconds. + Weight::from_parts(9_700_000, 0) .saturating_add(Weight::from_parts(0, 4725)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -87,8 +89,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `90` // Estimated: `4725` - // Minimum execution time: 9_259_000 picoseconds. - Weight::from_parts(9_499_000, 0) + // Minimum execution time: 9_612_000 picoseconds. + Weight::from_parts(16_194_000, 0) .saturating_add(Weight::from_parts(0, 4725)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -103,8 +105,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `90` // Estimated: `4725` - // Minimum execution time: 9_323_000 picoseconds. - Weight::from_parts(9_697_000, 0) + // Minimum execution time: 9_533_000 picoseconds. + Weight::from_parts(9_745_000, 0) .saturating_add(Weight::from_parts(0, 4725)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -129,8 +131,24 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `90` // Estimated: `4725` - // Minimum execution time: 9_270_000 picoseconds. - Weight::from_parts(9_527_000, 0) + // Minimum execution time: 9_463_000 picoseconds. + Weight::from_parts(9_852_000, 0) + .saturating_add(Weight::from_parts(0, 4725)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: Configuration PendingConfigs (r:1 w:1) + /// Proof Skipped: Configuration PendingConfigs (max_values: Some(1), max_size: None, mode: Measured) + /// Storage: Configuration BypassConsistencyCheck (r:1 w:0) + /// Proof Skipped: Configuration BypassConsistencyCheck (max_values: Some(1), max_size: None, mode: Measured) + /// Storage: ParasShared CurrentSessionIndex (r:1 w:0) + /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) + fn set_config_with_executor_params() -> Weight { + // Proof Size summary in bytes: + // Measured: `90` + // Estimated: `4725` + // Minimum execution time: 10_405_000 picoseconds. + Weight::from_parts(10_693_000, 0) .saturating_add(Weight::from_parts(0, 4725)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) diff --git a/runtime/kusama/src/xcm_config.rs b/runtime/kusama/src/xcm_config.rs index 289ea118d7bc..37def756000f 100644 --- a/runtime/kusama/src/xcm_config.rs +++ b/runtime/kusama/src/xcm_config.rs @@ -17,8 +17,9 @@ //! XCM configurations for the Kusama runtime. use super::{ - parachains_origin, AccountId, AllPalletsWithSystem, Balances, Fellows, ParaId, Runtime, - RuntimeCall, RuntimeEvent, RuntimeOrigin, StakingAdmin, WeightToFee, XcmPallet, + parachains_origin, AccountId, AllPalletsWithSystem, Balances, Dmp, Fellows, ParaId, Runtime, + RuntimeCall, RuntimeEvent, RuntimeOrigin, StakingAdmin, TransactionByteFee, WeightToFee, + XcmPallet, }; use frame_support::{ match_types, parameter_types, @@ -26,7 +27,12 @@ use frame_support::{ weights::Weight, }; use frame_system::EnsureRoot; -use runtime_common::{crowdloan, paras_registrar, xcm_sender, ToAuthor}; +use kusama_runtime_constants::currency::CENTS; +use runtime_common::{ + crowdloan, paras_registrar, + xcm_sender::{ChildParachainRouter, ExponentialPrice}, + ToAuthor, +}; use sp_core::ConstU32; use xcm::latest::prelude::*; use xcm_builder::{ @@ -101,13 +107,21 @@ parameter_types! { /// Maximum number of instructions in a single XCM fragment. A sanity check against weight /// calculations getting too crazy. pub const MaxInstructions: u32 = 100; + /// The asset ID for the asset that we use to pay for message delivery fees. + pub FeeAssetId: AssetId = Concrete(TokenLocation::get()); + /// The base fee for the message delivery fees. + pub const BaseDeliveryFee: u128 = CENTS.saturating_mul(3); } /// The XCM router. When we want to send an XCM message, we use this type. It amalgamates all of our /// individual routers. pub type XcmRouter = ( // Only one router so far - use DMP to communicate with child parachains. - xcm_sender::ChildParachainRouter, + ChildParachainRouter< + Runtime, + XcmPallet, + ExponentialPrice, + >, ); parameter_types! { @@ -303,7 +317,9 @@ impl Contains for SafeCallFilter { ) | RuntimeCall::XcmPallet(pallet_xcm::Call::limited_reserve_transfer_assets { .. - }) => true, + }) | + RuntimeCall::Whitelist(pallet_whitelist::Call::whitelist_call { .. }) | + RuntimeCall::Proxy(..) => true, _ => false, } } @@ -425,5 +441,5 @@ fn karura_liquid_staking_xcm_has_sane_weight_upper_limt() { let weight = ::Weigher::weight(&mut xcm) .expect("weighing XCM failed"); - assert_eq!(weight, Weight::from_parts(20_313_281_000, 65536)); + assert_eq!(weight, Weight::from_parts(20_313_281_000, 0)); } diff --git a/runtime/parachains/Cargo.toml b/runtime/parachains/Cargo.toml index 7f9b6bc9e422..a097fe4fa161 100644 --- a/runtime/parachains/Cargo.toml +++ b/runtime/parachains/Cargo.toml @@ -50,7 +50,7 @@ polkadot-runtime-metrics = { path = "../metrics", default-features = false} [dev-dependencies] futures = "0.3.21" -hex-literal = "0.3.4" +hex-literal = "0.4.1" keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate", branch = "master" } frame-support-test = { git = "https://github.com/paritytech/substrate", branch = "master" } sc-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" } @@ -58,7 +58,7 @@ test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../pri sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } thousands = "0.2.0" assert_matches = "1" -serde_json = "1.0.85" +serde_json = "1.0.96" [features] default = ["std"] diff --git a/runtime/parachains/src/configuration.rs b/runtime/parachains/src/configuration.rs index 802e4751e2d9..380ae4770851 100644 --- a/runtime/parachains/src/configuration.rs +++ b/runtime/parachains/src/configuration.rs @@ -24,8 +24,8 @@ use frame_system::pallet_prelude::*; use parity_scale_codec::{Decode, Encode}; use polkadot_parachain::primitives::{MAX_HORIZONTAL_MESSAGE_NUM, MAX_UPWARD_MESSAGE_NUM}; use primitives::{ - vstaging::AsyncBackingParams, Balance, SessionIndex, MAX_CODE_SIZE, MAX_HEAD_DATA_SIZE, - MAX_POV_SIZE, + vstaging::AsyncBackingParams, Balance, ExecutorParams, SessionIndex, MAX_CODE_SIZE, + MAX_HEAD_DATA_SIZE, MAX_POV_SIZE, }; use sp_runtime::traits::Zero; use sp_std::prelude::*; @@ -157,6 +157,8 @@ pub struct HostConfiguration { /// /// This parameter affects the upper bound of size of `CandidateCommitments`. pub hrmp_channel_max_message_size: u32, + /// The executor environment parameters + pub executor_params: ExecutorParams, /** * Parameters that will unlikely be needed by parachains. @@ -296,6 +298,7 @@ impl> Default for HostConfiguration Weight; fn set_config_with_balance() -> Weight; fn set_hrmp_open_request_ttl() -> Weight; + fn set_config_with_executor_params() -> Weight; } pub struct TestWeightInfo; @@ -472,6 +476,9 @@ impl WeightInfo for TestWeightInfo { fn set_hrmp_open_request_ttl() -> Weight { Weight::MAX } + fn set_config_with_executor_params() -> Weight { + Weight::MAX + } } #[frame_support::pallet] @@ -1163,6 +1170,19 @@ pub mod pallet { config.async_backing_params = new; }) } + + /// Set PVF executor parameters. + #[pallet::call_index(46)] + #[pallet::weight(( + T::WeightInfo::set_config_with_executor_params(), + DispatchClass::Operational, + ))] + pub fn set_executor_params(origin: OriginFor, new: ExecutorParams) -> DispatchResult { + ensure_root(origin)?; + Self::schedule_config_update(|config| { + config.executor_params = new; + }) + } } #[pallet::hooks] diff --git a/runtime/parachains/src/configuration/benchmarking.rs b/runtime/parachains/src/configuration/benchmarking.rs index b8237b014575..0b8b09fe729e 100644 --- a/runtime/parachains/src/configuration/benchmarking.rs +++ b/runtime/parachains/src/configuration/benchmarking.rs @@ -17,6 +17,7 @@ use crate::configuration::*; use frame_benchmarking::{benchmarks, BenchmarkError, BenchmarkResult}; use frame_system::RawOrigin; +use primitives::{ExecutorParam, ExecutorParams, PvfExecTimeoutKind, PvfPrepTimeoutKind}; use sp_runtime::traits::One; benchmarks! { @@ -36,6 +37,18 @@ benchmarks! { set_config_with_balance {}: set_hrmp_sender_deposit(RawOrigin::Root, 100_000_000_000) + set_config_with_executor_params {}: set_executor_params(RawOrigin::Root, ExecutorParams::from(&[ + ExecutorParam::MaxMemoryPages(2080), + ExecutorParam::StackLogicalMax(65536), + ExecutorParam::StackNativeMax(256 * 1024 * 1024), + ExecutorParam::WasmExtBulkMemory, + ExecutorParam::PrecheckingMaxMemory(2 * 1024 * 1024 * 1024), + ExecutorParam::PvfPrepTimeout(PvfPrepTimeoutKind::Precheck, 60_000), + ExecutorParam::PvfPrepTimeout(PvfPrepTimeoutKind::Lenient, 360_000), + ExecutorParam::PvfExecTimeout(PvfExecTimeoutKind::Backing, 2_000), + ExecutorParam::PvfExecTimeout(PvfExecTimeoutKind::Approval, 12_000), + ][..])) + impl_benchmark_test_suite!( Pallet, crate::mock::new_test_ext(Default::default()), diff --git a/runtime/parachains/src/configuration/migration.rs b/runtime/parachains/src/configuration/migration.rs index fae8fe31fbf1..3aef1761117b 100644 --- a/runtime/parachains/src/configuration/migration.rs +++ b/runtime/parachains/src/configuration/migration.rs @@ -28,7 +28,9 @@ use sp_std::vec::Vec; /// v1-v2: /// v2-v3: /// v3-v4: -/// v4-v5: + +/// v4-v5: +/// + +/// + pub const STORAGE_VERSION: StorageVersion = StorageVersion::new(5); pub mod v5 { @@ -230,6 +232,9 @@ minimum_validation_upgrade_delay : pre.minimum_validation_upgrade_delay, // Default values are zeroes, thus it's ensured allowed ancestry never crosses the upgrade block. async_backing_params : AsyncBackingParams { max_candidate_depth: 0, allowed_ancestry_len: 0 }, + +// Default executor parameters set is empty +executor_params : Default::default(), } }; @@ -268,6 +273,7 @@ async_backing_params : AsyncBackingParams { max_candidate_de mod tests { use super::*; use crate::mock::{new_test_ext, Test}; + use primitives::ExecutorParams; #[test] fn v4_deserialized_from_actual_data() { @@ -391,6 +397,7 @@ mod tests { // additional checks for async backing. assert_eq!(v5.async_backing_params.allowed_ancestry_len, 0); assert_eq!(v5.async_backing_params.max_candidate_depth, 0); + assert_eq!(v5.executor_params, ExecutorParams::new()); } }); } diff --git a/runtime/parachains/src/configuration/tests.rs b/runtime/parachains/src/configuration/tests.rs index d7cc934ce5db..d076865bf6bb 100644 --- a/runtime/parachains/src/configuration/tests.rs +++ b/runtime/parachains/src/configuration/tests.rs @@ -326,6 +326,7 @@ fn setting_pending_config_members() { pvf_checking_enabled: true, pvf_voting_ttl: 3, minimum_validation_upgrade_delay: 20, + executor_params: Default::default(), }; Configuration::set_validation_upgrade_cooldown( diff --git a/runtime/parachains/src/dmp.rs b/runtime/parachains/src/dmp.rs index e5ccac2647fe..5244406fcce4 100644 --- a/runtime/parachains/src/dmp.rs +++ b/runtime/parachains/src/dmp.rs @@ -14,13 +14,45 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +//! To prevent Out of Memory errors on the `DownwardMessageQueue`, an +//! exponential fee factor (`DeliveryFeeFactor`) is set. The fee factor +//! increments exponentially after the number of messages in the +//! `DownwardMessageQueue` pass a threshold. This threshold is set as: +//! +//! ```ignore +//! // Maximum max sized messages that can be send to +//! // the DownwardMessageQueue before it runs out of memory +//! max_messsages = MAX_POSSIBLE_ALLOCATION / max_downward_message_size +//! threshold = max_messages / THRESHOLD_FACTOR +//! ``` +//! Based on the THRESHOLD_FACTOR, the threshold is set as a fraction of the +//! total messages. The `DeliveryFeeFactor` increases for a message over the +//! threshold by: +//! +//! `DeliveryFeeFactor = DeliveryFeeFactor * +//! (EXPONENTIAL_FEE_BASE + MESSAGE_SIZE_FEE_BASE * encoded_message_size_in_KB)` +//! +//! And decreases when the number of messages in the `DownwardMessageQueue` fall +//! below the threshold by: +//! +//! `DeliveryFeeFactor = DeliveryFeeFactor / EXPONENTIAL_FEE_BASE` +//! +//! As an extra defensive measure, a `max_messages` hard +//! limit is set to the number of messages in the DownwardMessageQueue. Messages +//! that would increase the number of messages in the queue above this hard +//! limit are dropped. + use crate::{ configuration::{self, HostConfiguration}, - initializer, + initializer, FeeTracker, }; use frame_support::pallet_prelude::*; use primitives::{DownwardMessage, Hash, Id as ParaId, InboundDownwardMessage}; -use sp_runtime::traits::{BlakeTwo256, Hash as HashT, SaturatedConversion}; +use sp_core::MAX_POSSIBLE_ALLOCATION; +use sp_runtime::{ + traits::{BlakeTwo256, Hash as HashT, SaturatedConversion}, + FixedU128, Saturating, +}; use sp_std::{fmt, prelude::*}; use xcm::latest::SendError; @@ -29,7 +61,9 @@ pub use pallet::*; #[cfg(test)] mod tests; -pub const MAX_MESSAGE_QUEUE_SIZE: usize = 1024; +const THRESHOLD_FACTOR: u32 = 2; +const EXPONENTIAL_FEE_BASE: FixedU128 = FixedU128::from_rational(105, 100); // 1.05 +const MESSAGE_SIZE_FEE_BASE: FixedU128 = FixedU128::from_rational(1, 1000); // 0.001 /// An error sending a downward message. #[cfg_attr(test, derive(Debug))] @@ -102,10 +136,17 @@ pub mod pallet { pub(crate) type DownwardMessageQueueHeads = StorageMap<_, Twox64Concat, ParaId, Hash, ValueQuery>; - #[pallet::call] - impl Pallet {} -} + /// Initialization value for the DeliveryFee factor. + #[pallet::type_value] + pub fn InitialFactor() -> FixedU128 { + FixedU128::from_u32(1) + } + /// The number to multiply the base delivery fee by. + #[pallet::storage] + pub(crate) type DeliveryFeeFactor = + StorageMap<_, Twox64Concat, ParaId, FixedU128, ValueQuery, InitialFactor>; +} /// Routines and getters related to downward message passing. impl Pallet { /// Block initialization logic, called by initializer. @@ -151,7 +192,8 @@ impl Pallet { return Err(QueueDownwardMessageError::ExceedsMaxMessageSize) } - if DownwardMessageQueues::::decode_len(para).unwrap_or(0) > MAX_MESSAGE_QUEUE_SIZE { + // Hard limit on Queue size + if Self::dmq_length(*para) > Self::dmq_max_length(config.max_downward_message_size) { return Err(QueueDownwardMessageError::ExceedsMaxMessageSize) } @@ -176,7 +218,8 @@ impl Pallet { return Err(QueueDownwardMessageError::ExceedsMaxMessageSize) } - if DownwardMessageQueues::::decode_len(para).unwrap_or(0) > MAX_MESSAGE_QUEUE_SIZE { + // Hard limit on Queue size + if Self::dmq_length(para) > Self::dmq_max_length(config.max_downward_message_size) { return Err(QueueDownwardMessageError::ExceedsMaxMessageSize) } @@ -190,10 +233,20 @@ impl Pallet { *head = new_head; }); - DownwardMessageQueues::::mutate(para, |v| { + let q_len = DownwardMessageQueues::::mutate(para, |v| { v.push(inbound); + v.len() }); + let threshold = + Self::dmq_max_length(config.max_downward_message_size).saturating_div(THRESHOLD_FACTOR); + if q_len > (threshold as usize) { + let message_size_factor = + FixedU128::from_u32(serialized_len.saturating_div(1024) as u32) + .saturating_mul(MESSAGE_SIZE_FEE_BASE); + Self::increment_fee_factor(para, message_size_factor); + } + Ok(()) } @@ -219,7 +272,7 @@ impl Pallet { /// Prunes the specified number of messages from the downward message queue of the given para. pub(crate) fn prune_dmq(para: ParaId, processed_downward_messages: u32) -> Weight { - DownwardMessageQueues::::mutate(para, |q| { + let q_len = DownwardMessageQueues::::mutate(para, |q| { let processed_downward_messages = processed_downward_messages as usize; if processed_downward_messages > q.len() { // reaching this branch is unexpected due to the constraint established by @@ -228,7 +281,15 @@ impl Pallet { } else { *q = q.split_off(processed_downward_messages); } + q.len() }); + + let config = configuration::ActiveConfig::::get(); + let threshold = + Self::dmq_max_length(config.max_downward_message_size).saturating_div(THRESHOLD_FACTOR); + if q_len <= (threshold as usize) { + Self::decrement_fee_factor(para); + } T::DbWeight::get().reads_writes(1, 1) } @@ -248,10 +309,42 @@ impl Pallet { .saturated_into::() } + fn dmq_max_length(max_downward_message_size: u32) -> u32 { + MAX_POSSIBLE_ALLOCATION.checked_div(max_downward_message_size).unwrap_or(0) + } + /// Returns the downward message queue contents for the given para. /// /// The most recent messages are the latest in the vector. pub(crate) fn dmq_contents(recipient: ParaId) -> Vec> { DownwardMessageQueues::::get(&recipient) } + + /// Raise the delivery fee factor by a multiplicative factor and stores the resulting value. + /// + /// Returns the new delivery fee factor after the increment. + pub(crate) fn increment_fee_factor(para: ParaId, message_size_factor: FixedU128) -> FixedU128 { + >::mutate(para, |f| { + *f = f.saturating_mul(EXPONENTIAL_FEE_BASE + message_size_factor); + *f + }) + } + + /// Reduce the delivery fee factor by a multiplicative factor and stores the resulting value. + /// + /// Does not reduce the fee factor below the initial value, which is currently set as 1. + /// + /// Returns the new delivery fee factor after the decrement. + pub(crate) fn decrement_fee_factor(para: ParaId) -> FixedU128 { + >::mutate(para, |f| { + *f = InitialFactor::get().max(*f / EXPONENTIAL_FEE_BASE); + *f + }) + } +} + +impl FeeTracker for Pallet { + fn get_fee_factor(para: ParaId) -> FixedU128 { + DeliveryFeeFactor::::get(para) + } } diff --git a/runtime/parachains/src/dmp/tests.rs b/runtime/parachains/src/dmp/tests.rs index 6e05f6c735ed..234f5f7e43c7 100644 --- a/runtime/parachains/src/dmp/tests.rs +++ b/runtime/parachains/src/dmp/tests.rs @@ -15,7 +15,11 @@ // along with Polkadot. If not, see . use super::*; -use crate::mock::{new_test_ext, Configuration, Dmp, MockGenesisConfig, Paras, System, Test}; +use crate::{ + configuration::ActiveConfig, + mock::{new_test_ext, Configuration, Dmp, MockGenesisConfig, Paras, System, Test}, +}; +use frame_support::assert_ok; use hex_literal::hex; use parity_scale_codec::Encode; use primitives::BlockNumber; @@ -205,3 +209,69 @@ fn verify_dmq_mqc_head_is_externally_accessible() { ); }); } + +#[test] +fn verify_fee_increment_and_decrement() { + let a = ParaId::from(123); + let mut genesis = default_genesis_config(); + genesis.configuration.config.max_downward_message_size = 16777216; + new_test_ext(genesis).execute_with(|| { + let initial = InitialFactor::get(); + assert_eq!(DeliveryFeeFactor::::get(a), initial); + + // Under fee limit + queue_downward_message(a, vec![1]).unwrap(); + assert_eq!(DeliveryFeeFactor::::get(a), initial); + + // Limit reached so fee is increased + queue_downward_message(a, vec![1]).unwrap(); + let result = InitialFactor::get().saturating_mul(EXPONENTIAL_FEE_BASE); + assert_eq!(DeliveryFeeFactor::::get(a), result); + + Dmp::prune_dmq(a, 1); + assert_eq!(DeliveryFeeFactor::::get(a), initial); + + // 10 Kb message adds additional 0.001 per KB fee factor + let big_message = [0; 10240].to_vec(); + let msg_len_in_kb = big_message.len().saturating_div(1024) as u32; + let result = initial.saturating_mul( + EXPONENTIAL_FEE_BASE + + MESSAGE_SIZE_FEE_BASE.saturating_mul(FixedU128::from_u32(msg_len_in_kb)), + ); + queue_downward_message(a, big_message).unwrap(); + assert_eq!(DeliveryFeeFactor::::get(a), result); + + queue_downward_message(a, vec![1]).unwrap(); + let result = result.saturating_mul(EXPONENTIAL_FEE_BASE); + assert_eq!(DeliveryFeeFactor::::get(a), result); + + Dmp::prune_dmq(a, 3); + let result = result / EXPONENTIAL_FEE_BASE; + assert_eq!(DeliveryFeeFactor::::get(a), result); + assert_eq!(Dmp::dmq_length(a), 0); + + // Messages under limit will keep decreasing fee factor until base fee factor is reached + queue_downward_message(a, vec![1]).unwrap(); + Dmp::prune_dmq(a, 1); + queue_downward_message(a, vec![1]).unwrap(); + Dmp::prune_dmq(a, 1); + assert_eq!(DeliveryFeeFactor::::get(a), initial); + }); +} + +#[test] +fn verify_fee_factor_reaches_high_value() { + let a = ParaId::from(123); + let mut genesis = default_genesis_config(); + genesis.configuration.config.max_downward_message_size = 51200; + new_test_ext(genesis).execute_with(|| { + let max_messages = + Dmp::dmq_max_length(ActiveConfig::::get().max_downward_message_size); + let mut total_fee_factor = FixedU128::from_float(1.0); + for _ in 1..max_messages { + assert_ok!(queue_downward_message(a, vec![])); + total_fee_factor = total_fee_factor + (DeliveryFeeFactor::::get(a)); + } + assert!(total_fee_factor > FixedU128::from_u32(100_000_000)); + }); +} diff --git a/runtime/parachains/src/lib.rs b/runtime/parachains/src/lib.rs index ad884802ff19..9b2a49d34986 100644 --- a/runtime/parachains/src/lib.rs +++ b/runtime/parachains/src/lib.rs @@ -51,7 +51,12 @@ mod mock; pub use origin::{ensure_parachain, Origin}; pub use paras::ParaLifecycle; use primitives::{HeadData, Id as ParaId, ValidationCode}; -use sp_runtime::DispatchResult; +use sp_runtime::{DispatchResult, FixedU128}; + +/// Trait for tracking message delivery fees on a transport protocol. +pub trait FeeTracker { + fn get_fee_factor(para: ParaId) -> FixedU128; +} /// Schedule a para to be initialized at the start of the next session with the given genesis data. /// diff --git a/runtime/parachains/src/session_info.rs b/runtime/parachains/src/session_info.rs index dc2a2b5e46c7..9f47ad0b03d6 100644 --- a/runtime/parachains/src/session_info.rs +++ b/runtime/parachains/src/session_info.rs @@ -27,9 +27,7 @@ use frame_support::{ pallet_prelude::*, traits::{OneSessionHandler, ValidatorSet, ValidatorSetWithIdentification}, }; -use primitives::{ - AssignmentId, AuthorityDiscoveryId, ExecutorParam, ExecutorParams, SessionIndex, SessionInfo, -}; +use primitives::{AssignmentId, AuthorityDiscoveryId, ExecutorParams, SessionIndex, SessionInfo}; use sp_std::vec::Vec; pub use pallet::*; @@ -39,10 +37,6 @@ pub mod migration; #[cfg(test)] mod tests; -// The order of parameters should be deterministic, that is, one should not reorder them when -// changing the array contents to avoid creating excessive pressure to PVF execution subsys. -const EXECUTOR_PARAMS: [ExecutorParam; 0] = []; - /// A type for representing the validator account id in a session. pub type AccountId = <::ValidatorSet as ValidatorSet< ::AccountId, @@ -196,10 +190,8 @@ impl Pallet { dispute_period, }; Sessions::::insert(&new_session_index, &new_session_info); - SessionExecutorParams::::insert( - &new_session_index, - ExecutorParams::from(&EXECUTOR_PARAMS[..]), - ); + + SessionExecutorParams::::insert(&new_session_index, config.executor_params); } /// Called by the initializer to initialize the session info pallet. diff --git a/runtime/polkadot/Cargo.toml b/runtime/polkadot/Cargo.toml index 8f13542c727e..56f835ea5a47 100644 --- a/runtime/polkadot/Cargo.toml +++ b/runtime/polkadot/Cargo.toml @@ -23,11 +23,11 @@ block-builder-api = { package = "sp-block-builder", git = "https://github.com/pa inherents = { package = "sp-inherents", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } offchain-primitives = { package = "sp-offchain", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } tx-pool-api = { package = "sp-transaction-pool", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } +sp-arithmetic = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-api = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-io = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-mmr-primitives = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } -sp-arithmetic = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-staking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } @@ -88,7 +88,7 @@ pallet-election-provider-support-benchmarking = { git = "https://github.com/pari pallet-offences-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true } pallet-session-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true } pallet-nomination-pools-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true } -hex-literal = { version = "0.3.4", optional = true } +hex-literal = { version = "0.4.1", optional = true } runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false } runtime-parachains = { package = "polkadot-runtime-parachains", path = "../parachains", default-features = false } @@ -99,11 +99,11 @@ xcm-executor = { package = "xcm-executor", path = "../../xcm/xcm-executor", defa xcm-builder = { package = "xcm-builder", path = "../../xcm/xcm-builder", default-features = false } [dev-dependencies] -hex-literal = "0.3.4" +hex-literal = "0.4.1" tiny-keccak = { version = "2.0.2", features = ["keccak"] } keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate", branch = "master" } sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" } -serde_json = "1.0.81" +serde_json = "1.0.96" separator = "0.4.1" remote-externalities = { git = "https://github.com/paritytech/substrate", branch = "master", package = "frame-remote-externalities" } tokio = { version = "1.24.2", features = ["macros"] } @@ -124,6 +124,7 @@ std = [ "parity-scale-codec/std", "scale-info/std", "inherents/std", + "sp-arithmetic/std", "sp-core/std", "sp-api/std", "tx-pool-api/std", diff --git a/runtime/polkadot/src/governance/old.rs b/runtime/polkadot/src/governance/old.rs index f4eb7a2f2eae..0f142f530dd2 100644 --- a/runtime/polkadot/src/governance/old.rs +++ b/runtime/polkadot/src/governance/old.rs @@ -99,6 +99,7 @@ parameter_types! { pub CouncilMotionDuration: BlockNumber = prod_or_fast!(7 * DAYS, 2 * MINUTES, "DOT_MOTION_DURATION"); pub const CouncilMaxProposals: u32 = 100; pub const CouncilMaxMembers: u32 = 100; + pub MaxProposalWeight: Weight = Perbill::from_percent(50) * BlockWeights::get().max_block; } pub type CouncilCollective = pallet_collective::Instance1; @@ -112,6 +113,7 @@ impl pallet_collective::Config for Runtime { type DefaultVote = pallet_collective::PrimeDefaultVote; type SetMembersOrigin = EnsureRoot; type WeightInfo = weights::pallet_collective_council::WeightInfo; + type MaxProposalWeight = MaxProposalWeight; } parameter_types! { @@ -171,6 +173,7 @@ impl pallet_collective::Config for Runtime { type DefaultVote = pallet_collective::PrimeDefaultVote; type SetMembersOrigin = EnsureRoot; type WeightInfo = weights::pallet_collective_technical_committee::WeightInfo; + type MaxProposalWeight = MaxProposalWeight; } impl pallet_membership::Config for Runtime { diff --git a/runtime/polkadot/src/lib.rs b/runtime/polkadot/src/lib.rs index 70f1fc52bffe..ec68400dca86 100644 --- a/runtime/polkadot/src/lib.rs +++ b/runtime/polkadot/src/lib.rs @@ -121,13 +121,13 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("polkadot"), impl_name: create_runtime_str!("parity-polkadot"), authoring_version: 0, - spec_version: 9390, + spec_version: 9410, impl_version: 0, #[cfg(not(feature = "disable-runtime-api"))] apis: RUNTIME_API_VERSIONS, #[cfg(feature = "disable-runtime-api")] apis: sp_version::create_apis_vec![[]], - transaction_version: 21, + transaction_version: 22, state_version: 0, }; @@ -1367,7 +1367,7 @@ construct_runtime! { ParaScheduler: parachains_scheduler::{Pallet, Storage} = 55, Paras: parachains_paras::{Pallet, Call, Storage, Event, Config, ValidateUnsigned} = 56, Initializer: parachains_initializer::{Pallet, Call, Storage} = 57, - Dmp: parachains_dmp::{Pallet, Call, Storage} = 58, + Dmp: parachains_dmp::{Pallet, Storage} = 58, Ump: parachains_ump::{Pallet, Call, Storage, Event} = 59, Hrmp: parachains_hrmp::{Pallet, Call, Storage, Event, Config} = 60, ParaSessionInfo: parachains_session_info::{Pallet, Storage} = 61, @@ -1424,8 +1424,8 @@ pub type Migrations = ( Runtime, NominationPoolsMigrationV4OldPallet, >, - // Unreleased - add new migrations here: pallet_nomination_pools::migration::v5::MigrateToV5, + // Unreleased - add new migrations here: parachains_configuration::migration::v5::MigrateToV5, ); diff --git a/runtime/polkadot/src/weights/runtime_parachains_configuration.rs b/runtime/polkadot/src/weights/runtime_parachains_configuration.rs index 35ff4e8c9430..79f99e6551b3 100644 --- a/runtime/polkadot/src/weights/runtime_parachains_configuration.rs +++ b/runtime/polkadot/src/weights/runtime_parachains_configuration.rs @@ -16,24 +16,26 @@ //! Autogenerated weights for `runtime_parachains::configuration` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-03-16, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2023-04-07, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `bm4`, CPU: `Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz` +//! HOSTNAME: `bm3`, CPU: `Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("polkadot-dev"), DB CACHE: 1024 // Executed Command: -// ./target/production/polkadot +// target/production/polkadot // benchmark // pallet -// --chain=polkadot-dev // --steps=50 // --repeat=20 -// --pallet=runtime_parachains::configuration // --extrinsic=* // --execution=wasm // --wasm-execution=compiled +// --heap-pages=4096 +// --json-file=/var/lib/gitlab-runner/builds/zyw4fam_/0/parity/mirrors/polkadot/.git/.artifacts/bench.json +// --pallet=runtime_parachains::configuration +// --chain=polkadot-dev // --header=./file_header.txt -// --output=./runtime/polkadot/src/weights/runtime_parachains_configuration.rs +// --output=./runtime/polkadot/src/weights/ #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -55,11 +57,11 @@ impl runtime_parachains::configuration::WeightInfo for /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) fn set_config_with_block_number() -> Weight { // Proof Size summary in bytes: - // Measured: `317` - // Estimated: `7208` - // Minimum execution time: 12_097_000 picoseconds. - Weight::from_parts(12_745_000, 0) - .saturating_add(Weight::from_parts(0, 7208)) + // Measured: `393` + // Estimated: `7512` + // Minimum execution time: 13_232_000 picoseconds. + Weight::from_parts(13_585_000, 0) + .saturating_add(Weight::from_parts(0, 7512)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -73,11 +75,11 @@ impl runtime_parachains::configuration::WeightInfo for /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) fn set_config_with_u32() -> Weight { // Proof Size summary in bytes: - // Measured: `317` - // Estimated: `7208` - // Minimum execution time: 12_691_000 picoseconds. - Weight::from_parts(13_006_000, 0) - .saturating_add(Weight::from_parts(0, 7208)) + // Measured: `393` + // Estimated: `7512` + // Minimum execution time: 13_436_000 picoseconds. + Weight::from_parts(13_919_000, 0) + .saturating_add(Weight::from_parts(0, 7512)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -91,11 +93,11 @@ impl runtime_parachains::configuration::WeightInfo for /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) fn set_config_with_option_u32() -> Weight { // Proof Size summary in bytes: - // Measured: `317` - // Estimated: `7208` - // Minimum execution time: 12_533_000 picoseconds. - Weight::from_parts(13_126_000, 0) - .saturating_add(Weight::from_parts(0, 7208)) + // Measured: `393` + // Estimated: `7512` + // Minimum execution time: 13_053_000 picoseconds. + Weight::from_parts(13_330_000, 0) + .saturating_add(Weight::from_parts(0, 7512)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -109,11 +111,11 @@ impl runtime_parachains::configuration::WeightInfo for /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) fn set_config_with_weight() -> Weight { // Proof Size summary in bytes: - // Measured: `317` - // Estimated: `7208` - // Minimum execution time: 12_511_000 picoseconds. - Weight::from_parts(12_858_000, 0) - .saturating_add(Weight::from_parts(0, 7208)) + // Measured: `393` + // Estimated: `7512` + // Minimum execution time: 12_931_000 picoseconds. + Weight::from_parts(13_349_000, 0) + .saturating_add(Weight::from_parts(0, 7512)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -137,11 +139,29 @@ impl runtime_parachains::configuration::WeightInfo for /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) fn set_config_with_balance() -> Weight { // Proof Size summary in bytes: - // Measured: `317` - // Estimated: `7208` - // Minimum execution time: 12_525_000 picoseconds. - Weight::from_parts(13_130_000, 0) - .saturating_add(Weight::from_parts(0, 7208)) + // Measured: `393` + // Estimated: `7512` + // Minimum execution time: 12_861_000 picoseconds. + Weight::from_parts(13_341_000, 0) + .saturating_add(Weight::from_parts(0, 7512)) + .saturating_add(T::DbWeight::get().reads(4)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: Configuration PendingConfigs (r:1 w:1) + /// Proof Skipped: Configuration PendingConfigs (max_values: Some(1), max_size: None, mode: Measured) + /// Storage: Configuration ActiveConfig (r:1 w:0) + /// Proof Skipped: Configuration ActiveConfig (max_values: Some(1), max_size: None, mode: Measured) + /// Storage: Configuration BypassConsistencyCheck (r:1 w:0) + /// Proof Skipped: Configuration BypassConsistencyCheck (max_values: Some(1), max_size: None, mode: Measured) + /// Storage: ParasShared CurrentSessionIndex (r:1 w:0) + /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) + fn set_config_with_executor_params() -> Weight { + // Proof Size summary in bytes: + // Measured: `393` + // Estimated: `7512` + // Minimum execution time: 13_678_000 picoseconds. + Weight::from_parts(14_093_000, 0) + .saturating_add(Weight::from_parts(0, 7512)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(1)) } diff --git a/runtime/polkadot/src/xcm_config.rs b/runtime/polkadot/src/xcm_config.rs index 5aa841564bf5..e17b04a24edc 100644 --- a/runtime/polkadot/src/xcm_config.rs +++ b/runtime/polkadot/src/xcm_config.rs @@ -17,9 +17,9 @@ //! XCM configuration for Polkadot. use super::{ - parachains_origin, AccountId, AllPalletsWithSystem, Balances, CouncilCollective, + parachains_origin, AccountId, AllPalletsWithSystem, Balances, CouncilCollective, Dmp, FellowshipAdmin, ParaId, Runtime, RuntimeCall, RuntimeEvent, RuntimeOrigin, StakingAdmin, - WeightToFee, XcmPallet, + TransactionByteFee, WeightToFee, XcmPallet, }; use frame_support::{ match_types, parameter_types, @@ -28,8 +28,14 @@ use frame_support::{ }; use frame_system::EnsureRoot; use pallet_xcm::XcmPassthrough; -use polkadot_runtime_constants::{system_parachain::*, xcm::body::FELLOWSHIP_ADMIN_INDEX}; -use runtime_common::{crowdloan, paras_registrar, xcm_sender, ToAuthor}; +use polkadot_runtime_constants::{ + currency::CENTS, system_parachain::*, xcm::body::FELLOWSHIP_ADMIN_INDEX, +}; +use runtime_common::{ + crowdloan, paras_registrar, + xcm_sender::{ChildParachainRouter, ExponentialPrice}, + ToAuthor, +}; use sp_core::ConstU32; use xcm::latest::prelude::*; use xcm_builder::{ @@ -102,17 +108,28 @@ type LocalOriginConverter = ( parameter_types! { /// The amount of weight an XCM operation takes. This is a safe overestimate. - pub const BaseXcmWeight: Weight = Weight::from_parts(1_000_000_000, 64 * 1024); + pub const BaseXcmWeight: Weight = Weight::from_parts(1_000_000_000, 1024); + /// A temporary weight value for each XCM instruction. + /// NOTE: This should be removed after we account for PoV weights. + pub const TempFixedXcmWeight: Weight = Weight::from_parts(1_000_000_000, 0); /// Maximum number of instructions in a single XCM fragment. A sanity check against weight /// calculations getting too crazy. pub const MaxInstructions: u32 = 100; + /// The asset ID for the asset that we use to pay for message delivery fees. + pub FeeAssetId: AssetId = Concrete(TokenLocation::get()); + /// The base fee for the message delivery fees. + pub const BaseDeliveryFee: u128 = CENTS.saturating_mul(3); } /// The XCM router. When we want to send an XCM message, we use this type. It amalgamates all of our /// individual routers. pub type XcmRouter = ( // Only one router so far - use DMP to communicate with child parachains. - xcm_sender::ChildParachainRouter, + ChildParachainRouter< + Runtime, + XcmPallet, + ExponentialPrice, + >, ); parameter_types! { @@ -220,48 +237,16 @@ impl Contains for SafeCallFilter { RuntimeCall::Session(pallet_session::Call::purge_keys { .. }) | RuntimeCall::Grandpa(..) | RuntimeCall::ImOnline(..) | - RuntimeCall::Democracy( - pallet_democracy::Call::second { .. } | - pallet_democracy::Call::vote { .. } | - pallet_democracy::Call::emergency_cancel { .. } | - pallet_democracy::Call::fast_track { .. } | - pallet_democracy::Call::veto_external { .. } | - pallet_democracy::Call::cancel_referendum { .. } | - pallet_democracy::Call::delegate { .. } | - pallet_democracy::Call::undelegate { .. } | - pallet_democracy::Call::clear_public_proposals { .. } | - pallet_democracy::Call::unlock { .. } | - pallet_democracy::Call::remove_vote { .. } | - pallet_democracy::Call::remove_other_vote { .. } | - pallet_democracy::Call::blacklist { .. } | - pallet_democracy::Call::cancel_proposal { .. }, - ) | - RuntimeCall::Council( - pallet_collective::Call::vote { .. } | - pallet_collective::Call::disapprove_proposal { .. } | - pallet_collective::Call::close { .. }, - ) | - RuntimeCall::TechnicalCommittee( - pallet_collective::Call::vote { .. } | - pallet_collective::Call::disapprove_proposal { .. } | - pallet_collective::Call::close { .. }, - ) | - RuntimeCall::PhragmenElection( - pallet_elections_phragmen::Call::remove_voter { .. } | - pallet_elections_phragmen::Call::submit_candidacy { .. } | - pallet_elections_phragmen::Call::renounce_candidacy { .. } | - pallet_elections_phragmen::Call::remove_member { .. } | - pallet_elections_phragmen::Call::clean_defunct_voters { .. }, - ) | - RuntimeCall::TechnicalMembership( - pallet_membership::Call::add_member { .. } | - pallet_membership::Call::remove_member { .. } | - pallet_membership::Call::swap_member { .. } | - pallet_membership::Call::change_key { .. } | - pallet_membership::Call::set_prime { .. } | - pallet_membership::Call::clear_prime { .. }, - ) | RuntimeCall::Treasury(..) | + RuntimeCall::ConvictionVoting(..) | + RuntimeCall::Referenda( + pallet_referenda::Call::place_decision_deposit { .. } | + pallet_referenda::Call::refund_decision_deposit { .. } | + pallet_referenda::Call::cancel { .. } | + pallet_referenda::Call::kill { .. } | + pallet_referenda::Call::nudge_referendum { .. } | + pallet_referenda::Call::one_fewer_deciding { .. }, + ) | RuntimeCall::Claims( super::claims::Call::claim { .. } | super::claims::Call::mint_claim { .. } | @@ -323,7 +308,8 @@ impl Contains for SafeCallFilter { RuntimeCall::XcmPallet(pallet_xcm::Call::limited_reserve_transfer_assets { .. }) | - RuntimeCall::Whitelist(pallet_whitelist::Call::whitelist_call { .. }) => true, + RuntimeCall::Whitelist(pallet_whitelist::Call::whitelist_call { .. }) | + RuntimeCall::Proxy(..) => true, _ => false, } } @@ -340,7 +326,7 @@ impl xcm_executor::Config for XcmConfig { type IsTeleporter = TrustedTeleporters; type UniversalLocation = UniversalLocation; type Barrier = Barrier; - type Weigher = FixedWeightBounds; + type Weigher = FixedWeightBounds; // The weight trader piggybacks on the existing transaction-fee conversion logic. type Trader = UsingComponents>; diff --git a/runtime/rococo/Cargo.toml b/runtime/rococo/Cargo.toml index d6caacd7f5f4..3a10b27c4f35 100644 --- a/runtime/rococo/Cargo.toml +++ b/runtime/rococo/Cargo.toml @@ -79,7 +79,7 @@ pallet-xcm-benchmarks = { path = "../../xcm/pallet-xcm-benchmarks", default-feat frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true } frame-try-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true } frame-system-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true } -hex-literal = { version = "0.3.4" } +hex-literal = { version = "0.4.1" } runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false } runtime-parachains = { package = "polkadot-runtime-parachains", path = "../parachains", default-features = false } @@ -96,7 +96,7 @@ keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substra remote-externalities = { git = "https://github.com/paritytech/substrate", branch = "master", package = "frame-remote-externalities" } sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" } separator = "0.4.1" -serde_json = "1.0.81" +serde_json = "1.0.96" sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } tokio = { version = "1.24.2", features = ["macros"] } diff --git a/runtime/rococo/src/lib.rs b/runtime/rococo/src/lib.rs index a7606cf3298b..c4be4490192e 100644 --- a/runtime/rococo/src/lib.rs +++ b/runtime/rococo/src/lib.rs @@ -110,13 +110,13 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("rococo"), impl_name: create_runtime_str!("parity-rococo-v2.0"), authoring_version: 0, - spec_version: 9390, + spec_version: 9410, impl_version: 0, #[cfg(not(feature = "disable-runtime-api"))] apis: RUNTIME_API_VERSIONS, #[cfg(feature = "disable-runtime-api")] apis: sp_version::create_apis_vec![[]], - transaction_version: 19, + transaction_version: 20, state_version: 1, }; @@ -284,7 +284,7 @@ impl pallet_balances::Config for Runtime { type WeightInfo = weights::pallet_balances::WeightInfo; type FreezeIdentifier = (); type MaxFreezes = ConstU32<1>; - type HoldIdentifier = HoldReason; + type HoldIdentifier = RuntimeHoldReason; type MaxHolds = ConstU32<1>; } @@ -438,6 +438,7 @@ parameter_types! { pub CouncilMotionDuration: BlockNumber = prod_or_fast!(3 * DAYS, 2 * MINUTES, "ROC_MOTION_DURATION"); pub const CouncilMaxProposals: u32 = 100; pub const CouncilMaxMembers: u32 = 100; + pub MaxProposalWeight: Weight = Perbill::from_percent(50) * BlockWeights::get().max_block; } type CouncilCollective = pallet_collective::Instance1; @@ -451,6 +452,7 @@ impl pallet_collective::Config for Runtime { type DefaultVote = pallet_collective::PrimeDefaultVote; type SetMembersOrigin = EnsureRoot; type WeightInfo = weights::pallet_collective_council::WeightInfo; + type MaxProposalWeight = MaxProposalWeight; } parameter_types! { @@ -510,6 +512,7 @@ impl pallet_collective::Config for Runtime { type DefaultVote = pallet_collective::PrimeDefaultVote; type SetMembersOrigin = EnsureRoot; type WeightInfo = weights::pallet_collective_technical_committee::WeightInfo; + type MaxProposalWeight = MaxProposalWeight; } type MoreThanHalfCouncil = EitherOfDiverse< @@ -875,7 +878,7 @@ parameter_types! { Decode, RuntimeDebug, MaxEncodedLen, - scale_info::TypeInfo, + TypeInfo, )] pub enum ProxyType { Any, @@ -1196,27 +1199,7 @@ parameter_types! { pub const ThawThrottle: (Perquintill, BlockNumber) = (Perquintill::from_percent(25), 5); pub storage NisTarget: Perquintill = Perquintill::zero(); pub const NisPalletId: PalletId = PalletId(*b"py/nis "); - pub const NisHoldReason: HoldReason = HoldReason::Nis(HoldReasonNis::NftReceipt); -} - -/// A reason for placing a hold on funds. -#[derive( - Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Encode, Decode, MaxEncodedLen, Debug, TypeInfo, -)] -pub enum HoldReason { - /// Some reason of the NIS pallet. - #[codec(index = 38)] - Nis(HoldReasonNis), -} - -/// A reason for the NIS pallet placing a hold on funds. -#[derive( - Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Encode, Decode, MaxEncodedLen, Debug, TypeInfo, -)] -pub enum HoldReasonNis { - /// The NIS Pallet has reserved it for a non-fungible receipt. - #[codec(index = 0)] - NftReceipt = 0, + pub const NisHoldReason: RuntimeHoldReason = RuntimeHoldReason::Nis(pallet_nis::HoldReason::NftReceipt); } impl pallet_nis::Config for Runtime { @@ -1416,7 +1399,7 @@ construct_runtime! { Tips: pallet_tips::{Pallet, Call, Storage, Event} = 36, // NIS pallet. - Nis: pallet_nis::{Pallet, Call, Storage, Event} = 38, + Nis: pallet_nis::{Pallet, Call, Storage, Event, HoldReason} = 38, // pub type NisCounterpartInstance = pallet_balances::Instance2; NisCounterpartBalances: pallet_balances:: = 45, @@ -1429,7 +1412,7 @@ construct_runtime! { ParaScheduler: parachains_scheduler::{Pallet, Storage} = 55, Paras: parachains_paras::{Pallet, Call, Storage, Event, Config, ValidateUnsigned} = 56, Initializer: parachains_initializer::{Pallet, Call, Storage} = 57, - Dmp: parachains_dmp::{Pallet, Call, Storage} = 58, + Dmp: parachains_dmp::{Pallet, Storage} = 58, Ump: parachains_ump::{Pallet, Call, Storage, Event} = 59, Hrmp: parachains_hrmp::{Pallet, Call, Storage, Event, Config} = 60, ParaSessionInfo: parachains_session_info::{Pallet, Storage} = 61, @@ -1495,7 +1478,10 @@ pub type UncheckedExtrinsic = /// /// This contains the combined migrations of the last 10 releases. It allows to skip runtime /// upgrades in case governance decides to do so. -pub type Migrations = parachains_configuration::migration::v5::MigrateToV5; +pub type Migrations = ( + // Unreleased - add new migrations here: + parachains_configuration::migration::v5::MigrateToV5, +); /// Executive: handles dispatch to the various modules. pub type Executive = frame_executive::Executive< @@ -2178,6 +2164,16 @@ sp_api::impl_runtime_apis! { } } +#[cfg(test)] +mod encoding_tests { + use super::*; + + #[test] + fn nis_hold_reason_encoding_is_correct() { + assert_eq!(NisHoldReason::get().encode(), [38, 0]); + } +} + #[cfg(all(test, feature = "try-runtime"))] mod remote_tests { use super::*; diff --git a/runtime/rococo/src/weights/runtime_parachains_configuration.rs b/runtime/rococo/src/weights/runtime_parachains_configuration.rs index 8a7b1a354be0..42f5086c2b33 100644 --- a/runtime/rococo/src/weights/runtime_parachains_configuration.rs +++ b/runtime/rococo/src/weights/runtime_parachains_configuration.rs @@ -16,24 +16,26 @@ //! Autogenerated weights for `runtime_parachains::configuration` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-03-16, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2023-04-07, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `bm6`, CPU: `Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz` +//! HOSTNAME: `bm3`, CPU: `Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("rococo-dev"), DB CACHE: 1024 // Executed Command: -// ./target/production/polkadot +// target/production/polkadot // benchmark // pallet -// --chain=rococo-dev // --steps=50 // --repeat=20 -// --pallet=runtime_parachains::configuration // --extrinsic=* // --execution=wasm // --wasm-execution=compiled +// --heap-pages=4096 +// --json-file=/var/lib/gitlab-runner/builds/zyw4fam_/0/parity/mirrors/polkadot/.git/.artifacts/bench.json +// --pallet=runtime_parachains::configuration +// --chain=rococo-dev // --header=./file_header.txt -// --output=./runtime/rococo/src/weights/runtime_parachains_configuration.rs +// --output=./runtime/rococo/src/weights/ #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -55,11 +57,11 @@ impl runtime_parachains::configuration::WeightInfo for /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) fn set_config_with_block_number() -> Weight { // Proof Size summary in bytes: - // Measured: `392` - // Estimated: `7508` - // Minimum execution time: 12_943_000 picoseconds. - Weight::from_parts(13_396_000, 0) - .saturating_add(Weight::from_parts(0, 7508)) + // Measured: `397` + // Estimated: `7528` + // Minimum execution time: 12_920_000 picoseconds. + Weight::from_parts(13_270_000, 0) + .saturating_add(Weight::from_parts(0, 7528)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -73,11 +75,11 @@ impl runtime_parachains::configuration::WeightInfo for /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) fn set_config_with_u32() -> Weight { // Proof Size summary in bytes: - // Measured: `392` - // Estimated: `7508` - // Minimum execution time: 13_054_000 picoseconds. - Weight::from_parts(13_391_000, 0) - .saturating_add(Weight::from_parts(0, 7508)) + // Measured: `397` + // Estimated: `7528` + // Minimum execution time: 12_858_000 picoseconds. + Weight::from_parts(13_184_000, 0) + .saturating_add(Weight::from_parts(0, 7528)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -91,11 +93,11 @@ impl runtime_parachains::configuration::WeightInfo for /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) fn set_config_with_option_u32() -> Weight { // Proof Size summary in bytes: - // Measured: `392` - // Estimated: `7508` - // Minimum execution time: 13_012_000 picoseconds. - Weight::from_parts(13_464_000, 0) - .saturating_add(Weight::from_parts(0, 7508)) + // Measured: `397` + // Estimated: `7528` + // Minimum execution time: 13_117_000 picoseconds. + Weight::from_parts(13_517_000, 0) + .saturating_add(Weight::from_parts(0, 7528)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -109,11 +111,11 @@ impl runtime_parachains::configuration::WeightInfo for /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) fn set_config_with_weight() -> Weight { // Proof Size summary in bytes: - // Measured: `392` - // Estimated: `7508` - // Minimum execution time: 12_998_000 picoseconds. - Weight::from_parts(13_458_000, 0) - .saturating_add(Weight::from_parts(0, 7508)) + // Measured: `397` + // Estimated: `7528` + // Minimum execution time: 12_941_000 picoseconds. + Weight::from_parts(13_507_000, 0) + .saturating_add(Weight::from_parts(0, 7528)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -137,11 +139,29 @@ impl runtime_parachains::configuration::WeightInfo for /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) fn set_config_with_balance() -> Weight { // Proof Size summary in bytes: - // Measured: `392` - // Estimated: `7508` - // Minimum execution time: 12_815_000 picoseconds. - Weight::from_parts(13_368_000, 0) - .saturating_add(Weight::from_parts(0, 7508)) + // Measured: `397` + // Estimated: `7528` + // Minimum execution time: 12_814_000 picoseconds. + Weight::from_parts(13_258_000, 0) + .saturating_add(Weight::from_parts(0, 7528)) + .saturating_add(T::DbWeight::get().reads(4)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: Configuration PendingConfigs (r:1 w:1) + /// Proof Skipped: Configuration PendingConfigs (max_values: Some(1), max_size: None, mode: Measured) + /// Storage: Configuration ActiveConfig (r:1 w:0) + /// Proof Skipped: Configuration ActiveConfig (max_values: Some(1), max_size: None, mode: Measured) + /// Storage: Configuration BypassConsistencyCheck (r:1 w:0) + /// Proof Skipped: Configuration BypassConsistencyCheck (max_values: Some(1), max_size: None, mode: Measured) + /// Storage: ParasShared CurrentSessionIndex (r:1 w:0) + /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) + fn set_config_with_executor_params() -> Weight { + // Proof Size summary in bytes: + // Measured: `397` + // Estimated: `7528` + // Minimum execution time: 13_908_000 picoseconds. + Weight::from_parts(14_382_000, 0) + .saturating_add(Weight::from_parts(0, 7528)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(1)) } diff --git a/runtime/rococo/src/xcm_config.rs b/runtime/rococo/src/xcm_config.rs index 872bbbe1a035..de3bdfde4c0b 100644 --- a/runtime/rococo/src/xcm_config.rs +++ b/runtime/rococo/src/xcm_config.rs @@ -17,8 +17,8 @@ //! XCM configuration for Rococo. use super::{ - parachains_origin, AccountId, AllPalletsWithSystem, Balances, CouncilCollective, ParaId, - Runtime, RuntimeCall, RuntimeEvent, RuntimeOrigin, WeightToFee, XcmPallet, + parachains_origin, AccountId, AllPalletsWithSystem, Balances, CouncilCollective, Dmp, ParaId, + Runtime, RuntimeCall, RuntimeEvent, RuntimeOrigin, TransactionByteFee, WeightToFee, XcmPallet, }; use frame_support::{ match_types, parameter_types, @@ -26,7 +26,12 @@ use frame_support::{ weights::Weight, }; use frame_system::EnsureRoot; -use runtime_common::{crowdloan, paras_registrar, xcm_sender, ToAuthor}; +use rococo_runtime_constants::currency::CENTS; +use runtime_common::{ + crowdloan, paras_registrar, + xcm_sender::{ChildParachainRouter, ExponentialPrice}, + ToAuthor, +}; use sp_core::ConstU32; use xcm::latest::prelude::*; use xcm_builder::{ @@ -82,12 +87,21 @@ type LocalOriginConverter = ( parameter_types! { /// The amount of weight an XCM operation takes. This is a safe overestimate. pub const BaseXcmWeight: Weight = Weight::from_parts(1_000_000_000, 64 * 1024); + /// The asset ID for the asset that we use to pay for message delivery fees. + pub FeeAssetId: AssetId = Concrete(TokenLocation::get()); + /// The base fee for the message delivery fees. + pub const BaseDeliveryFee: u128 = CENTS.saturating_mul(3); } + /// The XCM router. When we want to send an XCM message, we use this type. It amalgamates all of our /// individual routers. pub type XcmRouter = ( // Only one router so far - use DMP to communicate with child parachains. - xcm_sender::ChildParachainRouter, + ChildParachainRouter< + Runtime, + XcmPallet, + ExponentialPrice, + >, ); parameter_types! { diff --git a/runtime/test-runtime/Cargo.toml b/runtime/test-runtime/Cargo.toml index 9b51f0ee4d4e..6d38a0283da0 100644 --- a/runtime/test-runtime/Cargo.toml +++ b/runtime/test-runtime/Cargo.toml @@ -64,11 +64,11 @@ xcm-executor = { path = "../../xcm/xcm-executor", default-features = false } xcm = { path = "../../xcm", default-features = false } [dev-dependencies] -hex-literal = "0.3.4" +hex-literal = "0.4.1" tiny-keccak = { version = "2.0.2", features = ["keccak"] } keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate", branch = "master" } sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master" } -serde_json = "1.0.81" +serde_json = "1.0.96" [build-dependencies] substrate-wasm-builder = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/runtime/test-runtime/src/lib.rs b/runtime/test-runtime/src/lib.rs index 616f1c955dc9..fb114df2ee46 100644 --- a/runtime/test-runtime/src/lib.rs +++ b/runtime/test-runtime/src/lib.rs @@ -671,7 +671,7 @@ construct_runtime! { ParaSessionInfo: parachains_session_info::{Pallet, Storage}, Hrmp: parachains_hrmp::{Pallet, Call, Storage, Event}, Ump: parachains_ump::{Pallet, Call, Storage, Event}, - Dmp: parachains_dmp::{Pallet, Call, Storage}, + Dmp: parachains_dmp::{Pallet, Storage}, Xcm: pallet_xcm::{Pallet, Call, Event, Origin}, ParasDisputes: parachains_disputes::{Pallet, Storage, Event}, diff --git a/runtime/westend/Cargo.toml b/runtime/westend/Cargo.toml index f4e47d53d1d0..4aa2f18067f6 100644 --- a/runtime/westend/Cargo.toml +++ b/runtime/westend/Cargo.toml @@ -85,7 +85,7 @@ pallet-election-provider-support-benchmarking = { git = "https://github.com/pari pallet-nomination-pools-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true } pallet-offences-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true } pallet-session-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true } -hex-literal = { version = "0.3.4", optional = true } +hex-literal = { version = "0.4.1", optional = true } runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false } primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false } @@ -97,10 +97,10 @@ xcm-executor = { package = "xcm-executor", path = "../../xcm/xcm-executor", defa xcm-builder = { package = "xcm-builder", path = "../../xcm/xcm-builder", default-features = false } [dev-dependencies] -hex-literal = "0.3.4" +hex-literal = "0.4.1" tiny-keccak = { version = "2.0.2", features = ["keccak"] } keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate", branch = "master" } -serde_json = "1.0.81" +serde_json = "1.0.96" remote-externalities = { git = "https://github.com/paritytech/substrate", branch = "master", package = "frame-remote-externalities" } tokio = { version = "1.24.2", features = ["macros"] } sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false } diff --git a/runtime/westend/src/lib.rs b/runtime/westend/src/lib.rs index da7d82fa16c4..7d2220c2c0d3 100644 --- a/runtime/westend/src/lib.rs +++ b/runtime/westend/src/lib.rs @@ -113,13 +113,13 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("westend"), impl_name: create_runtime_str!("parity-westend"), authoring_version: 2, - spec_version: 9390, + spec_version: 9410, impl_version: 0, #[cfg(not(feature = "disable-runtime-api"))] apis: RUNTIME_API_VERSIONS, #[cfg(feature = "disable-runtime-api")] apis: sp_version::create_apis_vec![[]], - transaction_version: 19, + transaction_version: 20, state_version: 1, }; @@ -1163,7 +1163,7 @@ construct_runtime! { ParaScheduler: parachains_scheduler::{Pallet, Storage} = 46, Paras: parachains_paras::{Pallet, Call, Storage, Event, Config, ValidateUnsigned} = 47, Initializer: parachains_initializer::{Pallet, Call, Storage} = 48, - Dmp: parachains_dmp::{Pallet, Call, Storage} = 49, + Dmp: parachains_dmp::{Pallet, Storage} = 49, Ump: parachains_ump::{Pallet, Call, Storage, Event} = 50, Hrmp: parachains_hrmp::{Pallet, Call, Storage, Event, Config} = 51, ParaSessionInfo: parachains_session_info::{Pallet, Storage} = 52, @@ -1224,8 +1224,8 @@ pub type Migrations = ( Runtime, NominationPoolsMigrationV4OldPallet, >, - // Unreleased - add new migrations here: pallet_nomination_pools::migration::v5::MigrateToV5, + // Unreleased - add new migrations here: parachains_configuration::migration::v5::MigrateToV5, ); diff --git a/runtime/westend/src/weights/runtime_parachains_configuration.rs b/runtime/westend/src/weights/runtime_parachains_configuration.rs index 2e1d5ac6bf67..8192c978dea9 100644 --- a/runtime/westend/src/weights/runtime_parachains_configuration.rs +++ b/runtime/westend/src/weights/runtime_parachains_configuration.rs @@ -16,24 +16,26 @@ //! Autogenerated weights for `runtime_parachains::configuration` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-03-15, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2023-04-07, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `bm6`, CPU: `Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz` +//! HOSTNAME: `bm3`, CPU: `Intel(R) Core(TM) i7-7700K CPU @ 4.20GHz` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("westend-dev"), DB CACHE: 1024 // Executed Command: -// ./target/production/polkadot +// target/production/polkadot // benchmark // pallet -// --chain=westend-dev // --steps=50 // --repeat=20 -// --pallet=runtime_parachains::configuration // --extrinsic=* // --execution=wasm // --wasm-execution=compiled +// --heap-pages=4096 +// --json-file=/var/lib/gitlab-runner/builds/zyw4fam_/0/parity/mirrors/polkadot/.git/.artifacts/bench.json +// --pallet=runtime_parachains::configuration +// --chain=westend-dev // --header=./file_header.txt -// --output=./runtime/westend/src/weights/runtime_parachains_configuration.rs +// --output=./runtime/westend/src/weights/ #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -55,8 +57,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `90` // Estimated: `4725` - // Minimum execution time: 9_426_000 picoseconds. - Weight::from_parts(9_764_000, 0) + // Minimum execution time: 9_585_000 picoseconds. + Weight::from_parts(9_914_000, 0) .saturating_add(Weight::from_parts(0, 4725)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -71,8 +73,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `90` // Estimated: `4725` - // Minimum execution time: 9_836_000 picoseconds. - Weight::from_parts(10_178_000, 0) + // Minimum execution time: 9_620_000 picoseconds. + Weight::from_parts(9_963_000, 0) .saturating_add(Weight::from_parts(0, 4725)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -87,8 +89,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `90` // Estimated: `4725` - // Minimum execution time: 9_515_000 picoseconds. - Weight::from_parts(9_857_000, 0) + // Minimum execution time: 9_687_000 picoseconds. + Weight::from_parts(9_932_000, 0) .saturating_add(Weight::from_parts(0, 4725)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -103,8 +105,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `90` // Estimated: `4725` - // Minimum execution time: 9_432_000 picoseconds. - Weight::from_parts(9_864_000, 0) + // Minimum execution time: 9_635_000 picoseconds. + Weight::from_parts(9_878_000, 0) .saturating_add(Weight::from_parts(0, 4725)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -129,8 +131,24 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `90` // Estimated: `4725` - // Minimum execution time: 9_655_000 picoseconds. - Weight::from_parts(9_975_000, 0) + // Minimum execution time: 9_626_000 picoseconds. + Weight::from_parts(9_987_000, 0) + .saturating_add(Weight::from_parts(0, 4725)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: Configuration PendingConfigs (r:1 w:1) + /// Proof Skipped: Configuration PendingConfigs (max_values: Some(1), max_size: None, mode: Measured) + /// Storage: Configuration BypassConsistencyCheck (r:1 w:0) + /// Proof Skipped: Configuration BypassConsistencyCheck (max_values: Some(1), max_size: None, mode: Measured) + /// Storage: ParasShared CurrentSessionIndex (r:1 w:0) + /// Proof Skipped: ParasShared CurrentSessionIndex (max_values: Some(1), max_size: None, mode: Measured) + fn set_config_with_executor_params() -> Weight { + // Proof Size summary in bytes: + // Measured: `90` + // Estimated: `4725` + // Minimum execution time: 10_483_000 picoseconds. + Weight::from_parts(10_695_000, 0) .saturating_add(Weight::from_parts(0, 4725)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) diff --git a/runtime/westend/src/xcm_config.rs b/runtime/westend/src/xcm_config.rs index 24b161e42a01..148405c7e4e7 100644 --- a/runtime/westend/src/xcm_config.rs +++ b/runtime/westend/src/xcm_config.rs @@ -17,16 +17,21 @@ //! XCM configurations for Westend. use super::{ - parachains_origin, weights, AccountId, AllPalletsWithSystem, Balances, ParaId, Runtime, - RuntimeCall, RuntimeEvent, RuntimeOrigin, WeightToFee, XcmPallet, + parachains_origin, weights, AccountId, AllPalletsWithSystem, Balances, Dmp, ParaId, Runtime, + RuntimeCall, RuntimeEvent, RuntimeOrigin, TransactionByteFee, WeightToFee, XcmPallet, }; use frame_support::{ parameter_types, traits::{Contains, Everything, Nothing}, }; use frame_system::EnsureRoot; -use runtime_common::{crowdloan, paras_registrar, xcm_sender, ToAuthor}; +use runtime_common::{ + crowdloan, paras_registrar, + xcm_sender::{ChildParachainRouter, ExponentialPrice}, + ToAuthor, +}; use sp_core::ConstU32; +use westend_runtime_constants::currency::CENTS; use xcm::latest::prelude::*; use xcm_builder::{ AccountId32Aliases, AllowExplicitUnpaidExecutionFrom, AllowKnownQueryResponses, @@ -44,6 +49,10 @@ parameter_types! { pub UniversalLocation: InteriorMultiLocation = ThisNetwork::get().into(); pub CheckAccount: AccountId = XcmPallet::check_account(); pub LocalCheckAccount: (AccountId, MintLocation) = (CheckAccount::get(), MintLocation::Local); + /// The asset ID for the asset that we use to pay for message delivery fees. + pub FeeAssetId: AssetId = Concrete(TokenLocation::get()); + /// The base fee for the message delivery fees. + pub const BaseDeliveryFee: u128 = CENTS.saturating_mul(3); } pub type LocationConverter = @@ -73,7 +82,11 @@ type LocalOriginConverter = ( /// individual routers. pub type XcmRouter = ( // Only one router so far - use DMP to communicate with child parachains. - xcm_sender::ChildParachainRouter, + ChildParachainRouter< + Runtime, + XcmPallet, + ExponentialPrice, + >, ); parameter_types! { diff --git a/scripts/ci/github/check_weights_swc.sh b/scripts/ci/github/check_weights_swc.sh new file mode 100755 index 000000000000..35652f4fde83 --- /dev/null +++ b/scripts/ci/github/check_weights_swc.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Need to set globstar for ** magic +shopt -s globstar + +RUNTIME=$1 +VERSION=$2 +echo "
" +echo "Weight changes for $RUNTIME" +echo +swc compare commits \ + --method asymptotic \ + --offline \ + --path-pattern "./runtime/$RUNTIME/src/weights/**/*.rs" \ + --no-color \ + --format markdown \ + --strip-path-prefix "runtime/$RUNTIME/src/weights/" \ + "$VERSION" + #--ignore-errors +echo +echo "
" diff --git a/scripts/ci/github/verify_updated_weights.sh b/scripts/ci/github/verify_updated_weights.sh new file mode 100755 index 000000000000..54db1de21ca0 --- /dev/null +++ b/scripts/ci/github/verify_updated_weights.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +ROOT="$(dirname "$0")/../../.." +RUNTIME="$1" + +# If we're on a mac, use gdate for date command (requires coreutils installed via brew) +if [[ "$OSTYPE" == "darwin"* ]]; then + DATE="gdate" +else + DATE="date" +fi + +function check_date() { + # Get the dates as input arguments + LAST_RUN="$1" + TODAY="$($DATE +%Y-%m-%d)" + # Calculate the date two days before today + CUTOFF=$($DATE -d "$TODAY - 2 days" +%Y-%m-%d) + + if [[ "$LAST_RUN" > "$CUTOFF" ]]; then + return 0 + else + return 1 + fi +} + +check_weights(){ + FILE=$1 + CUR_DATE=$2 + DATE_REGEX='[0-9]{4}-[0-9]{2}-[0-9]{2}' + LAST_UPDATE="$(grep -E "//! DATE: $DATE_REGEX" "$FILE" | sed -r "s/.*DATE: ($DATE_REGEX).*/\1/")" + # If the file does not contain a date, flag it as an error. + if [ -z "$LAST_UPDATE" ]; then + echo "Skipping $FILE, no date found." + return 0 + fi + if ! check_date "$LAST_UPDATE" ; then + echo "ERROR: $FILE was not updated for the current date. Last update: $LAST_UPDATE" + return 1 + fi + # echo "OK: $FILE" +} + +echo "Checking weights for $RUNTIME" +CUR_DATE="$(date +%Y-%m-%d)" +HAS_ERROR=0 +for FILE in "$ROOT"/runtime/"$RUNTIME"/src/weights/*.rs; do + if ! check_weights "$FILE" "$CUR_DATE"; then + HAS_ERROR=1 + fi +done + +if [ $HAS_ERROR -eq 1 ]; then + echo "ERROR: One or more weights files were not updated during the last benchmark run. Check the logs above." + exit 1 +fi \ No newline at end of file diff --git a/scripts/ci/gitlab/pipeline/check.yml b/scripts/ci/gitlab/pipeline/check.yml index 7a56777b09f1..9ad4f2880dd7 100644 --- a/scripts/ci/gitlab/pipeline/check.yml +++ b/scripts/ci/gitlab/pipeline/check.yml @@ -69,7 +69,7 @@ check-try-runtime: export RUST_LOG=remote-ext=debug,runtime=debug echo "---------- Running try-runtime for ${NETWORK} ----------" time cargo build --locked --release -p "$NETWORK"-runtime - time cargo run ---locked -release --features try-runtime try-runtime \ + time cargo run --locked --release --features try-runtime try-runtime \ --runtime ./target/release/wbuild/"$NETWORK"-runtime/target/wasm32-unknown-unknown/release/"$NETWORK"_runtime.wasm \ -lruntime=debug \ --chain=${NETWORK}-dev \ diff --git a/utils/staking-miner/src/main.rs b/utils/staking-miner/src/main.rs index 044ba4ed22df..1ba42b21d91c 100644 --- a/utils/staking-miner/src/main.rs +++ b/utils/staking-miner/src/main.rs @@ -50,7 +50,7 @@ use frame_election_provider_support::NposSolver; use frame_support::traits::Get; use futures_util::StreamExt; use jsonrpsee::ws_client::{WsClient, WsClientBuilder}; -use remote_externalities::{Builder, Mode, OnlineConfig}; +use remote_externalities::{Builder, Mode, OnlineConfig, Transport}; use rpc::{RpcApiClient, SharedRpcClient}; use runtime_versions::RuntimeVersions; use signal_hook::consts::signal::*; @@ -314,7 +314,7 @@ where pallets.extend(additional); Builder::::new() .mode(Mode::Online(OnlineConfig { - transport: client.into_inner().into(), + transport: Transport::Uri(client.uri().to_owned()), at, pallets, hashed_prefixes: vec![>::prefix_hash()], diff --git a/utils/staking-miner/src/rpc.rs b/utils/staking-miner/src/rpc.rs index ae978ee3382d..a95e89191a49 100644 --- a/utils/staking-miner/src/rpc.rs +++ b/utils/staking-miner/src/rpc.rs @@ -103,9 +103,11 @@ pub trait RpcApi { fn subscribe_finalized_heads(&self); } +type Uri = String; + /// Wraps a shared web-socket JSON-RPC client that can be cloned. #[derive(Clone, Debug)] -pub(crate) struct SharedRpcClient(Arc); +pub(crate) struct SharedRpcClient(Arc, Uri); impl Deref for SharedRpcClient { type Target = WsClient; @@ -116,9 +118,9 @@ impl Deref for SharedRpcClient { } impl SharedRpcClient { - /// Consume and extract the inner client. - pub fn into_inner(self) -> Arc { - self.0 + /// Get the URI of the client. + pub fn uri(&self) -> &str { + &self.1 } /// Create a new shared JSON-RPC web-socket client. @@ -134,7 +136,7 @@ impl SharedRpcClient { .max_concurrent_requests(u32::MAX as usize) .build(uri) .await?; - Ok(Self(Arc::new(client))) + Ok(Self(Arc::new(client), uri.to_owned())) } /// Get a storage item and decode it as `T`. diff --git a/xcm/Cargo.toml b/xcm/Cargo.toml index b6c0f3d92163..e2a123a16680 100644 --- a/xcm/Cargo.toml +++ b/xcm/Cargo.toml @@ -19,7 +19,7 @@ xcm-procedural = { path = "procedural" } [dev-dependencies] sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" } hex = "0.4.3" -hex-literal = "0.3.4" +hex-literal = "0.4.1" [features] default = ["std"] diff --git a/xcm/pallet-xcm/src/lib.rs b/xcm/pallet-xcm/src/lib.rs index 5af903ca37b3..cd6fb6321a8e 100644 --- a/xcm/pallet-xcm/src/lib.rs +++ b/xcm/pallet-xcm/src/lib.rs @@ -1146,7 +1146,10 @@ impl Pallet { BuyExecution { fees, weight_limit }, DepositAsset { assets: Wild(AllCounted(max_assets)), beneficiary }, ]); - let mut message = Xcm(vec![TransferReserveAsset { assets, dest, xcm }]); + let mut message = Xcm(vec![ + SetFeesMode { jit_withdraw: true }, + TransferReserveAsset { assets, dest, xcm }, + ]); let weight = T::Weigher::weight(&mut message).map_err(|()| Error::::UnweighableMessage)?; let hash = message.using_encoded(sp_io::hashing::blake2_256); @@ -1205,6 +1208,7 @@ impl Pallet { ]); let mut message = Xcm(vec![ WithdrawAsset(assets), + SetFeesMode { jit_withdraw: true }, InitiateTeleport { assets: Wild(AllCounted(max_assets)), dest, xcm }, ]); let weight = diff --git a/xcm/pallet-xcm/src/tests.rs b/xcm/pallet-xcm/src/tests.rs index 919eede0fe4a..ae359116e023 100644 --- a/xcm/pallet-xcm/src/tests.rs +++ b/xcm/pallet-xcm/src/tests.rs @@ -349,7 +349,7 @@ fn teleport_assets_works() { (ParaId::from(PARA_ID).into_account_truncating(), INITIAL_BALANCE), ]; new_test_ext_with_balances(balances).execute_with(|| { - let weight = BaseXcmWeight::get() * 2; + let weight = BaseXcmWeight::get() * 3; assert_eq!(Balances::total_balance(&ALICE), INITIAL_BALANCE); let dest: MultiLocation = AccountId32 { network: None, id: BOB.into() }.into(); assert_ok!(XcmPallet::teleport_assets( @@ -392,7 +392,7 @@ fn limited_teleport_assets_works() { (ParaId::from(PARA_ID).into_account_truncating(), INITIAL_BALANCE), ]; new_test_ext_with_balances(balances).execute_with(|| { - let weight = BaseXcmWeight::get() * 2; + let weight = BaseXcmWeight::get() * 3; assert_eq!(Balances::total_balance(&ALICE), INITIAL_BALANCE); let dest: MultiLocation = AccountId32 { network: None, id: BOB.into() }.into(); assert_ok!(XcmPallet::limited_teleport_assets( @@ -436,7 +436,7 @@ fn unlimited_teleport_assets_works() { (ParaId::from(PARA_ID).into_account_truncating(), INITIAL_BALANCE), ]; new_test_ext_with_balances(balances).execute_with(|| { - let weight = BaseXcmWeight::get() * 2; + let weight = BaseXcmWeight::get() * 3; assert_eq!(Balances::total_balance(&ALICE), INITIAL_BALANCE); let dest: MultiLocation = AccountId32 { network: None, id: BOB.into() }.into(); assert_ok!(XcmPallet::limited_teleport_assets( @@ -478,7 +478,7 @@ fn reserve_transfer_assets_works() { (ParaId::from(PARA_ID).into_account_truncating(), INITIAL_BALANCE), ]; new_test_ext_with_balances(balances).execute_with(|| { - let weight = BaseXcmWeight::get(); + let weight = BaseXcmWeight::get() * 2; let dest: MultiLocation = Junction::AccountId32 { network: None, id: ALICE.into() }.into(); assert_eq!(Balances::total_balance(&ALICE), INITIAL_BALANCE); assert_ok!(XcmPallet::reserve_transfer_assets( @@ -525,7 +525,7 @@ fn limited_reserve_transfer_assets_works() { (ParaId::from(PARA_ID).into_account_truncating(), INITIAL_BALANCE), ]; new_test_ext_with_balances(balances).execute_with(|| { - let weight = BaseXcmWeight::get(); + let weight = BaseXcmWeight::get() * 2; let dest: MultiLocation = Junction::AccountId32 { network: None, id: ALICE.into() }.into(); assert_eq!(Balances::total_balance(&ALICE), INITIAL_BALANCE); assert_ok!(XcmPallet::limited_reserve_transfer_assets( @@ -573,7 +573,7 @@ fn unlimited_reserve_transfer_assets_works() { (ParaId::from(PARA_ID).into_account_truncating(), INITIAL_BALANCE), ]; new_test_ext_with_balances(balances).execute_with(|| { - let weight = BaseXcmWeight::get(); + let weight = BaseXcmWeight::get() * 2; let dest: MultiLocation = Junction::AccountId32 { network: None, id: ALICE.into() }.into(); assert_eq!(Balances::total_balance(&ALICE), INITIAL_BALANCE); assert_ok!(XcmPallet::limited_reserve_transfer_assets( diff --git a/xcm/procedural/Cargo.toml b/xcm/procedural/Cargo.toml index 22ec39077c97..4da7ac1e2223 100644 --- a/xcm/procedural/Cargo.toml +++ b/xcm/procedural/Cargo.toml @@ -8,7 +8,7 @@ version.workspace = true proc-macro = true [dependencies] -proc-macro2 = "1.0.43" -quote = "1.0.20" -syn = "1.0.95" +proc-macro2 = "1.0.56" +quote = "1.0.26" +syn = "2.0.15" Inflector = "0.11.4" diff --git a/xcm/src/v3/mod.rs b/xcm/src/v3/mod.rs index 39a7ae32ee1e..d4e8e52af52c 100644 --- a/xcm/src/v3/mod.rs +++ b/xcm/src/v3/mod.rs @@ -925,7 +925,7 @@ pub enum Instruction { /// asset to be transferred. /// /// - `asset`: The asset to be unlocked. - /// - `owner`: The owner of the asset on the local chain. + /// - `target`: The owner of the asset on the local chain. /// /// Safety: No concerns. /// @@ -1191,8 +1191,9 @@ impl TryFrom> for Xcm { } } -/// Default value for the proof size weight component. Set at 64 KB. -const DEFAULT_PROOF_SIZE: u64 = 64 * 1024; +/// Default value for the proof size weight component. Set at 0 KB. +/// NOTE: Make sure this is removed after we properly account for PoV weights. +const DEFAULT_PROOF_SIZE: u64 = 0; // Convert from a v2 instruction to a v3 instruction. impl TryFrom> for Instruction { diff --git a/xcm/xcm-builder/src/universal_exports.rs b/xcm/xcm-builder/src/universal_exports.rs index ddf1c9ed336a..81042a1ad4a3 100644 --- a/xcm/xcm-builder/src/universal_exports.rs +++ b/xcm/xcm-builder/src/universal_exports.rs @@ -158,13 +158,11 @@ impl(bridge, message)?; - if let Some(payment) = maybe_payment { - cost.push(payment); - } - Ok((v, cost)) + let message = Xcm(vec![ + UnpaidExecution { weight_limit: Unlimited, check_origin: None }, + ExportMessage { network: remote_network, destination: remote_location, xcm }, + ]); + validate_send::(bridge, message) } fn deliver(validation: Router::Ticket) -> Result {