From f1b2189e8312821a9f8a79dc4f0183d314273faa Mon Sep 17 00:00:00 2001 From: Alexandru Vasile <60601340+lexnv@users.noreply.github.com> Date: Wed, 28 Feb 2024 11:43:58 +0200 Subject: [PATCH 01/13] rpc-v2/tx/tests: Add transaction broadcast tests and check propagated tx status (#3193) This PR adds tests for the `transaction_broadcast` method. The testing needs to coordinate the following components: - The `TestApi` marks transactions as invalid and implements `ChainApi::validate_transaction` - this is what dictates if a transaction is valid or not and is called from within the `BasicPool` - The `BasicPool` which maintains the transactions and implements `submit_and_watch` needed by the tx broadcast to submit the transaction - The status of the transaction pool is exposed by mocking the BasicPool - The `ChainHeadMockClient` which mocks the `BlockchainEvents::import_notification_stream` needed by the tx broadcast to know to which blocks the transaction is submitted The following changes have been added to the substrate testing to accommodate this: - `TestApi` gets ` remove_invalid`, counterpart to `add_invalid` to ensure an invalid transaction can become valid again; as well as a priority setter for extrinsics - `BasicPool` test constructor is extended with options for the `PoolRotator` - this mechanism is needed because transactions are banned for 30mins (default) after they are declared invalid - testing bypasses this by providing a `Duration::ZERO` ### Testing Scenarios - Capture the status of the transaction as it is normally broadcasted - `transaction_stop` is valid while the transaction is in progress - A future transaction is handled when the dependencies are completed - Try to resubmit the transaction at a later block (currently invalid) - An invalid transaction status is propagated; the transaction is marked as temporarily banned; then the ban expires and transaction is resubmitted This builds on top of: https://github.com/paritytech/polkadot-sdk/pull/3079 Part of: https://github.com/paritytech/polkadot-sdk/issues/3084 cc @paritytech/subxt-team --------- Signed-off-by: Alexandru Vasile Co-authored-by: James Wilson --- .../rpc-spec-v2/src/chain_head/test_utils.rs | 4 +- .../rpc-spec-v2/src/transaction/tests.rs | 238 -------- .../src/transaction/tests/executor.rs | 100 ++++ .../src/transaction/tests/middleware_pool.rs | 187 +++++++ .../rpc-spec-v2/src/transaction/tests/mod.rs | 24 + .../src/transaction/tests/setup.rs | 120 ++++ .../tests/transaction_broadcast_tests.rs | 523 ++++++++++++++++++ substrate/client/transaction-pool/src/lib.rs | 3 +- .../client/transaction-pool/tests/pool.rs | 3 +- .../runtime/transaction-pool/src/lib.rs | 27 +- 10 files changed, 985 insertions(+), 244 deletions(-) delete mode 100644 substrate/client/rpc-spec-v2/src/transaction/tests.rs create mode 100644 substrate/client/rpc-spec-v2/src/transaction/tests/executor.rs create mode 100644 substrate/client/rpc-spec-v2/src/transaction/tests/middleware_pool.rs create mode 100644 substrate/client/rpc-spec-v2/src/transaction/tests/mod.rs create mode 100644 substrate/client/rpc-spec-v2/src/transaction/tests/setup.rs create mode 100644 substrate/client/rpc-spec-v2/src/transaction/tests/transaction_broadcast_tests.rs diff --git a/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs b/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs index d63a98a5cb0d..e81bd4bfa0b0 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs @@ -63,7 +63,7 @@ impl ChainHeadMockClient { BlockImportNotification::new(header.hash(), BlockOrigin::Own, header, true, None, sink); for sink in self.import_sinks.lock().iter_mut() { - sink.unbounded_send(notification.clone()).unwrap(); + let _ = sink.unbounded_send(notification.clone()); } } @@ -83,7 +83,7 @@ impl ChainHeadMockClient { let notification = FinalityNotification::from_summary(summary, sink); for sink in self.finality_sinks.lock().iter_mut() { - sink.unbounded_send(notification.clone()).unwrap(); + let _ = sink.unbounded_send(notification.clone()); } } } diff --git a/substrate/client/rpc-spec-v2/src/transaction/tests.rs b/substrate/client/rpc-spec-v2/src/transaction/tests.rs deleted file mode 100644 index 382f5adeae19..000000000000 --- a/substrate/client/rpc-spec-v2/src/transaction/tests.rs +++ /dev/null @@ -1,238 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use super::*; -use crate::{ - chain_head::test_utils::ChainHeadMockClient, hex_string, - transaction::TransactionBroadcast as RpcTransactionBroadcast, -}; -use assert_matches::assert_matches; -use codec::Encode; -use futures::Future; -use jsonrpsee::{rpc_params, MethodsError as Error, RpcModule}; -use sc_transaction_pool::*; -use sc_transaction_pool_api::{ChainEvent, MaintainedTransactionPool, TransactionPool}; -use sp_core::{testing::TaskExecutor, traits::SpawnNamed}; -use std::{pin::Pin, sync::Arc, time::Duration}; -use substrate_test_runtime_client::{prelude::*, AccountKeyring::*, Client}; -use substrate_test_runtime_transaction_pool::{uxt, TestApi}; -use tokio::sync::mpsc; - -type Block = substrate_test_runtime_client::runtime::Block; - -/// Wrap the `TaskExecutor` to know when the broadcast future is dropped. -#[derive(Clone)] -struct TaskExecutorBroadcast { - executor: TaskExecutor, - sender: mpsc::UnboundedSender<()>, -} - -/// The channel that receives events when the broadcast futures are dropped. -type TaskExecutorRecv = mpsc::UnboundedReceiver<()>; - -impl TaskExecutorBroadcast { - /// Construct a new `TaskExecutorBroadcast` and a receiver to know when the broadcast futures - /// are dropped. - fn new() -> (Self, TaskExecutorRecv) { - let (sender, recv) = mpsc::unbounded_channel(); - - (Self { executor: TaskExecutor::new(), sender }, recv) - } -} - -impl SpawnNamed for TaskExecutorBroadcast { - fn spawn( - &self, - name: &'static str, - group: Option<&'static str>, - future: futures::future::BoxFuture<'static, ()>, - ) { - let sender = self.sender.clone(); - let future = Box::pin(async move { - future.await; - let _ = sender.send(()); - }); - - self.executor.spawn(name, group, future) - } - - fn spawn_blocking( - &self, - name: &'static str, - group: Option<&'static str>, - future: futures::future::BoxFuture<'static, ()>, - ) { - let sender = self.sender.clone(); - let future = Box::pin(async move { - future.await; - let _ = sender.send(()); - }); - - self.executor.spawn_blocking(name, group, future) - } -} - -/// Initial Alice account nonce. -const ALICE_NONCE: u64 = 209; - -fn create_basic_pool_with_genesis( - test_api: Arc, -) -> (BasicPool, Pin + Send>>) { - let genesis_hash = { - test_api - .chain() - .read() - .block_by_number - .get(&0) - .map(|blocks| blocks[0].0.header.hash()) - .expect("there is block 0. qed") - }; - BasicPool::new_test(test_api, genesis_hash, genesis_hash) -} - -fn maintained_pool() -> (BasicPool, Arc, futures::executor::ThreadPool) { - let api = Arc::new(TestApi::with_alice_nonce(ALICE_NONCE)); - let (pool, background_task) = create_basic_pool_with_genesis(api.clone()); - - let thread_pool = futures::executor::ThreadPool::new().unwrap(); - thread_pool.spawn_ok(background_task); - (pool, api, thread_pool) -} - -fn setup_api() -> ( - Arc, - Arc>, - Arc>>, - RpcModule< - TransactionBroadcast, ChainHeadMockClient>>, - >, - TaskExecutorRecv, -) { - let (pool, api, _) = maintained_pool(); - let pool = Arc::new(pool); - - let builder = TestClientBuilder::new(); - let client = Arc::new(builder.build()); - let client_mock = Arc::new(ChainHeadMockClient::new(client.clone())); - - let (task_executor, executor_recv) = TaskExecutorBroadcast::new(); - - let tx_api = - RpcTransactionBroadcast::new(client_mock.clone(), pool.clone(), Arc::new(task_executor)) - .into_rpc(); - - (api, pool, client_mock, tx_api, executor_recv) -} - -#[tokio::test] -async fn tx_broadcast_enters_pool() { - let (api, pool, client_mock, tx_api, _) = setup_api(); - - // Start at block 1. - let block_1_header = api.push_block(1, vec![], true); - - let uxt = uxt(Alice, ALICE_NONCE); - let xt = hex_string(&uxt.encode()); - - let operation_id: String = - tx_api.call("transaction_unstable_broadcast", rpc_params![&xt]).await.unwrap(); - - // Announce block 1 to `transaction_unstable_broadcast`. - client_mock.trigger_import_stream(block_1_header).await; - - // Ensure the tx propagated from `transaction_unstable_broadcast` to the transaction pool. - - // TODO: Improve testability by extending the `transaction_unstable_broadcast` with - // a middleware trait that intercepts the transaction status for testing. - let mut num_retries = 12; - while num_retries > 0 && pool.status().ready != 1 { - tokio::time::sleep(Duration::from_secs(5)).await; - num_retries -= 1; - } - assert_eq!(1, pool.status().ready); - assert_eq!(uxt.encode().len(), pool.status().ready_bytes); - - // Import block 2 with the transaction included. - let block_2_header = api.push_block(2, vec![uxt.clone()], true); - let block_2 = block_2_header.hash(); - - // Announce block 2 to the pool. - let event = ChainEvent::NewBestBlock { hash: block_2, tree_route: None }; - pool.maintain(event).await; - - assert_eq!(0, pool.status().ready); - - // Stop call can still be made. - let _: () = tx_api - .call("transaction_unstable_stop", rpc_params![&operation_id]) - .await - .unwrap(); -} - -#[tokio::test] -async fn tx_broadcast_invalid_tx() { - let (_, pool, _, tx_api, mut exec_recv) = setup_api(); - - // Invalid parameters. - let err = tx_api - .call::<_, serde_json::Value>("transaction_unstable_broadcast", [1u8]) - .await - .unwrap_err(); - assert_matches!(err, - Error::JsonRpc(err) if err.code() == super::error::json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid params" - ); - - assert_eq!(0, pool.status().ready); - - // Invalid transaction that cannot be decoded. The broadcast silently exits. - let xt = "0xdeadbeef"; - let operation_id: String = - tx_api.call("transaction_unstable_broadcast", rpc_params![&xt]).await.unwrap(); - - assert_eq!(0, pool.status().ready); - - // Await the broadcast future to exit. - // Without this we'd be subject to races, where we try to call the stop before the tx is - // dropped. - exec_recv.recv().await.unwrap(); - - // The broadcast future was dropped, and the operation is no longer active. - // When the operation is not active, either from the tx being finalized or a - // terminal error; the stop method should return an error. - let err = tx_api - .call::<_, serde_json::Value>("transaction_unstable_stop", rpc_params![&operation_id]) - .await - .unwrap_err(); - assert_matches!(err, - Error::JsonRpc(err) if err.code() == super::error::json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid operation id" - ); -} - -#[tokio::test] -async fn tx_invalid_stop() { - let (_, _, _, tx_api, _) = setup_api(); - - // Make an invalid stop call. - let err = tx_api - .call::<_, serde_json::Value>("transaction_unstable_stop", ["invalid_operation_id"]) - .await - .unwrap_err(); - assert_matches!(err, - Error::JsonRpc(err) if err.code() == super::error::json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid operation id" - ); -} diff --git a/substrate/client/rpc-spec-v2/src/transaction/tests/executor.rs b/substrate/client/rpc-spec-v2/src/transaction/tests/executor.rs new file mode 100644 index 000000000000..ff9aca79887c --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/transaction/tests/executor.rs @@ -0,0 +1,100 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use sp_core::{testing::TaskExecutor, traits::SpawnNamed}; +use std::sync::{atomic::AtomicUsize, Arc}; +use tokio::sync::mpsc; + +/// Wrap the `TaskExecutor` to know when the broadcast future is dropped. +#[derive(Clone)] +pub struct TaskExecutorBroadcast { + executor: TaskExecutor, + sender: mpsc::UnboundedSender<()>, + num_tasks: Arc, +} + +/// The channel that receives events when the broadcast futures are dropped. +pub type TaskExecutorRecv = mpsc::UnboundedReceiver<()>; + +/// The state of the `TaskExecutorBroadcast`. +pub struct TaskExecutorState { + pub recv: TaskExecutorRecv, + pub num_tasks: Arc, +} + +impl TaskExecutorState { + pub fn num_tasks(&self) -> usize { + self.num_tasks.load(std::sync::atomic::Ordering::Acquire) + } +} + +impl TaskExecutorBroadcast { + /// Construct a new `TaskExecutorBroadcast` and a receiver to know when the broadcast futures + /// are dropped. + pub fn new() -> (Self, TaskExecutorState) { + let (sender, recv) = mpsc::unbounded_channel(); + let num_tasks = Arc::new(AtomicUsize::new(0)); + + ( + Self { executor: TaskExecutor::new(), sender, num_tasks: num_tasks.clone() }, + TaskExecutorState { recv, num_tasks }, + ) + } +} + +impl SpawnNamed for TaskExecutorBroadcast { + fn spawn( + &self, + name: &'static str, + group: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ) { + let sender = self.sender.clone(); + let num_tasks = self.num_tasks.clone(); + + let future = Box::pin(async move { + num_tasks.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + future.await; + num_tasks.fetch_sub(1, std::sync::atomic::Ordering::AcqRel); + + let _ = sender.send(()); + }); + + self.executor.spawn(name, group, future) + } + + fn spawn_blocking( + &self, + name: &'static str, + group: Option<&'static str>, + future: futures::future::BoxFuture<'static, ()>, + ) { + let sender = self.sender.clone(); + let num_tasks = self.num_tasks.clone(); + + let future = Box::pin(async move { + num_tasks.fetch_add(1, std::sync::atomic::Ordering::AcqRel); + future.await; + num_tasks.fetch_sub(1, std::sync::atomic::Ordering::AcqRel); + + let _ = sender.send(()); + }); + + self.executor.spawn_blocking(name, group, future) + } +} diff --git a/substrate/client/rpc-spec-v2/src/transaction/tests/middleware_pool.rs b/substrate/client/rpc-spec-v2/src/transaction/tests/middleware_pool.rs new file mode 100644 index 000000000000..aa8ac572dec9 --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/transaction/tests/middleware_pool.rs @@ -0,0 +1,187 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use codec::Encode; +use futures::Future; +use sc_transaction_pool::BasicPool; +use sc_transaction_pool_api::{ + ImportNotificationStream, PoolFuture, PoolStatus, ReadyTransactions, TransactionFor, + TransactionPool, TransactionSource, TransactionStatusStreamFor, TxHash, +}; + +use crate::hex_string; +use futures::{FutureExt, StreamExt}; + +use sp_runtime::traits::{Block as BlockT, NumberFor}; +use std::{collections::HashMap, pin::Pin, sync::Arc}; +use substrate_test_runtime_transaction_pool::TestApi; +use tokio::sync::mpsc; + +pub type Block = substrate_test_runtime_client::runtime::Block; + +pub type TxTestPool = MiddlewarePool; +pub type TxStatusType = sc_transaction_pool_api::TransactionStatus< + sc_transaction_pool_api::TxHash, + sc_transaction_pool_api::BlockHash, +>; +pub type TxStatusTypeTest = TxStatusType; + +/// The type of the event that the middleware captures. +#[derive(Debug, PartialEq)] +pub enum MiddlewarePoolEvent { + TransactionStatus { + transaction: String, + status: sc_transaction_pool_api::TransactionStatus< + ::Hash, + ::Hash, + >, + }, + PoolError { + transaction: String, + err: String, + }, +} + +/// The channel that receives events when the broadcast futures are dropped. +pub type MiddlewarePoolRecv = mpsc::UnboundedReceiver; + +/// Add a middleware to the transaction pool. +/// +/// This wraps the `submit_and_watch` to gain access to the events. +pub struct MiddlewarePool { + pub inner_pool: Arc>, + /// Send the middleware events to the test. + sender: mpsc::UnboundedSender, +} + +impl MiddlewarePool { + /// Construct a new [`MiddlewarePool`]. + pub fn new(pool: Arc>) -> (Self, MiddlewarePoolRecv) { + let (sender, recv) = mpsc::unbounded_channel(); + (MiddlewarePool { inner_pool: pool, sender }, recv) + } +} + +impl TransactionPool for MiddlewarePool { + type Block = as TransactionPool>::Block; + type Hash = as TransactionPool>::Hash; + type InPoolTransaction = as TransactionPool>::InPoolTransaction; + type Error = as TransactionPool>::Error; + + fn submit_at( + &self, + at: ::Hash, + source: TransactionSource, + xts: Vec>, + ) -> PoolFuture, Self::Error>>, Self::Error> { + self.inner_pool.submit_at(at, source, xts) + } + + fn submit_one( + &self, + at: ::Hash, + source: TransactionSource, + xt: TransactionFor, + ) -> PoolFuture, Self::Error> { + self.inner_pool.submit_one(at, source, xt) + } + + fn submit_and_watch( + &self, + at: ::Hash, + source: TransactionSource, + xt: TransactionFor, + ) -> PoolFuture>>, Self::Error> { + let pool = self.inner_pool.clone(); + let sender = self.sender.clone(); + let transaction = hex_string(&xt.encode()); + + async move { + let watcher = match pool.submit_and_watch(at, source, xt).await { + Ok(watcher) => watcher, + Err(err) => { + let _ = sender.send(MiddlewarePoolEvent::PoolError { + transaction: transaction.clone(), + err: err.to_string(), + }); + return Err(err); + }, + }; + + let watcher = watcher.map(move |status| { + let sender = sender.clone(); + let transaction = transaction.clone(); + + let _ = sender.send(MiddlewarePoolEvent::TransactionStatus { + transaction, + status: status.clone(), + }); + + status + }); + + Ok(watcher.boxed()) + } + .boxed() + } + + fn remove_invalid(&self, hashes: &[TxHash]) -> Vec> { + self.inner_pool.remove_invalid(hashes) + } + + fn status(&self) -> PoolStatus { + self.inner_pool.status() + } + + fn import_notification_stream(&self) -> ImportNotificationStream> { + self.inner_pool.import_notification_stream() + } + + fn hash_of(&self, xt: &TransactionFor) -> TxHash { + self.inner_pool.hash_of(xt) + } + + fn on_broadcasted(&self, propagations: HashMap, Vec>) { + self.inner_pool.on_broadcasted(propagations) + } + + fn ready_transaction(&self, hash: &TxHash) -> Option> { + self.inner_pool.ready_transaction(hash) + } + + fn ready_at( + &self, + at: NumberFor, + ) -> Pin< + Box< + dyn Future< + Output = Box> + Send>, + > + Send, + >, + > { + self.inner_pool.ready_at(at) + } + + fn ready(&self) -> Box> + Send> { + self.inner_pool.ready() + } + + fn futures(&self) -> Vec { + self.inner_pool.futures() + } +} diff --git a/substrate/client/rpc-spec-v2/src/transaction/tests/mod.rs b/substrate/client/rpc-spec-v2/src/transaction/tests/mod.rs new file mode 100644 index 000000000000..ab0caaf906fd --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/transaction/tests/mod.rs @@ -0,0 +1,24 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +mod executor; +mod middleware_pool; +#[macro_use] +mod setup; + +mod transaction_broadcast_tests; diff --git a/substrate/client/rpc-spec-v2/src/transaction/tests/setup.rs b/substrate/client/rpc-spec-v2/src/transaction/tests/setup.rs new file mode 100644 index 000000000000..04ee7b9b4c94 --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/transaction/tests/setup.rs @@ -0,0 +1,120 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use crate::{ + chain_head::test_utils::ChainHeadMockClient, + transaction::{ + api::TransactionBroadcastApiServer, + tests::executor::{TaskExecutorBroadcast, TaskExecutorState}, + TransactionBroadcast as RpcTransactionBroadcast, + }, +}; +use futures::Future; +use jsonrpsee::RpcModule; +use sc_transaction_pool::*; +use std::{pin::Pin, sync::Arc}; +use substrate_test_runtime_client::{prelude::*, Client}; +use substrate_test_runtime_transaction_pool::TestApi; + +use crate::transaction::tests::middleware_pool::{MiddlewarePool, MiddlewarePoolRecv}; + +pub type Block = substrate_test_runtime_client::runtime::Block; + +/// Initial Alice account nonce. +pub const ALICE_NONCE: u64 = 209; + +fn create_basic_pool_with_genesis( + test_api: Arc, + options: Options, +) -> (BasicPool, Pin + Send>>) { + let genesis_hash = { + test_api + .chain() + .read() + .block_by_number + .get(&0) + .map(|blocks| blocks[0].0.header.hash()) + .expect("there is block 0. qed") + }; + BasicPool::new_test(test_api, genesis_hash, genesis_hash, options) +} + +fn maintained_pool( + options: Options, +) -> (BasicPool, Arc, futures::executor::ThreadPool) { + let api = Arc::new(TestApi::with_alice_nonce(ALICE_NONCE)); + let (pool, background_task) = create_basic_pool_with_genesis(api.clone(), options); + + let thread_pool = futures::executor::ThreadPool::new().unwrap(); + thread_pool.spawn_ok(background_task); + (pool, api, thread_pool) +} + +pub fn setup_api( + options: Options, +) -> ( + Arc, + Arc, + Arc>>, + RpcModule>>>, + TaskExecutorState, + MiddlewarePoolRecv, +) { + let (pool, api, _) = maintained_pool(options); + let (pool, pool_state) = MiddlewarePool::new(Arc::new(pool).clone()); + let pool = Arc::new(pool); + + let builder = TestClientBuilder::new(); + let client = Arc::new(builder.build()); + let client_mock = Arc::new(ChainHeadMockClient::new(client.clone())); + + let (task_executor, executor_recv) = TaskExecutorBroadcast::new(); + + let tx_api = + RpcTransactionBroadcast::new(client_mock.clone(), pool.clone(), Arc::new(task_executor)) + .into_rpc(); + + (api, pool, client_mock, tx_api, executor_recv, pool_state) +} + +/// Get the next event from the provided middleware in at most 5 seconds. +macro_rules! get_next_event { + ($middleware:expr) => { + tokio::time::timeout(std::time::Duration::from_secs(5), $middleware.recv()) + .await + .unwrap() + .unwrap() + }; +} + +/// Collect the next number of transaction events from the provided middleware. +macro_rules! get_next_tx_events { + ($middleware:expr, $num:expr) => {{ + let mut events = std::collections::HashMap::new(); + for _ in 0..$num { + let event = get_next_event!($middleware); + match event { + crate::transaction::tests::middleware_pool::MiddlewarePoolEvent::TransactionStatus { transaction, status } => { + events.entry(transaction).or_insert_with(|| vec![]).push(status); + }, + other => panic!("Expected TransactionStatus, received {:?}", other), + }; + } + events + }}; +} diff --git a/substrate/client/rpc-spec-v2/src/transaction/tests/transaction_broadcast_tests.rs b/substrate/client/rpc-spec-v2/src/transaction/tests/transaction_broadcast_tests.rs new file mode 100644 index 000000000000..690a1a64d746 --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/transaction/tests/transaction_broadcast_tests.rs @@ -0,0 +1,523 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use crate::{hex_string, transaction::error::json_rpc_spec}; +use assert_matches::assert_matches; +use codec::Encode; +use jsonrpsee::{rpc_params, MethodsError as Error}; +use sc_transaction_pool::{Options, PoolLimit}; +use sc_transaction_pool_api::{ChainEvent, MaintainedTransactionPool, TransactionPool}; +use std::sync::Arc; +use substrate_test_runtime_client::AccountKeyring::*; +use substrate_test_runtime_transaction_pool::uxt; + +// Test helpers. +use crate::transaction::tests::{ + middleware_pool::{MiddlewarePoolEvent, TxStatusTypeTest}, + setup::{setup_api, ALICE_NONCE}, +}; + +#[tokio::test] +async fn tx_broadcast_enters_pool() { + let (api, pool, client_mock, tx_api, mut exec_middleware, mut pool_middleware) = + setup_api(Default::default()); + + // Start at block 1. + let block_1_header = api.push_block(1, vec![], true); + + let uxt = uxt(Alice, ALICE_NONCE); + let xt = hex_string(&uxt.encode()); + + let operation_id: String = + tx_api.call("transaction_unstable_broadcast", rpc_params![&xt]).await.unwrap(); + + // Announce block 1 to `transaction_unstable_broadcast`. + client_mock.trigger_import_stream(block_1_header).await; + + // Ensure the tx propagated from `transaction_unstable_broadcast` to the transaction pool. + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Ready + } + ); + + assert_eq!(1, pool.inner_pool.status().ready); + assert_eq!(uxt.encode().len(), pool.inner_pool.status().ready_bytes); + + // Import block 2 with the transaction included. + let block_2_header = api.push_block(2, vec![uxt.clone()], true); + let block_2 = block_2_header.hash(); + + // Announce block 2 to the pool. + let event = ChainEvent::NewBestBlock { hash: block_2, tree_route: None }; + pool.inner_pool.maintain(event).await; + assert_eq!(0, pool.inner_pool.status().ready); + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::InBlock((block_2, 0)) + } + ); + + // The future broadcast awaits for the finalized status to be reached. + // Force the future to exit by calling stop. + let _: () = tx_api + .call("transaction_unstable_stop", rpc_params![&operation_id]) + .await + .unwrap(); + + // Ensure the broadcast future finishes. + let _ = get_next_event!(&mut exec_middleware.recv); + assert_eq!(0, exec_middleware.num_tasks()); +} + +#[tokio::test] +async fn tx_broadcast_invalid_tx() { + let (_, pool, _, tx_api, mut exec_middleware, _) = setup_api(Default::default()); + + // Invalid parameters. + let err = tx_api + .call::<_, serde_json::Value>("transaction_unstable_broadcast", [1u8]) + .await + .unwrap_err(); + assert_matches!(err, + Error::JsonRpc(err) if err.code() == json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid params" + ); + + assert_eq!(0, pool.status().ready); + + // Invalid transaction that cannot be decoded. The broadcast silently exits. + let xt = "0xdeadbeef"; + let operation_id: String = + tx_api.call("transaction_unstable_broadcast", rpc_params![&xt]).await.unwrap(); + + assert_eq!(0, pool.status().ready); + + // Await the broadcast future to exit. + // Without this we'd be subject to races, where we try to call the stop before the tx is + // dropped. + let _ = get_next_event!(&mut exec_middleware.recv); + assert_eq!(0, exec_middleware.num_tasks()); + + // The broadcast future was dropped, and the operation is no longer active. + // When the operation is not active, either from the tx being finalized or a + // terminal error; the stop method should return an error. + let err = tx_api + .call::<_, serde_json::Value>("transaction_unstable_stop", rpc_params![&operation_id]) + .await + .unwrap_err(); + assert_matches!(err, + Error::JsonRpc(err) if err.code() == json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid operation id" + ); +} + +#[tokio::test] +async fn tx_stop_with_invalid_operation_id() { + let (_, _, _, tx_api, _, _) = setup_api(Default::default()); + + // Make an invalid stop call. + let err = tx_api + .call::<_, serde_json::Value>("transaction_unstable_stop", ["invalid_operation_id"]) + .await + .unwrap_err(); + assert_matches!(err, + Error::JsonRpc(err) if err.code() == json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid operation id" + ); +} + +#[tokio::test] +async fn tx_broadcast_resubmits_future_nonce_tx() { + let (api, pool, client_mock, tx_api, mut exec_middleware, mut pool_middleware) = + setup_api(Default::default()); + + // Start at block 1. + let block_1_header = api.push_block(1, vec![], true); + let block_1 = block_1_header.hash(); + + let current_uxt = uxt(Alice, ALICE_NONCE); + let current_xt = hex_string(¤t_uxt.encode()); + // This lives in the future. + let future_uxt = uxt(Alice, ALICE_NONCE + 1); + let future_xt = hex_string(&future_uxt.encode()); + + let future_operation_id: String = tx_api + .call("transaction_unstable_broadcast", rpc_params![&future_xt]) + .await + .unwrap(); + + // Announce block 1 to `transaction_unstable_broadcast`. + client_mock.trigger_import_stream(block_1_header).await; + + // Ensure the tx propagated from `transaction_unstable_broadcast` to the transaction pool. + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: future_xt.clone(), + status: TxStatusTypeTest::Future + } + ); + + let event = ChainEvent::NewBestBlock { hash: block_1, tree_route: None }; + pool.inner_pool.maintain(event).await; + assert_eq!(0, pool.inner_pool.status().ready); + // Ensure the tx is in the future. + assert_eq!(1, pool.inner_pool.status().future); + + let block_2_header = api.push_block(2, vec![], true); + let block_2 = block_2_header.hash(); + + let operation_id: String = tx_api + .call("transaction_unstable_broadcast", rpc_params![¤t_xt]) + .await + .unwrap(); + assert_ne!(future_operation_id, operation_id); + + // Announce block 2 to `transaction_unstable_broadcast`. + client_mock.trigger_import_stream(block_2_header).await; + + // Collect the events of both transactions. + let events = get_next_tx_events!(&mut pool_middleware, 2); + // Transactions entered the ready queue. + assert_eq!(events.get(¤t_xt).unwrap(), &vec![TxStatusTypeTest::Ready]); + assert_eq!(events.get(&future_xt).unwrap(), &vec![TxStatusTypeTest::Ready]); + + let event = ChainEvent::NewBestBlock { hash: block_2, tree_route: None }; + pool.inner_pool.maintain(event).await; + assert_eq!(2, pool.inner_pool.status().ready); + assert_eq!(0, pool.inner_pool.status().future); + + // Finalize transactions. + let block_3_header = api.push_block(3, vec![current_uxt, future_uxt], true); + let block_3 = block_3_header.hash(); + client_mock.trigger_import_stream(block_3_header).await; + + let event = ChainEvent::Finalized { hash: block_3, tree_route: Arc::from(vec![]) }; + pool.inner_pool.maintain(event).await; + assert_eq!(0, pool.inner_pool.status().ready); + assert_eq!(0, pool.inner_pool.status().future); + + let events = get_next_tx_events!(&mut pool_middleware, 4); + assert_eq!( + events.get(¤t_xt).unwrap(), + &vec![TxStatusTypeTest::InBlock((block_3, 0)), TxStatusTypeTest::Finalized((block_3, 0))] + ); + assert_eq!( + events.get(&future_xt).unwrap(), + &vec![TxStatusTypeTest::InBlock((block_3, 1)), TxStatusTypeTest::Finalized((block_3, 1))] + ); + + // Both broadcast futures must exit. + let _ = get_next_event!(&mut exec_middleware.recv); + let _ = get_next_event!(&mut exec_middleware.recv); + assert_eq!(0, exec_middleware.num_tasks()); +} + +/// This test is similar to `tx_broadcast_enters_pool` +/// However the last block is announced as finalized to force the +/// broadcast future to exit before the `stop` is called. +#[tokio::test] +async fn tx_broadcast_stop_after_broadcast_finishes() { + let (api, pool, client_mock, tx_api, mut exec_middleware, mut pool_middleware) = + setup_api(Default::default()); + + // Start at block 1. + let block_1_header = api.push_block(1, vec![], true); + + let uxt = uxt(Alice, ALICE_NONCE); + let xt = hex_string(&uxt.encode()); + + let operation_id: String = + tx_api.call("transaction_unstable_broadcast", rpc_params![&xt]).await.unwrap(); + + // Announce block 1 to `transaction_unstable_broadcast`. + client_mock.trigger_import_stream(block_1_header).await; + + // Ensure the tx propagated from `transaction_unstable_broadcast` to the transaction + // pool.inner_pool. + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Ready + } + ); + + assert_eq!(1, pool.inner_pool.status().ready); + assert_eq!(uxt.encode().len(), pool.inner_pool.status().ready_bytes); + + // Import block 2 with the transaction included. + let block_2_header = api.push_block(2, vec![uxt.clone()], true); + let block_2 = block_2_header.hash(); + + // Announce block 2 to the pool.inner_pool. + let event = ChainEvent::Finalized { hash: block_2, tree_route: Arc::from(vec![]) }; + pool.inner_pool.maintain(event).await; + + assert_eq!(0, pool.inner_pool.status().ready); + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::InBlock((block_2, 0)) + } + ); + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Finalized((block_2, 0)) + } + ); + + // Ensure the broadcast future terminated properly. + let _ = get_next_event!(&mut exec_middleware.recv); + assert_eq!(0, exec_middleware.num_tasks()); + + // The operation ID is no longer valid, check that the broadcast future + // cleared out the inner state of the operation. + let err = tx_api + .call::<_, serde_json::Value>("transaction_unstable_stop", rpc_params![&operation_id]) + .await + .unwrap_err(); + assert_matches!(err, + Error::JsonRpc(err) if err.code() == json_rpc_spec::INVALID_PARAM_ERROR && err.message() == "Invalid operation id" + ); +} + +#[tokio::test] +async fn tx_broadcast_resubmits_invalid_tx() { + let limits = PoolLimit { count: 8192, total_bytes: 20 * 1024 * 1024 }; + let options = Options { + ready: limits.clone(), + future: limits, + reject_future_transactions: false, + // This ensures that a transaction is not banned. + ban_time: std::time::Duration::ZERO, + }; + + let (api, pool, client_mock, tx_api, mut exec_middleware, mut pool_middleware) = + setup_api(options); + + let uxt = uxt(Alice, ALICE_NONCE); + let xt = hex_string(&uxt.encode()); + let _operation_id: String = + tx_api.call("transaction_unstable_broadcast", rpc_params![&xt]).await.unwrap(); + + let block_1_header = api.push_block(1, vec![], true); + let block_1 = block_1_header.hash(); + // Announce block 1 to `transaction_unstable_broadcast`. + client_mock.trigger_import_stream(block_1_header).await; + + // Ensure the tx propagated from `transaction_unstable_broadcast` to the transaction pool. + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Ready, + } + ); + assert_eq!(1, pool.inner_pool.status().ready); + assert_eq!(uxt.encode().len(), pool.inner_pool.status().ready_bytes); + + // Mark the transaction as invalid from the API, causing a temporary ban. + api.add_invalid(&uxt); + + // Push an event to the pool to ensure the transaction is excluded. + let event = ChainEvent::NewBestBlock { hash: block_1, tree_route: None }; + pool.inner_pool.maintain(event).await; + assert_eq!(1, pool.inner_pool.status().ready); + + // Ensure the `transaction_unstable_broadcast` is aware of the invalid transaction. + let event = get_next_event!(&mut pool_middleware); + // Because we have received an `Invalid` status, we try to broadcast the transaction with the + // next announced block. + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Invalid + } + ); + + // Import block 2. + let block_2_header = api.push_block(2, vec![], true); + client_mock.trigger_import_stream(block_2_header).await; + + // Ensure we propagate the temporary ban error to `submit_and_watch`. + // This ensures we'll loop again with the next annmounced block and try to resubmit the + // transaction. The transaction remains temporarily banned until the pool is maintained. + let event = get_next_event!(&mut pool_middleware); + assert_matches!(event, MiddlewarePoolEvent::PoolError { transaction, err } if transaction == xt && err.contains("Transaction temporarily Banned")); + + // Import block 3. + let block_3_header = api.push_block(3, vec![], true); + let block_3 = block_3_header.hash(); + // Remove the invalid transaction from the pool to allow it to pass through. + api.remove_invalid(&uxt); + let event = ChainEvent::NewBestBlock { hash: block_3, tree_route: None }; + // We have to maintain the pool to ensure the transaction is no longer invalid. + // This clears out the banned transactions. + pool.inner_pool.maintain(event).await; + assert_eq!(0, pool.inner_pool.status().ready); + + // Announce block to `transaction_unstable_broadcast`. + client_mock.trigger_import_stream(block_3_header).await; + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Ready, + } + ); + assert_eq!(1, pool.inner_pool.status().ready); + + let block_4_header = api.push_block(4, vec![uxt], true); + let block_4 = block_4_header.hash(); + let event = ChainEvent::Finalized { hash: block_4, tree_route: Arc::from(vec![]) }; + pool.inner_pool.maintain(event).await; + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::InBlock((block_4, 0)), + } + ); + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: xt.clone(), + status: TxStatusTypeTest::Finalized((block_4, 0)), + } + ); + + // Ensure the broadcast future terminated properly. + let _ = get_next_event!(&mut exec_middleware.recv); + assert_eq!(0, exec_middleware.num_tasks()); +} + +/// This is similar to `tx_broadcast_resubmits_invalid_tx`. +/// However, it forces the tx to be resubmited because of the pool +/// limits. Which is a different code path than the invalid tx. +#[tokio::test] +async fn tx_broadcast_resubmits_dropped_tx() { + let limits = PoolLimit { count: 1, total_bytes: 1000 }; + let options = Options { + ready: limits.clone(), + future: limits, + reject_future_transactions: false, + // This ensures that a transaction is not banned. + ban_time: std::time::Duration::ZERO, + }; + + let (api, pool, client_mock, tx_api, _, mut pool_middleware) = setup_api(options); + + let current_uxt = uxt(Alice, ALICE_NONCE); + let current_xt = hex_string(¤t_uxt.encode()); + // This lives in the future. + let future_uxt = uxt(Alice, ALICE_NONCE + 1); + let future_xt = hex_string(&future_uxt.encode()); + + // By default the `validate_transaction` mock uses priority 1 for + // transactions. Bump the priority to ensure other transactions + // are immediately dropped. + api.set_priority(¤t_uxt, 10); + + let current_operation_id: String = tx_api + .call("transaction_unstable_broadcast", rpc_params![¤t_xt]) + .await + .unwrap(); + + // Announce block 1 to `transaction_unstable_broadcast`. + let block_1_header = api.push_block(1, vec![], true); + let event = + ChainEvent::Finalized { hash: block_1_header.hash(), tree_route: Arc::from(vec![]) }; + pool.inner_pool.maintain(event).await; + client_mock.trigger_import_stream(block_1_header).await; + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::TransactionStatus { + transaction: current_xt.clone(), + status: TxStatusTypeTest::Ready, + } + ); + assert_eq!(1, pool.inner_pool.status().ready); + + // The future tx has priority 2, smaller than the current 10. + api.set_priority(&future_uxt, 2); + let future_operation_id: String = tx_api + .call("transaction_unstable_broadcast", rpc_params![&future_xt]) + .await + .unwrap(); + assert_ne!(current_operation_id, future_operation_id); + + let block_2_header = api.push_block(2, vec![], true); + let event = + ChainEvent::Finalized { hash: block_2_header.hash(), tree_route: Arc::from(vec![]) }; + pool.inner_pool.maintain(event).await; + client_mock.trigger_import_stream(block_2_header).await; + + // We must have at most 1 transaction in the pool, as per limits above. + assert_eq!(1, pool.inner_pool.status().ready); + + let event = get_next_event!(&mut pool_middleware); + assert_eq!( + event, + MiddlewarePoolEvent::PoolError { + transaction: future_xt.clone(), + err: "Transaction couldn't enter the pool because of the limit".into() + } + ); + + let block_3_header = api.push_block(3, vec![current_uxt], true); + let event = + ChainEvent::Finalized { hash: block_3_header.hash(), tree_route: Arc::from(vec![]) }; + pool.inner_pool.maintain(event).await; + client_mock.trigger_import_stream(block_3_header.clone()).await; + + // The first tx is in a finalzied block; the future tx must enter the pool. + let events = get_next_tx_events!(&mut pool_middleware, 3); + assert_eq!( + events.get(¤t_xt).unwrap(), + &vec![ + TxStatusTypeTest::InBlock((block_3_header.hash(), 0)), + TxStatusTypeTest::Finalized((block_3_header.hash(), 0)) + ] + ); + // The dropped transaction was resubmitted. + assert_eq!(events.get(&future_xt).unwrap(), &vec![TxStatusTypeTest::Ready]); +} diff --git a/substrate/client/transaction-pool/src/lib.rs b/substrate/client/transaction-pool/src/lib.rs index faa3f455a580..64b301e6bf36 100644 --- a/substrate/client/transaction-pool/src/lib.rs +++ b/substrate/client/transaction-pool/src/lib.rs @@ -164,8 +164,9 @@ where pool_api: Arc, best_block_hash: Block::Hash, finalized_hash: Block::Hash, + options: graph::Options, ) -> (Self, Pin + Send>>) { - let pool = Arc::new(graph::Pool::new(Default::default(), true.into(), pool_api.clone())); + let pool = Arc::new(graph::Pool::new(options, true.into(), pool_api.clone())); let (revalidation_queue, background_task) = revalidation::RevalidationQueue::new_background( pool_api.clone(), pool.clone(), diff --git a/substrate/client/transaction-pool/tests/pool.rs b/substrate/client/transaction-pool/tests/pool.rs index 6b1a197440c1..461b9860d414 100644 --- a/substrate/client/transaction-pool/tests/pool.rs +++ b/substrate/client/transaction-pool/tests/pool.rs @@ -73,7 +73,7 @@ fn create_basic_pool_with_genesis( .map(|blocks| blocks[0].0.header.hash()) .expect("there is block 0. qed") }; - BasicPool::new_test(test_api, genesis_hash, genesis_hash) + BasicPool::new_test(test_api, genesis_hash, genesis_hash, Default::default()) } fn create_basic_pool(test_api: TestApi) -> BasicPool { @@ -994,6 +994,7 @@ fn import_notification_to_pool_maintain_works() { )), best_hash, finalized_hash, + Default::default(), ) .0, ); diff --git a/substrate/test-utils/runtime/transaction-pool/src/lib.rs b/substrate/test-utils/runtime/transaction-pool/src/lib.rs index 8c8345b06bd3..5202e6e65154 100644 --- a/substrate/test-utils/runtime/transaction-pool/src/lib.rs +++ b/substrate/test-utils/runtime/transaction-pool/src/lib.rs @@ -81,6 +81,7 @@ pub struct ChainState { pub block_by_hash: HashMap, pub nonces: HashMap, pub invalid_hashes: HashSet, + pub priorities: HashMap, } /// Test Api for transaction pool. @@ -214,6 +215,22 @@ impl TestApi { self.chain.write().invalid_hashes.insert(Self::hash_and_length_inner(xts).0); } + /// Remove a transaction that was previously declared as invalid via `[Self::add_invalid]`. + /// + /// Next time transaction pool will try to validate this + /// extrinsic, api will succeed. + pub fn remove_invalid(&self, xts: &Extrinsic) { + self.chain.write().invalid_hashes.remove(&Self::hash_and_length_inner(xts).0); + } + + /// Set a transaction priority. + pub fn set_priority(&self, xts: &Extrinsic, priority: u64) { + self.chain + .write() + .priorities + .insert(Self::hash_and_length_inner(xts).0, priority); + } + /// Query validation requests received. pub fn validation_requests(&self) -> Vec { self.validation_requests.read().clone() @@ -300,8 +317,14 @@ impl ChainApi for TestApi { return ready(Ok(Err(TransactionValidityError::Invalid(InvalidTransaction::Custom(0))))) } - let mut validity = - ValidTransaction { priority: 1, requires, provides, longevity: 64, propagate: true }; + let priority = self.chain.read().priorities.get(&self.hash_and_length(&uxt).0).cloned(); + let mut validity = ValidTransaction { + priority: priority.unwrap_or(1), + requires, + provides, + longevity: 64, + propagate: true, + }; (self.valid_modifier.read())(&mut validity); From f4eedcebafa4d2addb8b71ba31ab1bc579ba3c4a Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Wed, 28 Feb 2024 13:13:27 +0100 Subject: [PATCH 02/13] [prdoc] Optional SemVer bumps and Docs (#3441) Changes: - Add an optional `bump` field to the crates in a prdoc. - Explain the cargo semver interpretation for <1 versions in the release doc. --------- Signed-off-by: Oliver Tale-Yazdi --- docs/RELEASE.md | 12 +++++++++--- prdoc/schema_user.json | 33 +++++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 6d681d78f367..e73be2779a99 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -18,10 +18,16 @@ Rococo. To easily refer to a release, it shall be named by its date in the form ## Crate -We try to follow [SemVer 2.0.0](https://semver.org/) as best as possible for versioning our crates. SemVer requires a -piece of software to first declare a public API. The public API of the Polkadot SDK is hereby declared as the sum of all -crates' public APIs. +We try to follow [SemVer 2.0.0](https://semver.org/) as best as possible for versioning our crates. The definitions of +`major`, `minor` and `patch` version for Rust crates are slightly altered from their standard for pre `1.0.0` versions. +Quoting [rust-lang.org](https://doc.rust-lang.org/cargo/reference/semver.html): +>Initial development releases starting with “0.y.z” can treat changes in “y” as a major release, and “z” as a minor +release. “0.0.z” releases are always major changes. This is because Cargo uses the convention that only changes in the +left-most non-zero component are considered incompatible. + +SemVer requires a piece of software to first declare a public API. The public API of the Polkadot SDK +is hereby declared as the sum of all crates' public APIs. Inductively, the public API of our library crates is declared as all public items that are neither: - Inside a `__private` module diff --git a/prdoc/schema_user.json b/prdoc/schema_user.json index 82215d51866b..1bd0b3b93ee4 100644 --- a/prdoc/schema_user.json +++ b/prdoc/schema_user.json @@ -3,9 +3,8 @@ "$id": "https://raw.githubusercontent.com/paritytech/prdoc/master/prdoc_schema_user.json", "version": { "major": 1, - "minor": 0, - "patch": 0, - "timestamp": 20230817152351 + "minor": 1, + "patch": 0 }, "title": "Polkadot SDK PRDoc Schema", "description": "JSON Schema definition for the Polkadot SDK PR documentation", @@ -125,10 +124,16 @@ "name": { "type": "string" }, + "bump": { + "$ref": "#/$defs/semver_bump" + }, "note": { "type": "string" } - } + }, + "required": [ + "name" + ] }, "migration_db": { "type": "object", @@ -165,6 +170,26 @@ "description" ] }, + "semver_bump": { + "description": "The type of bump to apply to the crate version according to Cargo SemVer: https://doc.rust-lang.org/cargo/reference/semver.html. Please check docs/RELEASE.md for more information.", + "oneOf": [ + { + "const": "major", + "title": "Major", + "description": "A bump to the leftmost non-zero digit of the version number." + }, + { + "const": "minor", + "title": "Minor", + "description": "A bump to the second leftmost non-zero digit of the version number." + }, + { + "const": "patch", + "title": "Patch", + "description": "A bump to the third leftmost non-zero digit of the version number." + } + ] + }, "doc": { "type": "object", "description": "You have the the option to provide different description of your PR for different audiences.", From 14530269b7a76f50dc939bf57b1e3d38270b662b Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Wed, 28 Feb 2024 13:07:14 +0000 Subject: [PATCH 03/13] Add documentation around FRAME Offchain workers (#3463) - deprecation companion: https://github.com/substrate-developer-hub/substrate-docs/pull/2136 - inspired by https://substrate.stackexchange.com/questions/11058/how-can-i-create-ocw-that-wont-activates-every-block-but-will-activates-only-w/11060#11060 --------- Co-authored-by: Sergej Sakac <73715684+Szegoo@users.noreply.github.com> --- Cargo.lock | 2 + docs/sdk/Cargo.toml | 5 +- .../reference_docs/frame_offchain_workers.rs | 115 ++++++++++++++++++ docs/sdk/src/reference_docs/mod.rs | 4 + substrate/frame/support/src/traits/hooks.rs | 2 +- 5 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 docs/sdk/src/reference_docs/frame_offchain_workers.rs diff --git a/Cargo.lock b/Cargo.lock index 026786378d5a..ef510986257a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13416,6 +13416,7 @@ dependencies = [ "pallet-collective", "pallet-default-config-example", "pallet-democracy", + "pallet-example-offchain-worker", "pallet-example-single-block-migrations", "pallet-examples", "pallet-multisig", @@ -13442,6 +13443,7 @@ dependencies = [ "sp-core", "sp-io", "sp-keyring", + "sp-offchain", "sp-runtime", "sp-version", "staging-chain-spec-builder", diff --git a/docs/sdk/Cargo.toml b/docs/sdk/Cargo.toml index b1b60a2d77db..735b3d7df61d 100644 --- a/docs/sdk/Cargo.toml +++ b/docs/sdk/Cargo.toml @@ -23,6 +23,7 @@ frame = { path = "../../substrate/frame", features = [ ] } pallet-examples = { path = "../../substrate/frame/examples" } pallet-default-config-example = { path = "../../substrate/frame/examples/default-config" } +pallet-example-offchain-worker = { path = "../../substrate/frame/examples/offchain-worker" } # How we build docs in rust-docs simple-mermaid = "0.1.1" @@ -38,7 +39,7 @@ frame-support = { path = "../../substrate/frame/support", default-features = fal frame-executive = { path = "../../substrate/frame/executive", default-features = false } pallet-example-single-block-migrations = { path = "../../substrate/frame/examples/single-block-migrations" } -# Substrate +# Substrate Client sc-network = { path = "../../substrate/client/network" } sc-rpc-api = { path = "../../substrate/client/rpc-api" } sc-rpc = { path = "../../substrate/client/rpc" } @@ -50,6 +51,7 @@ sc-consensus-grandpa = { path = "../../substrate/client/consensus/grandpa" } sc-consensus-beefy = { path = "../../substrate/client/consensus/beefy" } sc-consensus-manual-seal = { path = "../../substrate/client/consensus/manual-seal" } sc-consensus-pow = { path = "../../substrate/client/consensus/pow" } + substrate-wasm-builder = { path = "../../substrate/utils/wasm-builder" } # Cumulus @@ -78,6 +80,7 @@ sp-api = { path = "../../substrate/primitives/api" } sp-core = { path = "../../substrate/primitives/core" } sp-keyring = { path = "../../substrate/primitives/keyring" } sp-runtime = { path = "../../substrate/primitives/runtime" } +sp-offchain = { path = "../../substrate/primitives/offchain" } sp-version = { path = "../../substrate/primitives/version" } # XCM diff --git a/docs/sdk/src/reference_docs/frame_offchain_workers.rs b/docs/sdk/src/reference_docs/frame_offchain_workers.rs new file mode 100644 index 000000000000..7999707e5ee0 --- /dev/null +++ b/docs/sdk/src/reference_docs/frame_offchain_workers.rs @@ -0,0 +1,115 @@ +//! # Offchain Workers +//! +//! This reference document explains how offchain workers work in Substrate and FRAME. The main +//! focus is upon FRAME's implementation of this functionality. Nonetheless, offchain workers are a +//! Substrate-provided feature and can be used with possible alternatives to [`frame`] as well. +//! +//! Offchain workers are a commonly misunderstood topic, therefore we explain them bottom-up, +//! starting at the fundamentals and then describing the developer interface. +//! +//! ## Context +//! +//! Recall from [`crate::reference_docs::wasm_meta_protocol`] that the node and the runtime +//! communicate with one another via host functions and runtime APIs. Many of these interactions +//! contribute to the actual state transition of the blockchain. For example [`sp_api::Core`] is the +//! main runtime API that is called to execute new blocks. +//! +//! Offchain workers are in principle not different in any way: It is a runtime API exposed by the +//! wasm blob ([`sp_offchain::OffchainWorkerApi`]), and the node software calls into it when it +//! deems fit. But, crucially, this API call is different in that: +//! +//! 1. It can have no impact on the state ie. it is _OFF (the) CHAIN_. If any state is altered +//! during the execution of this API call, it is discarded. +//! 2. It has access to an extended set of host functions that allow the wasm blob to do more. For +//! example, call into HTTP requests. +//! +//! > The main way through which an offchain worker can interact with the state is by submitting an +//! > extrinsic to the chain. This is the ONLY way to alter the state from an offchain worker. +//! > [`pallet_example_offchain_worker`] provides an example of this. +//! +//! +//! Given the "Off Chain" nature of this API, it is important to remember that calling this API is +//! entirely optional. Some nodes might call into it, some might not, and it would have no impact on +//! the execution of your blockchain because no state is altered no matter the execution of the +//! offchain worker API. +//! +//! Substrate's CLI allows some degree of configuration about this, allowing node operators to +//! specify when they want to run the offchain worker API. See +//! [`sc_cli::RunCmd::offchain_worker_params`]. +//! +//! ## Nondeterministic Execution +//! +//! Needless to say, given the above description, the code in your offchain worker API can be +//! nondeterministic, as it is not part of the blockchain's STF, so it can be executed at unknown +//! times, by unknown nodes, and has no impact on the state. This is why an HTTP +//! ([`sp_runtime::offchain::http`]) API is readily provided to the offchain worker APIs. Because +//! there is no need for determinism in this context. +//! +//! > A common mistake here is for novice developers to see this HTTP API, and imagine that +//! > `polkadot-sdk` somehow magically solved the determinism in blockchains, and now a blockchain +//! > can make HTTP calls and it will all work. This is absolutely NOT the case. An HTTP call made +//! > by the offchain worker is non-deterministic by design. Blockchains can't and always won't be +//! > able to perform non-deterministic operations such as making HTTP calls to a foreign server. +//! +//! ## FRAME's API +//! +//! [`frame`] provides a simple API through which pallets can define offchain worker functions. This +//! is part of [`frame::traits::Hooks`], which is implemented as a part of +//! [`frame::pallet_macros::hooks`]. +//! +//! ``` +//! +//! #[frame::pallet] +//! pub mod pallet { +//! use frame::prelude::*; +//! +//! #[pallet::config] +//! pub trait Config: frame_system::Config {} +//! +//! #[pallet::pallet] +//! pub struct Pallet(_); +//! +//! #[pallet::hooks] +//! impl Hooks> for Pallet { +//! fn offchain_worker(block_number: BlockNumberFor) { +//! // ... +//! } +//! } +//! } +//! ``` +//! +//! Additionally, [`sp_runtime::offchain`] provides a set of utilities that can be used to moderate +//! the execution of offchain workers. +//! +//! ## Think Twice: Why Use Substrate's Offchain Workers? +//! +//! Consider the fact that in principle, an offchain worker code written using the above API is no +//! different than an equivalent written with an _actual offchain interaction library_, such as +//! [Polkadot-JS](https://polkadot.js.org/docs/), or any of the other ones listed [here](https://github.com/substrate-developer-hub/awesome-substrate?tab=readme-ov-file#client-libraries). +//! +//! They can both read from the state, and have no means of updating the state, other than the route +//! of submitting an extrinsic to the chain. Therefore, it is worth thinking twice before embedding +//! a logic as a part of Substrate's offchain worker API. Does it have to be there? can it not be a +//! simple, actual offchain application that lives outside of the chain's WASM blob? +//! +//! Some of the reasons why you might want to do the opposite, and actually embed an offchain worker +//! API into the WASM blob are: +//! +//! * Accessing the state is easier within the `offchain_worker` function, as it is already a part +//! of the runtime, and [`frame::pallet_macros::storage`] provides all the tools needed to read +//! the state. Other client libraries might provide varying degrees of capability here. +//! * It will be updated in synchrony with the runtime. A Substrate's offchain application is part +//! of the same WASM blob, and is therefore guaranteed to be up to date. +//! +//! For example, imagine you have modified a storage item to have a new type. This will possibly +//! require a [`crate::reference_docs::frame_runtime_upgrades_and_migrations`], and any offchain +//! code, such as a Polkadot-JS application, will have to be updated to reflect this change. Whereas +//! the WASM offchain worker code is guaranteed to already be updated, or else the runtime code will +//! not even compile. +//! +//! +//! ## Further References +//! +//! - +//! - +//! - [Offchain worker example](https://github.com/paritytech/polkadot-sdk/tree/master/substrate/frame/examples/offchain-worker) diff --git a/docs/sdk/src/reference_docs/mod.rs b/docs/sdk/src/reference_docs/mod.rs index b2c751420ce7..0bd204e4b6ab 100644 --- a/docs/sdk/src/reference_docs/mod.rs +++ b/docs/sdk/src/reference_docs/mod.rs @@ -99,3 +99,7 @@ pub mod frame_runtime_upgrades_and_migrations; /// light-node-first out of the box. // TODO: @jsdw @josepot https://github.com/paritytech/polkadot-sdk-docs/issues/68 pub mod light_nodes; + +/// Learn about the offchain workers, how they function, and how to use them, as provided by the +/// [`frame`] APIs. +pub mod frame_offchain_workers; diff --git a/substrate/frame/support/src/traits/hooks.rs b/substrate/frame/support/src/traits/hooks.rs index 72f047c3a73d..c37fb0f54bcd 100644 --- a/substrate/frame/support/src/traits/hooks.rs +++ b/substrate/frame/support/src/traits/hooks.rs @@ -446,7 +446,7 @@ pub trait Hooks { } /// Implementing this function on a pallet allows you to perform long-running tasks that are - /// dispatched as separate threads, and entirely independent of the main wasm runtime. + /// dispatched as separate threads, and entirely independent of the main blockchain execution. /// /// This function can freely read from the state, but any change it makes to the state is /// meaningless. Writes can be pushed back to the chain by submitting extrinsics from the From 426136671a42ff759127b4a7a71eac6335744675 Mon Sep 17 00:00:00 2001 From: maksimryndin Date: Wed, 28 Feb 2024 17:29:27 +0100 Subject: [PATCH 04/13] PVF: re-preparing artifact on failed runtime construction (#3187) resolve https://github.com/paritytech/polkadot-sdk/issues/3139 - [x] use a distinguishable error for `execute_artifact` - [x] remove artifact in case of a `RuntimeConstruction` error during the execution - [x] augment the `validate_candidate_with_retry` of `ValidationBackend` with the case of retriable `RuntimeConstruction` error during the execution - [x] update the book (https://paritytech.github.io/polkadot-sdk/book/node/utility/pvf-host-and-workers.html#retrying-execution-requests) - [x] add a test - [x] run zombienet tests --------- Co-authored-by: s0me0ne-unkn0wn <48632512+s0me0ne-unkn0wn@users.noreply.github.com> --- .../node/core/candidate-validation/src/lib.rs | 53 +++++++----- polkadot/node/core/pvf/common/src/error.rs | 1 + polkadot/node/core/pvf/common/src/execute.rs | 15 ++++ .../core/pvf/common/src/executor_interface.rs | 4 +- .../node/core/pvf/execute-worker/src/lib.rs | 10 ++- polkadot/node/core/pvf/src/artifacts.rs | 8 ++ polkadot/node/core/pvf/src/error.rs | 4 + polkadot/node/core/pvf/src/execute/mod.rs | 2 +- polkadot/node/core/pvf/src/execute/queue.rs | 68 ++++++++++++--- .../core/pvf/src/execute/worker_interface.rs | 8 ++ polkadot/node/core/pvf/src/host.rs | 53 +++++++++++- polkadot/node/core/pvf/tests/it/main.rs | 83 +++++++++++++++++-- polkadot/primitives/src/v6/executor_params.rs | 10 +-- .../src/node/utility/pvf-host-and-workers.md | 8 ++ prdoc/pr_3187.prdoc | 16 ++++ 15 files changed, 294 insertions(+), 49 deletions(-) create mode 100644 prdoc/pr_3187.prdoc diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index bf6e09fd1b69..8237137fdca0 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -695,6 +695,8 @@ async fn validate_candidate_exhaustive( ))), Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(err))) => Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(err))), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::RuntimeConstruction(err))) => + Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(err))), Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousJobDeath(err))) => Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(format!( @@ -780,40 +782,50 @@ trait ValidationBackend { return validation_result } + macro_rules! break_if_no_retries_left { + ($counter:ident) => { + if $counter > 0 { + $counter -= 1; + } else { + break + } + }; + } + // Allow limited retries for each kind of error. let mut num_death_retries_left = 1; let mut num_job_error_retries_left = 1; let mut num_internal_retries_left = 1; + let mut num_runtime_construction_retries_left = 1; loop { // Stop retrying if we exceeded the timeout. if total_time_start.elapsed() + retry_delay > exec_timeout { break } - + let mut retry_immediately = false; match validation_result { Err(ValidationError::PossiblyInvalid( PossiblyInvalidError::AmbiguousWorkerDeath | PossiblyInvalidError::AmbiguousJobDeath(_), - )) => - if num_death_retries_left > 0 { - num_death_retries_left -= 1; - } else { - break - }, + )) => break_if_no_retries_left!(num_death_retries_left), Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(_))) => - if num_job_error_retries_left > 0 { - num_job_error_retries_left -= 1; - } else { - break - }, + break_if_no_retries_left!(num_job_error_retries_left), Err(ValidationError::Internal(_)) => - if num_internal_retries_left > 0 { - num_internal_retries_left -= 1; - } else { - break - }, + break_if_no_retries_left!(num_internal_retries_left), + + Err(ValidationError::PossiblyInvalid( + PossiblyInvalidError::RuntimeConstruction(_), + )) => { + break_if_no_retries_left!(num_runtime_construction_retries_left); + self.precheck_pvf(pvf.clone()).await?; + // In this case the error is deterministic + // And a retry forces the ValidationBackend + // to re-prepare the artifact so + // there is no need to wait before the retry + retry_immediately = true; + }, Ok(_) | Err(ValidationError::Invalid(_) | ValidationError::Preparation(_)) => break, } @@ -821,8 +833,11 @@ trait ValidationBackend { // If we got a possibly transient error, retry once after a brief delay, on the // assumption that the conditions that caused this error may have resolved on their own. { - // Wait a brief delay before retrying. - futures_timer::Delay::new(retry_delay).await; + // In case of many transient errors it is necessary to wait a little bit + // for the error to be probably resolved + if !retry_immediately { + futures_timer::Delay::new(retry_delay).await; + } let new_timeout = exec_timeout.saturating_sub(total_time_start.elapsed()); diff --git a/polkadot/node/core/pvf/common/src/error.rs b/polkadot/node/core/pvf/common/src/error.rs index f8faefc24e65..cf274044456f 100644 --- a/polkadot/node/core/pvf/common/src/error.rs +++ b/polkadot/node/core/pvf/common/src/error.rs @@ -16,6 +16,7 @@ use crate::prepare::{PrepareSuccess, PrepareWorkerSuccess}; use parity_scale_codec::{Decode, Encode}; +pub use sc_executor_common::error::Error as ExecuteError; /// Result of PVF preparation from a worker, with checksum of the compiled PVF and stats of the /// preparation if successful. diff --git a/polkadot/node/core/pvf/common/src/execute.rs b/polkadot/node/core/pvf/common/src/execute.rs index 6b3becf524d7..18c97b03cbcd 100644 --- a/polkadot/node/core/pvf/common/src/execute.rs +++ b/polkadot/node/core/pvf/common/src/execute.rs @@ -40,6 +40,9 @@ pub enum WorkerResponse { }, /// The candidate is invalid. InvalidCandidate(String), + /// Instantiation of the WASM module instance failed during an execution. + /// Possibly related to local issues or dirty node update. May be retried with re-preparation. + RuntimeConstruction(String), /// The job timed out. JobTimedOut, /// The job process has died. We must kill the worker just in case. @@ -68,6 +71,9 @@ pub enum JobResponse { /// The result of parachain validation. result_descriptor: ValidationResult, }, + /// A possibly transient runtime instantiation error happened during the execution; may be + /// retried with re-preparation + RuntimeConstruction(String), /// The candidate is invalid. InvalidCandidate(String), } @@ -81,6 +87,15 @@ impl JobResponse { Self::InvalidCandidate(format!("{}: {}", ctx, msg)) } } + + /// Creates a may retry response from a context `ctx` and a message `msg` (which can be empty). + pub fn runtime_construction(ctx: &'static str, msg: &str) -> Self { + if msg.is_empty() { + Self::RuntimeConstruction(ctx.to_string()) + } else { + Self::RuntimeConstruction(format!("{}: {}", ctx, msg)) + } + } } /// An unexpected error occurred in the execution job process. Because this comes from the job, diff --git a/polkadot/node/core/pvf/common/src/executor_interface.rs b/polkadot/node/core/pvf/common/src/executor_interface.rs index 4cd2f5c85eec..b69a87890f60 100644 --- a/polkadot/node/core/pvf/common/src/executor_interface.rs +++ b/polkadot/node/core/pvf/common/src/executor_interface.rs @@ -16,6 +16,7 @@ //! Interface to the Substrate Executor +use crate::error::ExecuteError; use polkadot_primitives::{ executor_params::{DEFAULT_LOGICAL_STACK_MAX, DEFAULT_NATIVE_STACK_MAX}, ExecutorParam, ExecutorParams, @@ -109,7 +110,7 @@ pub unsafe fn execute_artifact( compiled_artifact_blob: &[u8], executor_params: &ExecutorParams, params: &[u8], -) -> Result, String> { +) -> Result, ExecuteError> { let mut extensions = sp_externalities::Extensions::new(); extensions.register(sp_core::traits::ReadRuntimeVersionExt::new(ReadRuntimeVersion)); @@ -123,7 +124,6 @@ pub unsafe fn execute_artifact( Ok(Ok(ok)) => Ok(ok), Ok(Err(err)) | Err(err) => Err(err), } - .map_err(|err| format!("execute error: {:?}", err)) } /// Constructs the runtime for the given PVF, given the artifact bytes. diff --git a/polkadot/node/core/pvf/execute-worker/src/lib.rs b/polkadot/node/core/pvf/execute-worker/src/lib.rs index 0cfa5a786946..bd7e76010a6d 100644 --- a/polkadot/node/core/pvf/execute-worker/src/lib.rs +++ b/polkadot/node/core/pvf/execute-worker/src/lib.rs @@ -16,7 +16,9 @@ //! Contains the logic for executing PVFs. Used by the polkadot-execute-worker binary. -pub use polkadot_node_core_pvf_common::executor_interface::execute_artifact; +pub use polkadot_node_core_pvf_common::{ + error::ExecuteError, executor_interface::execute_artifact, +}; // NOTE: Initializing logging in e.g. tests will not have an effect in the workers, as they are // separate spawned processes. Run with e.g. `RUST_LOG=parachain::pvf-execute-worker=trace`. @@ -237,7 +239,9 @@ fn validate_using_artifact( // [`executor_interface::prepare`]. execute_artifact(compiled_artifact_blob, executor_params, params) } { - Err(err) => return JobResponse::format_invalid("execute", &err), + Err(ExecuteError::RuntimeConstruction(wasmerr)) => + return JobResponse::runtime_construction("execute", &wasmerr.to_string()), + Err(err) => return JobResponse::format_invalid("execute", &err.to_string()), Ok(d) => d, }; @@ -550,6 +554,8 @@ fn handle_parent_process( Ok(WorkerResponse::Ok { result_descriptor, duration: cpu_tv }) }, Ok(JobResponse::InvalidCandidate(err)) => Ok(WorkerResponse::InvalidCandidate(err)), + Ok(JobResponse::RuntimeConstruction(err)) => + Ok(WorkerResponse::RuntimeConstruction(err)), Err(job_error) => { gum::warn!( target: LOG_TARGET, diff --git a/polkadot/node/core/pvf/src/artifacts.rs b/polkadot/node/core/pvf/src/artifacts.rs index 78dfe71adadd..6288755526d4 100644 --- a/polkadot/node/core/pvf/src/artifacts.rs +++ b/polkadot/node/core/pvf/src/artifacts.rs @@ -238,6 +238,14 @@ impl Artifacts { .is_none()); } + /// Remove artifact by its id. + pub fn remove(&mut self, artifact_id: ArtifactId) -> Option<(ArtifactId, PathBuf)> { + self.inner.remove(&artifact_id).and_then(|state| match state { + ArtifactState::Prepared { path, .. } => Some((artifact_id, path)), + _ => None, + }) + } + /// Remove artifacts older than the given TTL and return id and path of the removed ones. pub fn prune(&mut self, artifact_ttl: Duration) -> Vec<(ArtifactId, PathBuf)> { let now = SystemTime::now(); diff --git a/polkadot/node/core/pvf/src/error.rs b/polkadot/node/core/pvf/src/error.rs index 80d41d5c64be..8dc96305eadb 100644 --- a/polkadot/node/core/pvf/src/error.rs +++ b/polkadot/node/core/pvf/src/error.rs @@ -86,6 +86,10 @@ pub enum PossiblyInvalidError { /// vote invalid. #[error("possibly invalid: job error: {0}")] JobError(String), + /// Instantiation of the WASM module instance failed during an execution. + /// Possibly related to local issues or dirty node update. May be retried with re-preparation. + #[error("possibly invalid: runtime construction: {0}")] + RuntimeConstruction(String), } impl From for ValidationError { diff --git a/polkadot/node/core/pvf/src/execute/mod.rs b/polkadot/node/core/pvf/src/execute/mod.rs index c6d9cf90fa28..365e98196cae 100644 --- a/polkadot/node/core/pvf/src/execute/mod.rs +++ b/polkadot/node/core/pvf/src/execute/mod.rs @@ -23,4 +23,4 @@ mod queue; mod worker_interface; -pub use queue::{start, PendingExecutionRequest, ToQueue}; +pub use queue::{start, FromQueue, PendingExecutionRequest, ToQueue}; diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index aa91d11781fc..bdc3c7327b06 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -25,7 +25,7 @@ use crate::{ InvalidCandidate, PossiblyInvalidError, ValidationError, LOG_TARGET, }; use futures::{ - channel::mpsc, + channel::{mpsc, oneshot}, future::BoxFuture, stream::{FuturesUnordered, StreamExt as _}, Future, FutureExt, @@ -54,6 +54,12 @@ pub enum ToQueue { Enqueue { artifact: ArtifactPathId, pending_execution_request: PendingExecutionRequest }, } +/// A response from queue. +#[derive(Debug)] +pub enum FromQueue { + RemoveArtifact { artifact: ArtifactId, reply_to: oneshot::Sender<()> }, +} + /// An execution request that should execute the PVF (known in the context) and send the results /// to the given result sender. #[derive(Debug)] @@ -137,6 +143,8 @@ struct Queue { /// The receiver that receives messages to the pool. to_queue_rx: mpsc::Receiver, + /// The sender to send messages back to validation host. + from_queue_tx: mpsc::UnboundedSender, // Some variables related to the current session. program_path: PathBuf, @@ -161,6 +169,7 @@ impl Queue { node_version: Option, security_status: SecurityStatus, to_queue_rx: mpsc::Receiver, + from_queue_tx: mpsc::UnboundedSender, ) -> Self { Self { metrics, @@ -170,6 +179,7 @@ impl Queue { node_version, security_status, to_queue_rx, + from_queue_tx, queue: VecDeque::new(), mux: Mux::new(), workers: Workers { @@ -301,7 +311,7 @@ async fn handle_mux(queue: &mut Queue, event: QueueEvent) { handle_worker_spawned(queue, idle, handle, job); }, QueueEvent::StartWork(worker, outcome, artifact_id, result_tx) => { - handle_job_finish(queue, worker, outcome, artifact_id, result_tx); + handle_job_finish(queue, worker, outcome, artifact_id, result_tx).await; }, } } @@ -327,42 +337,69 @@ fn handle_worker_spawned( /// If there are pending jobs in the queue, schedules the next of them onto the just freed up /// worker. Otherwise, puts back into the available workers list. -fn handle_job_finish( +async fn handle_job_finish( queue: &mut Queue, worker: Worker, outcome: Outcome, artifact_id: ArtifactId, result_tx: ResultSender, ) { - let (idle_worker, result, duration) = match outcome { + let (idle_worker, result, duration, sync_channel) = match outcome { Outcome::Ok { result_descriptor, duration, idle_worker } => { // TODO: propagate the soft timeout - (Some(idle_worker), Ok(result_descriptor), Some(duration)) + (Some(idle_worker), Ok(result_descriptor), Some(duration), None) }, Outcome::InvalidCandidate { err, idle_worker } => ( Some(idle_worker), Err(ValidationError::Invalid(InvalidCandidate::WorkerReportedInvalid(err))), None, + None, ), - Outcome::InternalError { err } => (None, Err(ValidationError::Internal(err)), None), + Outcome::RuntimeConstruction { err, idle_worker } => { + // The task for artifact removal is executed concurrently with + // the message to the host on the execution result. + let (result_tx, result_rx) = oneshot::channel(); + queue + .from_queue_tx + .unbounded_send(FromQueue::RemoveArtifact { + artifact: artifact_id.clone(), + reply_to: result_tx, + }) + .expect("from execute queue receiver is listened by the host; qed"); + ( + Some(idle_worker), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::RuntimeConstruction( + err, + ))), + None, + Some(result_rx), + ) + }, + Outcome::InternalError { err } => (None, Err(ValidationError::Internal(err)), None, None), // Either the worker or the job timed out. Kill the worker in either case. Treated as // definitely-invalid, because if we timed out, there's no time left for a retry. Outcome::HardTimeout => - (None, Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)), None), + (None, Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)), None, None), // "Maybe invalid" errors (will retry). Outcome::WorkerIntfErr => ( None, Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), None, + None, ), Outcome::JobDied { err } => ( None, Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousJobDeath(err))), None, + None, + ), + Outcome::JobError { err } => ( + None, + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(err))), + None, + None, ), - Outcome::JobError { err } => - (None, Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(err))), None), }; queue.metrics.execute_finished(); @@ -386,6 +423,12 @@ fn handle_job_finish( ); } + if let Some(sync_channel) = sync_channel { + // err means the sender is dropped (the artifact is already removed from the cache) + // so that's legitimate to ignore the result + let _ = sync_channel.await; + } + // First we send the result. It may fail due to the other end of the channel being dropped, // that's legitimate and we don't treat that as an error. let _ = result_tx.send(result); @@ -521,8 +564,10 @@ pub fn start( spawn_timeout: Duration, node_version: Option, security_status: SecurityStatus, -) -> (mpsc::Sender, impl Future) { +) -> (mpsc::Sender, mpsc::UnboundedReceiver, impl Future) { let (to_queue_tx, to_queue_rx) = mpsc::channel(20); + let (from_queue_tx, from_queue_rx) = mpsc::unbounded(); + let run = Queue::new( metrics, program_path, @@ -532,7 +577,8 @@ pub fn start( node_version, security_status, to_queue_rx, + from_queue_tx, ) .run(); - (to_queue_tx, run) + (to_queue_tx, from_queue_rx, run) } diff --git a/polkadot/node/core/pvf/src/execute/worker_interface.rs b/polkadot/node/core/pvf/src/execute/worker_interface.rs index 9f7738f00e69..db81da118d7b 100644 --- a/polkadot/node/core/pvf/src/execute/worker_interface.rs +++ b/polkadot/node/core/pvf/src/execute/worker_interface.rs @@ -87,6 +87,10 @@ pub enum Outcome { /// a trap. Errors related to the preparation process are not expected to be encountered by the /// execution workers. InvalidCandidate { err: String, idle_worker: IdleWorker }, + /// The error is probably transient. It may be for example + /// because the artifact was prepared with a Wasmtime version different from the version + /// in the current execution environment. + RuntimeConstruction { err: String, idle_worker: IdleWorker }, /// The execution time exceeded the hard limit. The worker is terminated. HardTimeout, /// An I/O error happened during communication with the worker. This may mean that the worker @@ -193,6 +197,10 @@ pub async fn start_work( err, idle_worker: IdleWorker { stream, pid, worker_dir }, }, + WorkerResponse::RuntimeConstruction(err) => Outcome::RuntimeConstruction { + err, + idle_worker: IdleWorker { stream, pid, worker_dir }, + }, WorkerResponse::JobTimedOut => Outcome::HardTimeout, WorkerResponse::JobDied { err, job_pid: _ } => Outcome::JobDied { err }, WorkerResponse::JobError(err) => Outcome::JobError { err }, diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index ae9fdc7d2dea..8ec46f4b08f1 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -274,7 +274,7 @@ pub async fn start( from_prepare_pool, ); - let (to_execute_queue_tx, run_execute_queue) = execute::start( + let (to_execute_queue_tx, from_execute_queue_rx, run_execute_queue) = execute::start( metrics, config.execute_worker_program_path.to_owned(), config.cache_path.clone(), @@ -296,6 +296,7 @@ pub async fn start( to_prepare_queue_tx, from_prepare_queue_rx, to_execute_queue_tx, + from_execute_queue_rx, to_sweeper_tx, awaiting_prepare: AwaitingPrepare::default(), }) @@ -342,6 +343,8 @@ struct Inner { from_prepare_queue_rx: mpsc::UnboundedReceiver, to_execute_queue_tx: mpsc::Sender, + from_execute_queue_rx: mpsc::UnboundedReceiver, + to_sweeper_tx: mpsc::Sender, awaiting_prepare: AwaitingPrepare, @@ -358,6 +361,7 @@ async fn run( to_host_rx, from_prepare_queue_rx, mut to_prepare_queue_tx, + from_execute_queue_rx, mut to_execute_queue_tx, mut to_sweeper_tx, mut awaiting_prepare, @@ -384,10 +388,21 @@ async fn run( let mut to_host_rx = to_host_rx.fuse(); let mut from_prepare_queue_rx = from_prepare_queue_rx.fuse(); + let mut from_execute_queue_rx = from_execute_queue_rx.fuse(); loop { // biased to make it behave deterministically for tests. futures::select_biased! { + from_execute_queue_rx = from_execute_queue_rx.next() => { + let from_queue = break_if_fatal!(from_execute_queue_rx.ok_or(Fatal)); + let execute::FromQueue::RemoveArtifact { artifact, reply_to } = from_queue; + break_if_fatal!(handle_artifact_removal( + &mut to_sweeper_tx, + &mut artifacts, + artifact, + reply_to, + ).await); + }, () = cleanup_pulse.select_next_some() => { // `select_next_some` because we don't expect this to fail, but if it does, we // still don't fail. The trade-off is that the compiled cache will start growing @@ -861,6 +876,37 @@ async fn handle_cleanup_pulse( Ok(()) } +async fn handle_artifact_removal( + sweeper_tx: &mut mpsc::Sender, + artifacts: &mut Artifacts, + artifact_id: ArtifactId, + reply_to: oneshot::Sender<()>, +) -> Result<(), Fatal> { + let (artifact_id, path) = if let Some(artifact) = artifacts.remove(artifact_id) { + artifact + } else { + // if we haven't found the artifact by its id, + // it has been probably removed + // anyway with the randomness of the artifact name + // it is safe to ignore + return Ok(()); + }; + reply_to + .send(()) + .expect("the execute queue waits for the artifact remove confirmation; qed"); + // Thanks to the randomness of the artifact name (see + // `artifacts::generate_artifact_path`) there is no issue with any name conflict on + // future repreparation. + // So we can confirm the artifact removal already + gum::debug!( + target: LOG_TARGET, + validation_code_hash = ?artifact_id.code_hash, + "PVF pruning: pruning artifact by request from the execute queue", + ); + sweeper_tx.send(path).await.map_err(|_| Fatal)?; + Ok(()) +} + /// A simple task which sole purpose is to delete files thrown at it. async fn sweeper_task(mut sweeper_rx: mpsc::Receiver) { loop { @@ -968,6 +1014,8 @@ pub(crate) mod tests { to_prepare_queue_rx: mpsc::Receiver, from_prepare_queue_tx: mpsc::UnboundedSender, to_execute_queue_rx: mpsc::Receiver, + #[allow(unused)] + from_execute_queue_tx: mpsc::UnboundedSender, to_sweeper_rx: mpsc::Receiver, run: BoxFuture<'static, ()>, @@ -979,6 +1027,7 @@ pub(crate) mod tests { let (to_prepare_queue_tx, to_prepare_queue_rx) = mpsc::channel(10); let (from_prepare_queue_tx, from_prepare_queue_rx) = mpsc::unbounded(); let (to_execute_queue_tx, to_execute_queue_rx) = mpsc::channel(10); + let (from_execute_queue_tx, from_execute_queue_rx) = mpsc::unbounded(); let (to_sweeper_tx, to_sweeper_rx) = mpsc::channel(10); let run = run(Inner { @@ -989,6 +1038,7 @@ pub(crate) mod tests { to_prepare_queue_tx, from_prepare_queue_rx, to_execute_queue_tx, + from_execute_queue_rx, to_sweeper_tx, awaiting_prepare: AwaitingPrepare::default(), }) @@ -999,6 +1049,7 @@ pub(crate) mod tests { to_prepare_queue_rx, from_prepare_queue_tx, to_execute_queue_rx, + from_execute_queue_tx, to_sweeper_rx, run, } diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index bcc10749e746..cdfbcd8e5785 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -21,13 +21,14 @@ use parity_scale_codec::Encode as _; #[cfg(all(feature = "ci-only-tests", target_os = "linux"))] use polkadot_node_core_pvf::SecurityStatus; use polkadot_node_core_pvf::{ - start, testing::build_workers_and_get_paths, Config, InvalidCandidate, Metrics, PrepareError, - PrepareJobKind, PvfPrepData, ValidationError, ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR, + start, testing::build_workers_and_get_paths, Config, InvalidCandidate, Metrics, + PossiblyInvalidError, PrepareError, PrepareJobKind, PvfPrepData, ValidationError, + ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR, }; use polkadot_parachain_primitives::primitives::{BlockData, ValidationParams, ValidationResult}; use polkadot_primitives::{ExecutorParam, ExecutorParams}; -use std::time::Duration; +use std::{io::Write, time::Duration}; use tokio::sync::Mutex; mod adder; @@ -352,10 +353,80 @@ async fn deleting_prepared_artifact_does_not_dispute() { ) .await; - match result { - Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)) => {}, - r => panic!("{:?}", r), + assert_matches!(result, Err(ValidationError::Invalid(InvalidCandidate::HardTimeout))); +} + +// Test that corruption of a prepared artifact does not lead to a dispute when we try to execute it. +#[tokio::test] +async fn corrupted_prepared_artifact_does_not_dispute() { + let host = TestHost::new().await; + let cache_dir = host.cache_dir.path(); + + let _stats = host.precheck_pvf(halt::wasm_binary_unwrap(), Default::default()).await.unwrap(); + + // Manually corrupting the prepared artifact from disk. The in-memory artifacts table won't + // change. + let artifact_path = { + // Get the artifact path (asserting it exists). + let mut cache_dir: Vec<_> = std::fs::read_dir(cache_dir).unwrap().collect(); + // Should contain the artifact and the worker dir. + assert_eq!(cache_dir.len(), 2); + let mut artifact_path = cache_dir.pop().unwrap().unwrap(); + if artifact_path.path().is_dir() { + artifact_path = cache_dir.pop().unwrap().unwrap(); + } + + // Corrupt the artifact. + let mut f = std::fs::OpenOptions::new() + .write(true) + .truncate(true) + .open(artifact_path.path()) + .unwrap(); + f.write_all(b"corrupted wasm").unwrap(); + f.flush().unwrap(); + artifact_path + }; + + assert!(artifact_path.path().exists()); + + // Try to validate, artifact should get removed because of the corruption. + let result = host + .validate_candidate( + halt::wasm_binary_unwrap(), + ValidationParams { + block_data: BlockData(Vec::new()), + parent_head: Default::default(), + relay_parent_number: 1, + relay_parent_storage_root: Default::default(), + }, + Default::default(), + ) + .await; + + assert_matches!( + result, + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::RuntimeConstruction(_))) + ); + + // because of RuntimeConstruction we may retry + host.precheck_pvf(halt::wasm_binary_unwrap(), Default::default()).await.unwrap(); + + // The actual artifact removal is done concurrently + // with sending of the result of the execution + // it is not a problem for further re-preparation as + // artifact filenames are random + for _ in 1..5 { + if !artifact_path.path().exists() { + break; + } + tokio::time::sleep(Duration::from_secs(1)).await; } + + assert!( + !artifact_path.path().exists(), + "the corrupted artifact ({}) should be deleted by the host", + artifact_path.path().display() + ); } #[tokio::test] diff --git a/polkadot/primitives/src/v6/executor_params.rs b/polkadot/primitives/src/v6/executor_params.rs index 112a529f62b0..1e19f3b23fec 100644 --- a/polkadot/primitives/src/v6/executor_params.rs +++ b/polkadot/primitives/src/v6/executor_params.rs @@ -159,7 +159,9 @@ impl sp_std::fmt::LowerHex for ExecutorParamsHash { // into individual fields of the structure. Thus, complex migrations shall be avoided when adding // new entries and removing old ones. At the moment, there's no mandatory parameters defined. If // they show up, they must be clearly documented as mandatory ones. -#[derive(Clone, Debug, Encode, Decode, PartialEq, Eq, TypeInfo, Serialize, Deserialize)] +#[derive( + Clone, Debug, Default, Encode, Decode, PartialEq, Eq, TypeInfo, Serialize, Deserialize, +)] pub struct ExecutorParams(Vec); impl ExecutorParams { @@ -334,9 +336,3 @@ impl From<&[ExecutorParam]> for ExecutorParams { ExecutorParams(arr.to_vec()) } } - -impl Default for ExecutorParams { - fn default() -> Self { - ExecutorParams(vec![]) - } -} diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md b/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md index e0984bd58d1d..39a317a07c84 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md @@ -125,6 +125,14 @@ execution request: reason, which may or may not be independent of the candidate or PVF. 5. **Internal errors:** See "Internal Errors" section. In this case, after the retry we abstain from voting. +6. **RuntimeConstruction** error. The precheck handles a general case of a wrong + artifact but doesn't guarantee its consistency between the preparation and + the execution. If something happened with the artifact between + the preparation of the artifact and its execution (e.g. the artifact was + corrupted on disk or a dirty node upgrade happened when the prepare worker + has a wasmtime version different from the execute worker's wasmtime version). + We treat such an error as possibly transient due to local issues and retry + one time. ### Preparation timeouts diff --git a/prdoc/pr_3187.prdoc b/prdoc/pr_3187.prdoc new file mode 100644 index 000000000000..bda41142d86c --- /dev/null +++ b/prdoc/pr_3187.prdoc @@ -0,0 +1,16 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Retrying an execution on failed runtime construction + +doc: + - audience: Node Dev + description: | + If a runtime construction error happened during the execution request, then the artifact is re-prepared + and the execution request is retried at most once. See also the related issue. + +crates: + - name: polkadot-node-core-candidate-validation + - name: polkadot-node-core-pvf + - name: polkadot-node-core-pvf-execute-worker + - name: polkadot-node-core-pvf-common From 576681b867a234b42aac282940455f9be71d108b Mon Sep 17 00:00:00 2001 From: Clara van Staden Date: Wed, 28 Feb 2024 21:42:35 +0200 Subject: [PATCH 05/13] Snowbridge - Extract Ethereum Chain ID (#3501) While adding runtime tests to https://github.com/polkadot-fellows/runtimes/pull/130, I noticed the Ethereum chain ID was hardcoded. For Kusama + Polkadot, the Ethereum chain ID should 1 (Mainnet), whereas on Rococo it is 11155111 (Sepolia). This PR also updates the Snowbridge crates versions to the current versions on crates.io. --------- Co-authored-by: claravanstaden --- Cargo.lock | 28 +++++++++---------- .../pallets/ethereum-client/Cargo.toml | 2 +- .../pallets/inbound-queue/Cargo.toml | 2 +- .../pallets/inbound-queue/fixtures/Cargo.toml | 2 +- .../pallets/outbound-queue/Cargo.toml | 2 +- .../outbound-queue/merkle-tree/Cargo.toml | 2 +- .../outbound-queue/runtime-api/Cargo.toml | 2 +- bridges/snowbridge/pallets/system/Cargo.toml | 2 +- .../pallets/system/runtime-api/Cargo.toml | 2 +- .../snowbridge/primitives/beacon/Cargo.toml | 2 +- bridges/snowbridge/primitives/core/Cargo.toml | 2 +- .../snowbridge/primitives/ethereum/Cargo.toml | 2 +- .../snowbridge/primitives/router/Cargo.toml | 2 +- .../runtime/runtime-common/Cargo.toml | 2 +- .../snowbridge/runtime/test-common/Cargo.toml | 2 +- .../snowbridge/runtime/test-common/src/lib.rs | 12 ++++++-- .../snowbridge/scripts/contribute-upstream.sh | 2 ++ .../bridge-hub-rococo/tests/snowbridge.rs | 5 ++++ 18 files changed, 45 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef510986257a..197d05e26339 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17856,7 +17856,7 @@ dependencies = [ [[package]] name = "snowbridge-beacon-primitives" -version = "0.0.0" +version = "0.2.0" dependencies = [ "byte-slice-cast", "frame-support", @@ -17880,7 +17880,7 @@ dependencies = [ [[package]] name = "snowbridge-core" -version = "0.0.0" +version = "0.2.0" dependencies = [ "ethabi-decode", "frame-support", @@ -17903,7 +17903,7 @@ dependencies = [ [[package]] name = "snowbridge-ethereum" -version = "0.1.0" +version = "0.3.0" dependencies = [ "ethabi-decode", "ethbloom", @@ -17942,7 +17942,7 @@ dependencies = [ [[package]] name = "snowbridge-outbound-queue-merkle-tree" -version = "0.1.1" +version = "0.3.0" dependencies = [ "array-bytes 4.2.0", "env_logger 0.9.3", @@ -17957,7 +17957,7 @@ dependencies = [ [[package]] name = "snowbridge-outbound-queue-runtime-api" -version = "0.0.0" +version = "0.2.0" dependencies = [ "frame-support", "parity-scale-codec", @@ -17971,7 +17971,7 @@ dependencies = [ [[package]] name = "snowbridge-pallet-ethereum-client" -version = "0.0.0" +version = "0.2.0" dependencies = [ "bp-runtime", "byte-slice-cast", @@ -18017,7 +18017,7 @@ dependencies = [ [[package]] name = "snowbridge-pallet-inbound-queue" -version = "0.0.0" +version = "0.2.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -18050,7 +18050,7 @@ dependencies = [ [[package]] name = "snowbridge-pallet-inbound-queue-fixtures" -version = "0.9.0" +version = "0.10.0" dependencies = [ "frame-benchmarking", "frame-support", @@ -18064,7 +18064,7 @@ dependencies = [ [[package]] name = "snowbridge-pallet-outbound-queue" -version = "0.0.0" +version = "0.2.0" dependencies = [ "bridge-hub-common", "ethabi-decode", @@ -18089,7 +18089,7 @@ dependencies = [ [[package]] name = "snowbridge-pallet-system" -version = "0.0.0" +version = "0.2.0" dependencies = [ "ethabi-decode", "frame-benchmarking", @@ -18117,7 +18117,7 @@ dependencies = [ [[package]] name = "snowbridge-router-primitives" -version = "0.0.0" +version = "0.9.0" dependencies = [ "ethabi-decode", "frame-support", @@ -18140,7 +18140,7 @@ dependencies = [ [[package]] name = "snowbridge-runtime-common" -version = "0.0.0" +version = "0.2.0" dependencies = [ "frame-support", "frame-system", @@ -18156,7 +18156,7 @@ dependencies = [ [[package]] name = "snowbridge-runtime-test-common" -version = "0.0.0" +version = "0.2.0" dependencies = [ "assets-common", "bridge-hub-test-utils", @@ -18233,7 +18233,7 @@ dependencies = [ [[package]] name = "snowbridge-system-runtime-api" -version = "0.0.0" +version = "0.2.0" dependencies = [ "parity-scale-codec", "snowbridge-core", diff --git a/bridges/snowbridge/pallets/ethereum-client/Cargo.toml b/bridges/snowbridge/pallets/ethereum-client/Cargo.toml index 99b429053114..c8999633c97a 100644 --- a/bridges/snowbridge/pallets/ethereum-client/Cargo.toml +++ b/bridges/snowbridge/pallets/ethereum-client/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-pallet-ethereum-client" description = "Snowbridge Ethereum Client Pallet" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/inbound-queue/Cargo.toml b/bridges/snowbridge/pallets/inbound-queue/Cargo.toml index c26ef90a22db..b850496cd4e1 100644 --- a/bridges/snowbridge/pallets/inbound-queue/Cargo.toml +++ b/bridges/snowbridge/pallets/inbound-queue/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-pallet-inbound-queue" description = "Snowbridge Inbound Queue Pallet" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/inbound-queue/fixtures/Cargo.toml b/bridges/snowbridge/pallets/inbound-queue/fixtures/Cargo.toml index 61f1421e0567..64605a42f0d3 100644 --- a/bridges/snowbridge/pallets/inbound-queue/fixtures/Cargo.toml +++ b/bridges/snowbridge/pallets/inbound-queue/fixtures/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-pallet-inbound-queue-fixtures" description = "Snowbridge Inbound Queue Test Fixtures" -version = "0.9.0" +version = "0.10.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/outbound-queue/Cargo.toml b/bridges/snowbridge/pallets/outbound-queue/Cargo.toml index 40e57db4bbd9..f16a28cb1e45 100644 --- a/bridges/snowbridge/pallets/outbound-queue/Cargo.toml +++ b/bridges/snowbridge/pallets/outbound-queue/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-pallet-outbound-queue" description = "Snowbridge Outbound Queue Pallet" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/outbound-queue/merkle-tree/Cargo.toml b/bridges/snowbridge/pallets/outbound-queue/merkle-tree/Cargo.toml index c185d5af7062..0606e9de3305 100644 --- a/bridges/snowbridge/pallets/outbound-queue/merkle-tree/Cargo.toml +++ b/bridges/snowbridge/pallets/outbound-queue/merkle-tree/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-outbound-queue-merkle-tree" description = "Snowbridge Outbound Queue Merkle Tree" -version = "0.1.1" +version = "0.3.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/outbound-queue/runtime-api/Cargo.toml b/bridges/snowbridge/pallets/outbound-queue/runtime-api/Cargo.toml index 347b3bae493b..cb68fd0a250a 100644 --- a/bridges/snowbridge/pallets/outbound-queue/runtime-api/Cargo.toml +++ b/bridges/snowbridge/pallets/outbound-queue/runtime-api/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-outbound-queue-runtime-api" description = "Snowbridge Outbound Queue Runtime API" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/system/Cargo.toml b/bridges/snowbridge/pallets/system/Cargo.toml index f6c642e7376f..5ad04290de04 100644 --- a/bridges/snowbridge/pallets/system/Cargo.toml +++ b/bridges/snowbridge/pallets/system/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-pallet-system" description = "Snowbridge System Pallet" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/pallets/system/runtime-api/Cargo.toml b/bridges/snowbridge/pallets/system/runtime-api/Cargo.toml index 355d2d29147f..eb02ae1db529 100644 --- a/bridges/snowbridge/pallets/system/runtime-api/Cargo.toml +++ b/bridges/snowbridge/pallets/system/runtime-api/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-system-runtime-api" description = "Snowbridge System Runtime API" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/primitives/beacon/Cargo.toml b/bridges/snowbridge/primitives/beacon/Cargo.toml index e021bb010714..d181fa1d3945 100644 --- a/bridges/snowbridge/primitives/beacon/Cargo.toml +++ b/bridges/snowbridge/primitives/beacon/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-beacon-primitives" description = "Snowbridge Beacon Primitives" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/primitives/core/Cargo.toml b/bridges/snowbridge/primitives/core/Cargo.toml index 090c48c403fe..9a299ad0ae92 100644 --- a/bridges/snowbridge/primitives/core/Cargo.toml +++ b/bridges/snowbridge/primitives/core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-core" description = "Snowbridge Core" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/primitives/ethereum/Cargo.toml b/bridges/snowbridge/primitives/ethereum/Cargo.toml index 399139cef381..9fa725a6c056 100644 --- a/bridges/snowbridge/primitives/ethereum/Cargo.toml +++ b/bridges/snowbridge/primitives/ethereum/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-ethereum" description = "Snowbridge Ethereum" -version = "0.1.0" +version = "0.3.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/primitives/router/Cargo.toml b/bridges/snowbridge/primitives/router/Cargo.toml index 750a2a73e637..ded773e0d389 100644 --- a/bridges/snowbridge/primitives/router/Cargo.toml +++ b/bridges/snowbridge/primitives/router/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-router-primitives" description = "Snowbridge Router Primitives" -version = "0.0.0" +version = "0.9.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/runtime/runtime-common/Cargo.toml b/bridges/snowbridge/runtime/runtime-common/Cargo.toml index d4c86f8aa750..bf5e9a8832dc 100644 --- a/bridges/snowbridge/runtime/runtime-common/Cargo.toml +++ b/bridges/snowbridge/runtime/runtime-common/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-runtime-common" description = "Snowbridge Runtime Common" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition.workspace = true repository.workspace = true diff --git a/bridges/snowbridge/runtime/test-common/Cargo.toml b/bridges/snowbridge/runtime/test-common/Cargo.toml index ec4466b34265..4e8b311cb978 100644 --- a/bridges/snowbridge/runtime/test-common/Cargo.toml +++ b/bridges/snowbridge/runtime/test-common/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "snowbridge-runtime-test-common" description = "Snowbridge Runtime Tests" -version = "0.0.0" +version = "0.2.0" authors = ["Snowfork "] edition = "2021" license = "Apache-2.0" diff --git a/bridges/snowbridge/runtime/test-common/src/lib.rs b/bridges/snowbridge/runtime/test-common/src/lib.rs index 29b0e738c182..7455adf76170 100644 --- a/bridges/snowbridge/runtime/test-common/src/lib.rs +++ b/bridges/snowbridge/runtime/test-common/src/lib.rs @@ -40,6 +40,7 @@ where } pub fn send_transfer_token_message( + ethereum_chain_id: u64, assethub_parachain_id: u32, weth_contract_address: H160, destination_address: H160, @@ -89,7 +90,7 @@ where WithdrawAsset(Assets::from(vec![fee.clone()])), BuyExecution { fees: fee, weight_limit: Unlimited }, ExportMessage { - network: Ethereum { chain_id: 11155111 }, + network: Ethereum { chain_id: ethereum_chain_id }, destination: Here, xcm: inner_xcm, }, @@ -107,6 +108,7 @@ where } pub fn send_transfer_token_message_success( + ethereum_chain_id: u64, collator_session_key: CollatorSessionKeys, runtime_para_id: u32, assethub_parachain_id: u32, @@ -149,6 +151,7 @@ pub fn send_transfer_token_message_success( initial_fund::(assethub_parachain_id, 5_000_000_000_000); let outcome = send_transfer_token_message::( + ethereum_chain_id, assethub_parachain_id, weth_contract_address, destination_address, @@ -205,6 +208,7 @@ pub fn ethereum_outbound_queue_processes_messages_before_message_queue_works< XcmConfig, AllPalletsWithoutSystem, >( + ethereum_chain_id: u64, collator_session_key: CollatorSessionKeys, runtime_para_id: u32, assethub_parachain_id: u32, @@ -249,6 +253,7 @@ pub fn ethereum_outbound_queue_processes_messages_before_message_queue_works< initial_fund::(assethub_parachain_id, 5_000_000_000_000); let outcome = send_transfer_token_message::( + ethereum_chain_id, assethub_parachain_id, weth_contract_address, destination_address, @@ -290,6 +295,7 @@ pub fn ethereum_outbound_queue_processes_messages_before_message_queue_works< } pub fn send_unpaid_transfer_token_message( + ethereum_chain_id: u64, collator_session_key: CollatorSessionKeys, runtime_para_id: u32, assethub_parachain_id: u32, @@ -353,7 +359,7 @@ pub fn send_unpaid_transfer_token_message( let xcm = Xcm(vec![ UnpaidExecution { weight_limit: Unlimited, check_origin: None }, ExportMessage { - network: Ethereum { chain_id: 11155111 }, + network: Ethereum { chain_id: ethereum_chain_id }, destination: Here, xcm: inner_xcm, }, @@ -375,6 +381,7 @@ pub fn send_unpaid_transfer_token_message( #[allow(clippy::too_many_arguments)] pub fn send_transfer_token_message_failure( + ethereum_chain_id: u64, collator_session_key: CollatorSessionKeys, runtime_para_id: u32, assethub_parachain_id: u32, @@ -414,6 +421,7 @@ pub fn send_transfer_token_message_failure( initial_fund::(assethub_parachain_id, initial_amount); let outcome = send_transfer_token_message::( + ethereum_chain_id, assethub_parachain_id, weth_contract_address, destination_address, diff --git a/bridges/snowbridge/scripts/contribute-upstream.sh b/bridges/snowbridge/scripts/contribute-upstream.sh index 8aa2d2a7035e..32005b770ecf 100755 --- a/bridges/snowbridge/scripts/contribute-upstream.sh +++ b/bridges/snowbridge/scripts/contribute-upstream.sh @@ -40,10 +40,12 @@ git checkout "$branch_name" # remove everything we think is not required for our needs rm -rf rust-toolchain.toml +rm -rf codecov.yml rm -rf $SNOWBRIDGE_FOLDER/.cargo rm -rf $SNOWBRIDGE_FOLDER/.github rm -rf $SNOWBRIDGE_FOLDER/SECURITY.md rm -rf $SNOWBRIDGE_FOLDER/.gitignore +rm -rf $SNOWBRIDGE_FOLDER/rustfmt.toml rm -rf $SNOWBRIDGE_FOLDER/templates rm -rf $SNOWBRIDGE_FOLDER/pallets/ethereum-client/fuzz diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/snowbridge.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/snowbridge.rs index b9f43624b652..239bd946e759 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/snowbridge.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/snowbridge.rs @@ -51,6 +51,7 @@ fn collator_session_keys() -> bridge_hub_test_utils::CollatorSessionKeys( + 11155111, collator_session_keys(), 1013, 1000, @@ -69,6 +70,7 @@ pub fn transfer_token_to_ethereum_works() { #[test] pub fn unpaid_transfer_token_to_ethereum_fails_with_barrier() { snowbridge_runtime_test_common::send_unpaid_transfer_token_message::( + 11155111, collator_session_keys(), 1013, 1000, @@ -80,6 +82,7 @@ pub fn unpaid_transfer_token_to_ethereum_fails_with_barrier() { #[test] pub fn transfer_token_to_ethereum_fee_not_enough() { snowbridge_runtime_test_common::send_transfer_token_message_failure::( + 11155111, collator_session_keys(), 1013, 1000, @@ -95,6 +98,7 @@ pub fn transfer_token_to_ethereum_fee_not_enough() { #[test] pub fn transfer_token_to_ethereum_insufficient_fund() { snowbridge_runtime_test_common::send_transfer_token_message_failure::( + 11155111, collator_session_keys(), 1013, 1000, @@ -146,6 +150,7 @@ pub fn ethereum_outbound_queue_processes_messages_before_message_queue_works() { XcmConfig, AllPalletsWithoutSystem, >( + 11155111, collator_session_keys(), 1013, 1000, From eefd5fe4499515da66d088505093f75b5aa22550 Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Wed, 28 Feb 2024 20:49:00 +0100 Subject: [PATCH 06/13] Multi-Block-Migrations, `poll` hook and new System callbacks (#1781) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This MR is the merge of https://github.com/paritytech/substrate/pull/14414 and https://github.com/paritytech/substrate/pull/14275. It implements [RFC#13](https://github.com/polkadot-fellows/RFCs/pull/13), closes https://github.com/paritytech/polkadot-sdk/issues/198. ----- This Merge request introduces three major topicals: 1. Multi-Block-Migrations 1. New pallet `poll` hook for periodic service work 1. Replacement hooks for `on_initialize` and `on_finalize` in cases where `poll` cannot be used and some more general changes to FRAME. The changes for each topical span over multiple crates. They are listed in topical order below. # 1.) Multi-Block-Migrations Multi-Block-Migrations are facilitated by creating `pallet_migrations` and configuring `System::Config::MultiBlockMigrator` to point to it. Executive picks this up and triggers one step of the migrations pallet per block. The chain is in lockdown mode for as long as an MBM is ongoing. Executive does this by polling `MultiBlockMigrator::ongoing` and not allowing any transaction in a block, if true. A MBM is defined through trait `SteppedMigration`. A condensed version looks like this: ```rust /// A migration that can proceed in multiple steps. pub trait SteppedMigration { type Cursor: FullCodec + MaxEncodedLen; type Identifier: FullCodec + MaxEncodedLen; fn id() -> Self::Identifier; fn max_steps() -> Option; fn step( cursor: Option, meter: &mut WeightMeter, ) -> Result, SteppedMigrationError>; } ``` `pallet_migrations` can be configured with an aggregated tuple of these migrations. It then starts to migrate them one-by-one on the next runtime upgrade. Two things are important here: - 1. Doing another runtime upgrade while MBMs are ongoing is not a good idea and can lead to messed up state. - 2. **Pallet Migrations MUST BE CONFIGURED IN `System::Config`, otherwise it is not used.** The pallet supports an `UpgradeStatusHandler` that can be used to notify external logic of upgrade start/finish (for example to pause XCM dispatch). Error recovery is very limited in the case that a migration errors or times out (exceeds its `max_steps`). Currently the runtime dev can decide in `FailedMigrationHandler::failed` how to handle this. One follow-up would be to pair this with the `SafeMode` pallet and enact safe mode when an upgrade fails, to allow governance to rescue the chain. This is currently not possible, since governance is not `Mandatory`. ## Runtime API - `Core`: `initialize_block` now returns `ExtrinsicInclusionMode` to inform the Block Author whether they can push transactions. ### Integration Add it to your runtime implementation of `Core` and `BlockBuilder`: ```patch diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs @@ impl_runtime_apis! { impl sp_block_builder::Core for Runtime { - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> RuntimeExecutiveMode { Executive::initialize_block(header) } ... } ``` # 2.) `poll` hook A new pallet hook is introduced: `poll`. `Poll` is intended to replace mostly all usage of `on_initialize`. The reason for this is that any code that can be called from `on_initialize` cannot be migrated through an MBM. Currently there is no way to statically check this; the implication is to use `on_initialize` as rarely as possible. Failing to do so can result in broken storage invariants. The implementation of the poll hook depends on the `Runtime API` changes that are explained above. # 3.) Hard-Deadline callbacks Three new callbacks are introduced and configured on `System::Config`: `PreInherents`, `PostInherents` and `PostTransactions`. These hooks are meant as replacement for `on_initialize` and `on_finalize` in cases where the code that runs cannot be moved to `poll`. The reason for this is to make the usage of HD-code (hard deadline) more explicit - again to prevent broken invariants by MBMs. # 4.) FRAME (general changes) ## `frame_system` pallet A new memorize storage item `InherentsApplied` is added. It is used by executive to track whether inherents have already been applied. Executive and can then execute the MBMs directly between inherents and transactions. The `Config` gets five new items: - `SingleBlockMigrations` this is the new way of configuring migrations that run in a single block. Previously they were defined as last generic argument of `Executive`. This shift is brings all central configuration about migrations closer into view of the developer (migrations that are configured in `Executive` will still work for now but is deprecated). - `MultiBlockMigrator` this can be configured to an engine that drives MBMs. One example would be the `pallet_migrations`. Note that this is only the engine; the exact MBMs are injected into the engine. - `PreInherents` a callback that executes after `on_initialize` but before inherents. - `PostInherents` a callback that executes after all inherents ran (including MBMs and `poll`). - `PostTransactions` in symmetry to `PreInherents`, this one is called before `on_finalize` but after all transactions. A sane default is to set all of these to `()`. Example diff suitable for any chain: ```patch @@ impl frame_system::Config for Test { type MaxConsumers = ConstU32<16>; + type SingleBlockMigrations = (); + type MultiBlockMigrator = (); + type PreInherents = (); + type PostInherents = (); + type PostTransactions = (); } ``` An overview of how the block execution now looks like is here. The same graph is also in the rust doc.
Block Execution Flow

![Screenshot 2023-12-04 at 19 11 29](https://github.com/paritytech/polkadot-sdk/assets/10380170/e88a80c4-ef11-4faa-8df5-8b33a724c054)

## Inherent Order Moved to https://github.com/paritytech/polkadot-sdk/pull/2154 --------------- ## TODO - [ ] Check that `try-runtime` still works - [ ] Ensure backwards compatibility with old Runtime APIs - [x] Consume weight correctly - [x] Cleanup --------- Signed-off-by: Oliver Tale-Yazdi Co-authored-by: Liam Aharon Co-authored-by: Juan Girini Co-authored-by: command-bot <> Co-authored-by: Francisco Aguirre Co-authored-by: Gavin Wood Co-authored-by: Bastian Köcher --- Cargo.lock | 110 +- Cargo.toml | 1 + .../pallets/inbound-queue/src/mock.rs | 13 +- .../pallets/outbound-queue/src/mock.rs | 14 +- bridges/snowbridge/pallets/system/src/mock.rs | 15 +- .../ethereum-beacon-client/src/mock.rs | 259 +++ cumulus/parachain-template/runtime/src/lib.rs | 2 +- .../assets/asset-hub-rococo/src/lib.rs | 2 +- .../assets/asset-hub-westend/src/lib.rs | 2 +- .../bridge-hubs/bridge-hub-rococo/src/lib.rs | 2 +- .../bridge-hubs/bridge-hub-westend/src/lib.rs | 2 +- .../collectives-westend/src/lib.rs | 2 +- .../contracts/contracts-rococo/src/lib.rs | 2 +- .../coretime/coretime-rococo/src/lib.rs | 2 +- .../coretime/coretime-westend/src/lib.rs | 2 +- .../glutton/glutton-westend/src/lib.rs | 2 +- .../runtimes/people/people-rococo/src/lib.rs | 2 +- .../runtimes/people/people-westend/src/lib.rs | 2 +- .../runtimes/starters/seedling/src/lib.rs | 2 +- .../runtimes/starters/shell/src/lib.rs | 2 +- .../runtimes/testing/penpal/src/lib.rs | 2 +- .../testing/rococo-parachain/src/lib.rs | 2 +- .../asset_hub_polkadot_aura.rs | 2 +- .../src/fake_runtime_api/aura.rs | 2 +- cumulus/test/runtime/src/lib.rs | 2 +- polkadot/node/service/src/fake_runtime_api.rs | 2 +- polkadot/runtime/common/src/claims.rs | 29 +- polkadot/runtime/rococo/src/lib.rs | 2 +- polkadot/runtime/test-runtime/src/lib.rs | 2 +- polkadot/runtime/westend/src/lib.rs | 2 +- prdoc/pr_1781.prdoc | 45 + substrate/bin/minimal/runtime/src/lib.rs | 5 +- .../bin/node-template/runtime/src/lib.rs | 2 +- substrate/bin/node/runtime/Cargo.toml | 4 + substrate/bin/node/runtime/src/lib.rs | 24 +- .../basic-authorship/src/basic_authorship.rs | 13 +- substrate/client/block-builder/src/lib.rs | 26 +- substrate/client/proposer-metrics/src/lib.rs | 4 + .../rpc-spec-v2/src/chain_head/tests.rs | 6 +- substrate/client/rpc/src/state/tests.rs | 2 +- substrate/frame/executive/Cargo.toml | 2 + substrate/frame/executive/src/lib.rs | 1194 +++----------- substrate/frame/executive/src/tests.rs | 1389 +++++++++++++++++ substrate/frame/migrations/Cargo.toml | 64 + .../frame/migrations/src/benchmarking.rs | 222 +++ substrate/frame/migrations/src/lib.rs | 746 +++++++++ substrate/frame/migrations/src/mock.rs | 163 ++ .../frame/migrations/src/mock_helpers.rs | 142 ++ substrate/frame/migrations/src/tests.rs | 335 ++++ substrate/frame/migrations/src/weights.rs | 358 +++++ substrate/frame/src/lib.rs | 2 +- .../src/construct_runtime/expand/inherent.rs | 63 +- .../procedural/src/pallet/expand/hooks.rs | 16 + substrate/frame/support/src/migrations.rs | 636 +++++++- .../support/src/storage/transactional.rs | 16 + substrate/frame/support/src/traits.rs | 11 +- substrate/frame/support/src/traits/hooks.rs | 70 + substrate/frame/support/src/traits/misc.rs | 17 +- .../undefined_inherent_part.stderr | 23 +- .../test/tests/pallet_outer_enums_explicit.rs | 1 + .../test/tests/pallet_outer_enums_implicit.rs | 1 + .../support/test/tests/runtime_metadata.rs | 6 +- substrate/frame/system/src/lib.rs | 74 +- substrate/frame/system/src/mock.rs | 16 + substrate/frame/system/src/tests.rs | 22 + .../api/proc-macro/src/decl_runtime_apis.rs | 1 + .../proc-macro/src/mock_impl_runtime_apis.rs | 2 +- substrate/primitives/api/src/lib.rs | 18 +- .../api/test/tests/decl_and_impl.rs | 2 +- .../ui/impl_incorrect_method_signature.rs | 2 +- .../api/test/tests/ui/impl_missing_version.rs | 2 +- .../test/tests/ui/missing_versioned_method.rs | 2 +- .../missing_versioned_method_multiple_vers.rs | 2 +- .../ui/positive_cases/custom_where_bound.rs | 2 +- .../tests/ui/positive_cases/default_impls.rs | 2 +- ...ype_reference_in_impl_runtime_apis_call.rs | 2 +- substrate/primitives/runtime/src/lib.rs | 10 + substrate/test-utils/runtime/src/lib.rs | 6 +- 78 files changed, 5072 insertions(+), 1188 deletions(-) create mode 100644 bridges/snowbridge/parachain/pallets/ethereum-beacon-client/src/mock.rs create mode 100644 prdoc/pr_1781.prdoc create mode 100644 substrate/frame/executive/src/tests.rs create mode 100644 substrate/frame/migrations/Cargo.toml create mode 100644 substrate/frame/migrations/src/benchmarking.rs create mode 100644 substrate/frame/migrations/src/lib.rs create mode 100644 substrate/frame/migrations/src/mock.rs create mode 100644 substrate/frame/migrations/src/mock_helpers.rs create mode 100644 substrate/frame/migrations/src/tests.rs create mode 100644 substrate/frame/migrations/src/weights.rs diff --git a/Cargo.lock b/Cargo.lock index 197d05e26339..3838ed0e94a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -322,6 +322,20 @@ dependencies = [ "num-traits", ] +[[package]] +name = "aquamarine" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1da02abba9f9063d786eab1509833ebb2fac0f966862ca59439c76b9c566760" +dependencies = [ + "include_dir", + "itertools 0.10.5", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "aquamarine" version = "0.5.0" @@ -4055,7 +4069,7 @@ dependencies = [ "cumulus-primitives-core", "cumulus-primitives-proof-size-hostfunction", "cumulus-test-runtime", - "docify", + "docify 0.2.7", "frame-support", "frame-system", "log", @@ -4736,13 +4750,39 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "docify" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1b04e6ef3d21119d3eb7b032bca17f99fe041e9c072f30f32cc0e1a2b1f3c4" +dependencies = [ + "docify_macros 0.1.16", +] + [[package]] name = "docify" version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cc4fd38aaa9fb98ac70794c82a00360d1e165a87fbf96a8a91f9dfc602aaee2" dependencies = [ - "docify_macros", + "docify_macros 0.2.7", +] + +[[package]] +name = "docify_macros" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5610df7f2acf89a1bb5d1a66ae56b1c7fcdcfe3948856fb3ace3f644d70eb7" +dependencies = [ + "common-path", + "derive-syn-parse", + "lazy_static", + "proc-macro2", + "quote", + "regex", + "syn 2.0.50", + "termcolor", + "walkdir", ] [[package]] @@ -5451,7 +5491,7 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" name = "frame" version = "0.0.1-dev" dependencies = [ - "docify", + "docify 0.2.7", "frame-executive", "frame-support", "frame-system", @@ -5619,6 +5659,7 @@ dependencies = [ name = "frame-executive" version = "28.0.0" dependencies = [ + "aquamarine 0.3.3", "array-bytes 6.1.0", "frame-support", "frame-system", @@ -5675,11 +5716,11 @@ dependencies = [ name = "frame-support" version = "28.0.0" dependencies = [ - "aquamarine", + "aquamarine 0.5.0", "array-bytes 6.1.0", "assert_matches", "bitflags 1.3.2", - "docify", + "docify 0.2.7", "environmental", "frame-metadata", "frame-support-procedural", @@ -5822,7 +5863,7 @@ version = "28.0.0" dependencies = [ "cfg-if", "criterion 0.4.0", - "docify", + "docify 0.2.7", "frame-support", "log", "parity-scale-codec", @@ -7081,6 +7122,7 @@ dependencies = [ "pallet-lottery", "pallet-membership", "pallet-message-queue", + "pallet-migrations", "pallet-mixnet", "pallet-mmr", "pallet-multisig", @@ -9301,8 +9343,8 @@ dependencies = [ name = "pallet-bags-list" version = "27.0.0" dependencies = [ - "aquamarine", - "docify", + "aquamarine 0.5.0", + "docify 0.2.7", "frame-benchmarking", "frame-election-provider-support", "frame-support", @@ -9350,7 +9392,7 @@ dependencies = [ name = "pallet-balances" version = "28.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -9969,7 +10011,7 @@ dependencies = [ name = "pallet-example-single-block-migrations" version = "0.0.1" dependencies = [ - "docify", + "docify 0.2.7", "frame-executive", "frame-support", "frame-system", @@ -10035,7 +10077,7 @@ dependencies = [ name = "pallet-fast-unstake" version = "27.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-election-provider-support", "frame-support", @@ -10232,6 +10274,30 @@ dependencies = [ "sp-weights", ] +[[package]] +name = "pallet-migrations" +version = "1.0.0" +dependencies = [ + "docify 0.1.16", + "frame-benchmarking", + "frame-executive", + "frame-support", + "frame-system", + "impl-trait-for-tuples", + "log", + "parity-scale-codec", + "pretty_assertions", + "scale-info", + "sp-api", + "sp-block-builder", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std 14.0.0", + "sp-tracing 16.0.0", + "sp-version", +] + [[package]] name = "pallet-mixnet" version = "0.4.0" @@ -10507,7 +10573,7 @@ dependencies = [ name = "pallet-paged-list" version = "0.6.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -10550,7 +10616,7 @@ dependencies = [ name = "pallet-parameters" version = "0.0.1" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -10711,7 +10777,7 @@ dependencies = [ name = "pallet-safe-mode" version = "9.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -10768,7 +10834,7 @@ dependencies = [ name = "pallet-scheduler" version = "29.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -10981,7 +11047,7 @@ dependencies = [ name = "pallet-sudo" version = "28.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -11012,7 +11078,7 @@ dependencies = [ name = "pallet-timestamp" version = "27.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -11116,7 +11182,7 @@ dependencies = [ name = "pallet-treasury" version = "27.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -11136,7 +11202,7 @@ dependencies = [ name = "pallet-tx-pause" version = "9.0.0" dependencies = [ - "docify", + "docify 0.2.7", "frame-benchmarking", "frame-support", "frame-system", @@ -13404,7 +13470,7 @@ version = "0.0.1" dependencies = [ "cumulus-pallet-aura-ext", "cumulus-pallet-parachain-system", - "docify", + "docify 0.2.7", "frame", "frame-executive", "frame-support", @@ -15665,7 +15731,7 @@ name = "sc-chain-spec" version = "27.0.0" dependencies = [ "array-bytes 6.1.0", - "docify", + "docify 0.2.7", "log", "memmap2 0.9.3", "parity-scale-codec", @@ -18931,7 +18997,7 @@ dependencies = [ name = "sp-runtime" version = "31.0.1" dependencies = [ - "docify", + "docify 0.2.7", "either", "hash256-std-hasher", "impl-trait-for-tuples", diff --git a/Cargo.toml b/Cargo.toml index 654367cc1e21..48aa25f5c5a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -352,6 +352,7 @@ members = [ "substrate/frame/membership", "substrate/frame/merkle-mountain-range", "substrate/frame/message-queue", + "substrate/frame/migrations", "substrate/frame/mixnet", "substrate/frame/multisig", "substrate/frame/nft-fractionalization", diff --git a/bridges/snowbridge/pallets/inbound-queue/src/mock.rs b/bridges/snowbridge/pallets/inbound-queue/src/mock.rs index 110f611c6766..749fb0367f33 100644 --- a/bridges/snowbridge/pallets/inbound-queue/src/mock.rs +++ b/bridges/snowbridge/pallets/inbound-queue/src/mock.rs @@ -3,7 +3,7 @@ use super::*; use frame_support::{ - parameter_types, + derive_impl, parameter_types, traits::{ConstU128, ConstU32, Everything}, weights::IdentityFee, }; @@ -47,10 +47,9 @@ parameter_types! { type Balance = u128; +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] impl frame_system::Config for Test { type BaseCallFilter = Everything; - type BlockWeights = (); - type BlockLength = (); type RuntimeOrigin = RuntimeOrigin; type RuntimeCall = RuntimeCall; type RuntimeTask = RuntimeTask; @@ -60,16 +59,8 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type RuntimeEvent = RuntimeEvent; type BlockHashCount = BlockHashCount; - type DbWeight = (); - type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - type OnSetCode = (); - type MaxConsumers = frame_support::traits::ConstU32<16>; type Nonce = u64; type Block = Block; } diff --git a/bridges/snowbridge/pallets/outbound-queue/src/mock.rs b/bridges/snowbridge/pallets/outbound-queue/src/mock.rs index dd8fee4e2ed0..6e78fb446721 100644 --- a/bridges/snowbridge/pallets/outbound-queue/src/mock.rs +++ b/bridges/snowbridge/pallets/outbound-queue/src/mock.rs @@ -3,7 +3,7 @@ use super::*; use frame_support::{ - parameter_types, + derive_impl, parameter_types, traits::{Everything, Hooks}, weights::IdentityFee, }; @@ -37,10 +37,9 @@ parameter_types! { pub const BlockHashCount: u64 = 250; } +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] impl frame_system::Config for Test { type BaseCallFilter = Everything; - type BlockWeights = (); - type BlockLength = (); type RuntimeOrigin = RuntimeOrigin; type RuntimeCall = RuntimeCall; type RuntimeTask = RuntimeTask; @@ -50,16 +49,7 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type RuntimeEvent = RuntimeEvent; type BlockHashCount = BlockHashCount; - type DbWeight = (); - type Version = (); type PalletInfo = PalletInfo; - type AccountData = (); - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - type OnSetCode = (); - type MaxConsumers = frame_support::traits::ConstU32<16>; type Nonce = u64; type Block = Block; } diff --git a/bridges/snowbridge/pallets/system/src/mock.rs b/bridges/snowbridge/pallets/system/src/mock.rs index edc3f141b073..de2970dd550b 100644 --- a/bridges/snowbridge/pallets/system/src/mock.rs +++ b/bridges/snowbridge/pallets/system/src/mock.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: 2023 Snowfork use crate as snowbridge_system; use frame_support::{ - parameter_types, - traits::{tokens::fungible::Mutate, ConstU128, ConstU16, ConstU64, ConstU8}, + derive_impl, parameter_types, + traits::{tokens::fungible::Mutate, ConstU128, ConstU64, ConstU8}, weights::IdentityFee, PalletId, }; @@ -95,11 +95,9 @@ frame_support::construct_runtime!( } ); +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; - type BlockWeights = (); - type BlockLength = (); - type DbWeight = (); type RuntimeOrigin = RuntimeOrigin; type RuntimeCall = RuntimeCall; type RuntimeTask = RuntimeTask; @@ -109,15 +107,8 @@ impl frame_system::Config for Test { type Lookup = IdentityLookup; type RuntimeEvent = RuntimeEvent; type BlockHashCount = ConstU64<250>; - type Version = (); type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = ConstU16<42>; - type OnSetCode = (); - type MaxConsumers = frame_support::traits::ConstU32<16>; type Nonce = u64; type Block = Block; } diff --git a/bridges/snowbridge/parachain/pallets/ethereum-beacon-client/src/mock.rs b/bridges/snowbridge/parachain/pallets/ethereum-beacon-client/src/mock.rs new file mode 100644 index 000000000000..77b5c1aa631d --- /dev/null +++ b/bridges/snowbridge/parachain/pallets/ethereum-beacon-client/src/mock.rs @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2023 Snowfork +use crate as ethereum_beacon_client; +use frame_support::parameter_types; +use pallet_timestamp; +use primitives::{Fork, ForkVersions}; +use sp_core::H256; +use sp_runtime::traits::{BlakeTwo256, IdentityLookup}; + +#[cfg(not(feature = "beacon-spec-mainnet"))] +pub mod minimal { + use super::*; + + use crate::config; + use frame_support::derive_impl; + use hex_literal::hex; + use primitives::CompactExecutionHeader; + use snowbridge_core::inbound::{Log, Proof}; + use sp_runtime::BuildStorage; + use std::{fs::File, path::PathBuf}; + + type Block = frame_system::mocking::MockBlock; + + frame_support::construct_runtime!( + pub enum Test { + System: frame_system::{Pallet, Call, Storage, Event}, + Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent}, + EthereumBeaconClient: ethereum_beacon_client::{Pallet, Call, Storage, Event}, + } + ); + + parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const SS58Prefix: u8 = 42; + } + + #[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] + impl frame_system::Config for Test { + type BaseCallFilter = frame_support::traits::Everything; + type RuntimeOrigin = RuntimeOrigin; + type RuntimeCall = RuntimeCall; + type RuntimeTask = RuntimeTask; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type RuntimeEvent = RuntimeEvent; + type BlockHashCount = BlockHashCount; + type PalletInfo = PalletInfo; + type SS58Prefix = SS58Prefix; + type Nonce = u64; + type Block = Block; + } + + impl pallet_timestamp::Config for Test { + type Moment = u64; + type OnTimestampSet = (); + type MinimumPeriod = (); + type WeightInfo = (); + } + + parameter_types! { + pub const ExecutionHeadersPruneThreshold: u32 = 10; + pub const ChainForkVersions: ForkVersions = ForkVersions{ + genesis: Fork { + version: [0, 0, 0, 1], // 0x00000001 + epoch: 0, + }, + altair: Fork { + version: [1, 0, 0, 1], // 0x01000001 + epoch: 0, + }, + bellatrix: Fork { + version: [2, 0, 0, 1], // 0x02000001 + epoch: 0, + }, + capella: Fork { + version: [3, 0, 0, 1], // 0x03000001 + epoch: 0, + }, + }; + } + + impl ethereum_beacon_client::Config for Test { + type RuntimeEvent = RuntimeEvent; + type ForkVersions = ChainForkVersions; + type MaxExecutionHeadersToKeep = ExecutionHeadersPruneThreshold; + type WeightInfo = (); + } + + // Build genesis storage according to the mock runtime. + pub fn new_tester() -> sp_io::TestExternalities { + let t = frame_system::GenesisConfig::::default().build_storage().unwrap(); + let mut ext = sp_io::TestExternalities::new(t); + let _ = ext.execute_with(|| Timestamp::set(RuntimeOrigin::signed(1), 30_000)); + ext + } + + fn load_fixture(basename: &str) -> Result + where + T: for<'de> serde::Deserialize<'de>, + { + let filepath: PathBuf = + [env!("CARGO_MANIFEST_DIR"), "tests", "fixtures", basename].iter().collect(); + serde_json::from_reader(File::open(filepath).unwrap()) + } + + pub fn load_execution_header_update_fixture() -> primitives::ExecutionHeaderUpdate { + load_fixture("execution-header-update.minimal.json").unwrap() + } + + pub fn load_checkpoint_update_fixture( + ) -> primitives::CheckpointUpdate<{ config::SYNC_COMMITTEE_SIZE }> { + load_fixture("initial-checkpoint.minimal.json").unwrap() + } + + pub fn load_sync_committee_update_fixture( + ) -> primitives::Update<{ config::SYNC_COMMITTEE_SIZE }, { config::SYNC_COMMITTEE_BITS_SIZE }> { + load_fixture("sync-committee-update.minimal.json").unwrap() + } + + pub fn load_finalized_header_update_fixture( + ) -> primitives::Update<{ config::SYNC_COMMITTEE_SIZE }, { config::SYNC_COMMITTEE_BITS_SIZE }> { + load_fixture("finalized-header-update.minimal.json").unwrap() + } + + pub fn load_next_sync_committee_update_fixture( + ) -> primitives::Update<{ config::SYNC_COMMITTEE_SIZE }, { config::SYNC_COMMITTEE_BITS_SIZE }> { + load_fixture("next-sync-committee-update.minimal.json").unwrap() + } + + pub fn load_next_finalized_header_update_fixture( + ) -> primitives::Update<{ config::SYNC_COMMITTEE_SIZE }, { config::SYNC_COMMITTEE_BITS_SIZE }> { + load_fixture("next-finalized-header-update.minimal.json").unwrap() + } + + pub fn get_message_verification_payload() -> (Log, Proof) { + ( + Log { + address: hex!("ee9170abfbf9421ad6dd07f6bdec9d89f2b581e0").into(), + topics: vec![ + hex!("1b11dcf133cc240f682dab2d3a8e4cd35c5da8c9cf99adac4336f8512584c5ad").into(), + hex!("00000000000000000000000000000000000000000000000000000000000003e8").into(), + hex!("0000000000000000000000000000000000000000000000000000000000000001").into(), + ], + data: hex!("0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000004b000f000000000000000100d184c103f7acc340847eee82a0b909e3358bc28d440edffa1352b13227e8ee646f3ea37456dec701345772617070656420457468657210574554481235003511000000000000000000000000000000000000000000").into(), + }, + Proof { + block_hash: hex!("05aaa60b0f27cce9e71909508527264b77ee14da7b5bf915fcc4e32715333213").into(), + tx_index: 0, + data: (vec![ + hex!("cf0d1c1ba57d1e0edfb59786c7e30c2b7e12bd54612b00cd21c4eaeecedf44fb").to_vec(), + hex!("d21fc4f68ab05bc4dcb23c67008e92c4d466437cdd6ed7aad0c008944c185510").to_vec(), + hex!("b9890f91ca0d77aa2a4adfaf9b9e40c94cac9e638b6d9797923865872944b646").to_vec(), + ], vec![ + hex!("f90131a0b601337b3aa10a671caa724eba641e759399979856141d3aea6b6b4ac59b889ba00c7d5dd48be9060221a02fb8fa213860b4c50d47046c8fa65ffaba5737d569e0a094601b62a1086cd9c9cb71a7ebff9e718f3217fd6e837efe4246733c0a196f63a06a4b0dd0aefc37b3c77828c8f07d1b7a2455ceb5dbfd3c77d7d6aeeddc2f7e8ca0d6e8e23142cdd8ec219e1f5d8b56aa18e456702b195deeaa210327284d42ade4a08a313d4c87023005d1ab631bbfe3f5de1e405d0e66d0bef3e033f1e5711b5521a0bf09a5d9a48b10ade82b8d6a5362a15921c8b5228a3487479b467db97411d82fa0f95cccae2a7c572ef3c566503e30bac2b2feb2d2f26eebf6d870dcf7f8cf59cea0d21fc4f68ab05bc4dcb23c67008e92c4d466437cdd6ed7aad0c008944c1855108080808080808080").to_vec(), + hex!("f851a0b9890f91ca0d77aa2a4adfaf9b9e40c94cac9e638b6d9797923865872944b646a060a634b9280e3a23fb63375e7bbdd9ab07fd379ab6a67e2312bbc112195fa358808080808080808080808080808080").to_vec(), + hex!("f9030820b9030402f90300018301d6e2b9010000000000000800000000000020040008000000000000000000000000400000008000000000000000000000000000000000000000000000000000000000042010000000001000000000000000000000000000000000040000000000000000000000000000000000000000000000008000000000000000002000000000000000000000000200000000000000200000000000100000000040000001000200008000000000000200000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000f901f5f87a942ffa5ecdbe006d30397c7636d3e015eee251369ff842a0c965575a00553e094ca7c5d14f02e107c258dda06867cbf9e0e69f80e71bbcc1a000000000000000000000000000000000000000000000000000000000000003e8a000000000000000000000000000000000000000000000000000000000000003e8f9011c94ee9170abfbf9421ad6dd07f6bdec9d89f2b581e0f863a01b11dcf133cc240f682dab2d3a8e4cd35c5da8c9cf99adac4336f8512584c5ada000000000000000000000000000000000000000000000000000000000000003e8a00000000000000000000000000000000000000000000000000000000000000001b8a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000004b000f000000000000000100d184c103f7acc340847eee82a0b909e3358bc28d440edffa1352b13227e8ee646f3ea37456dec701345772617070656420457468657210574554481235003511000000000000000000000000000000000000000000f858948cf6147918a5cbb672703f879f385036f8793a24e1a01449abf21e49fd025f33495e77f7b1461caefdd3d4bb646424a3f445c4576a5ba0000000000000000000000000440edffa1352b13227e8ee646f3ea37456dec701").to_vec(), + ]), + } + ) + } + + pub fn get_message_verification_header() -> CompactExecutionHeader { + CompactExecutionHeader { + parent_hash: hex!("04a7f6ab8282203562c62f38b0ab41d32aaebe2c7ea687702b463148a6429e04") + .into(), + block_number: 55, + state_root: hex!("894d968712976d613519f973a317cb0781c7b039c89f27ea2b7ca193f7befdb3") + .into(), + receipts_root: hex!("cf0d1c1ba57d1e0edfb59786c7e30c2b7e12bd54612b00cd21c4eaeecedf44fb") + .into(), + } + } +} + +#[cfg(feature = "beacon-spec-mainnet")] +pub mod mainnet { + use super::*; + use frame_support::derive_impl; + + type Block = frame_system::mocking::MockBlock; + use sp_runtime::BuildStorage; + + frame_support::construct_runtime!( + pub enum Test { + System: frame_system::{Pallet, Call, Storage, Event}, + Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent}, + EthereumBeaconClient: ethereum_beacon_client::{Pallet, Call, Storage, Event}, + } + ); + + parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const SS58Prefix: u8 = 42; + } + + #[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] + impl frame_system::Config for Test { + type BaseCallFilter = frame_support::traits::Everything; + type RuntimeOrigin = RuntimeOrigin; + type RuntimeCall = RuntimeCall; + type RuntimeTask = RuntimeTask; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type RuntimeEvent = RuntimeEvent; + type BlockHashCount = BlockHashCount; + type PalletInfo = PalletInfo; + type SS58Prefix = SS58Prefix; + type Nonce = u64; + type Block = Block; + } + + impl pallet_timestamp::Config for Test { + type Moment = u64; + type OnTimestampSet = (); + type MinimumPeriod = (); + type WeightInfo = (); + } + + parameter_types! { + pub const ChainForkVersions: ForkVersions = ForkVersions{ + genesis: Fork { + version: [0, 0, 16, 32], // 0x00001020 + epoch: 0, + }, + altair: Fork { + version: [1, 0, 16, 32], // 0x01001020 + epoch: 36660, + }, + bellatrix: Fork { + version: [2, 0, 16, 32], // 0x02001020 + epoch: 112260, + }, + capella: Fork { + version: [3, 0, 16, 32], // 0x03001020 + epoch: 162304, + }, + }; + pub const ExecutionHeadersPruneThreshold: u32 = 10; + } + + impl ethereum_beacon_client::Config for Test { + type RuntimeEvent = RuntimeEvent; + type ForkVersions = ChainForkVersions; + type MaxExecutionHeadersToKeep = ExecutionHeadersPruneThreshold; + type WeightInfo = (); + } + + // Build genesis storage according to the mock runtime. + pub fn new_tester() -> sp_io::TestExternalities { + let t = frame_system::GenesisConfig::::default().build_storage().unwrap(); + let mut ext = sp_io::TestExternalities::new(t); + let _ = ext.execute_with(|| Timestamp::set(RuntimeOrigin::signed(1), 30_000)); + ext + } +} diff --git a/cumulus/parachain-template/runtime/src/lib.rs b/cumulus/parachain-template/runtime/src/lib.rs index cee9b33bf37c..004a5d70ebc9 100644 --- a/cumulus/parachain-template/runtime/src/lib.rs +++ b/cumulus/parachain-template/runtime/src/lib.rs @@ -560,7 +560,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs index 6791dc4064ed..1c4f49fea0ed 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs @@ -1093,7 +1093,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs index 5352eeb1a1b7..e015c7bfca35 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs @@ -1124,7 +1124,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs index db1dfbd45d63..0f6af1b51f1e 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs @@ -816,7 +816,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs index 1c8c9aa20755..206743ba6aca 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs @@ -566,7 +566,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/collectives/collectives-westend/src/lib.rs b/cumulus/parachains/runtimes/collectives/collectives-westend/src/lib.rs index 36eb0e07f87c..3ed9d6e8e168 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-westend/src/lib.rs @@ -803,7 +803,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs index 541978098caa..c4df5b9a5652 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs @@ -466,7 +466,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/coretime/coretime-rococo/src/lib.rs b/cumulus/parachains/runtimes/coretime/coretime-rococo/src/lib.rs index fcd7576b1e17..0cdfebce742d 100644 --- a/cumulus/parachains/runtimes/coretime/coretime-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/coretime/coretime-rococo/src/lib.rs @@ -521,7 +521,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/coretime/coretime-westend/src/lib.rs b/cumulus/parachains/runtimes/coretime/coretime-westend/src/lib.rs index 80eb7863803b..7797329b5266 100644 --- a/cumulus/parachains/runtimes/coretime/coretime-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/coretime/coretime-westend/src/lib.rs @@ -512,7 +512,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/glutton/glutton-westend/src/lib.rs b/cumulus/parachains/runtimes/glutton/glutton-westend/src/lib.rs index 10408aaf39a7..e09ad42ea9f2 100644 --- a/cumulus/parachains/runtimes/glutton/glutton-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/glutton/glutton-westend/src/lib.rs @@ -332,7 +332,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/people/people-rococo/src/lib.rs b/cumulus/parachains/runtimes/people/people-rococo/src/lib.rs index 90c398917695..616da5519a5b 100644 --- a/cumulus/parachains/runtimes/people/people-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/people/people-rococo/src/lib.rs @@ -495,7 +495,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/people/people-westend/src/lib.rs b/cumulus/parachains/runtimes/people/people-westend/src/lib.rs index a904f7c3521b..9f572330cac8 100644 --- a/cumulus/parachains/runtimes/people/people-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/people/people-westend/src/lib.rs @@ -495,7 +495,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/starters/seedling/src/lib.rs b/cumulus/parachains/runtimes/starters/seedling/src/lib.rs index ba077ef88794..4cc0a81ef49a 100644 --- a/cumulus/parachains/runtimes/starters/seedling/src/lib.rs +++ b/cumulus/parachains/runtimes/starters/seedling/src/lib.rs @@ -300,7 +300,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/starters/shell/src/lib.rs b/cumulus/parachains/runtimes/starters/shell/src/lib.rs index 457394760d98..829754731a42 100644 --- a/cumulus/parachains/runtimes/starters/shell/src/lib.rs +++ b/cumulus/parachains/runtimes/starters/shell/src/lib.rs @@ -357,7 +357,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/testing/penpal/src/lib.rs b/cumulus/parachains/runtimes/testing/penpal/src/lib.rs index bf8dcbc24c8d..0030287edb3b 100644 --- a/cumulus/parachains/runtimes/testing/penpal/src/lib.rs +++ b/cumulus/parachains/runtimes/testing/penpal/src/lib.rs @@ -699,7 +699,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs b/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs index 57969d9a4f18..b69c2341d6b0 100644 --- a/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs +++ b/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs @@ -689,7 +689,7 @@ impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/cumulus/polkadot-parachain/src/fake_runtime_api/asset_hub_polkadot_aura.rs b/cumulus/polkadot-parachain/src/fake_runtime_api/asset_hub_polkadot_aura.rs index 76dd7347ccbc..7778d1bf7d2d 100644 --- a/cumulus/polkadot-parachain/src/fake_runtime_api/asset_hub_polkadot_aura.rs +++ b/cumulus/polkadot-parachain/src/fake_runtime_api/asset_hub_polkadot_aura.rs @@ -39,7 +39,7 @@ sp_api::impl_runtime_apis! { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/cumulus/polkadot-parachain/src/fake_runtime_api/aura.rs b/cumulus/polkadot-parachain/src/fake_runtime_api/aura.rs index 0f01b85ebcf6..880f5d760c74 100644 --- a/cumulus/polkadot-parachain/src/fake_runtime_api/aura.rs +++ b/cumulus/polkadot-parachain/src/fake_runtime_api/aura.rs @@ -39,7 +39,7 @@ sp_api::impl_runtime_apis! { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/cumulus/test/runtime/src/lib.rs b/cumulus/test/runtime/src/lib.rs index 5fb314109844..5ccec8983e91 100644 --- a/cumulus/test/runtime/src/lib.rs +++ b/cumulus/test/runtime/src/lib.rs @@ -377,7 +377,7 @@ impl_runtime_apis! { Executive::execute_block(block) } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/polkadot/node/service/src/fake_runtime_api.rs b/polkadot/node/service/src/fake_runtime_api.rs index ccc3da22400d..085ea93fdc78 100644 --- a/polkadot/node/service/src/fake_runtime_api.rs +++ b/polkadot/node/service/src/fake_runtime_api.rs @@ -60,7 +60,7 @@ sp_api::impl_runtime_apis! { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/polkadot/runtime/common/src/claims.rs b/polkadot/runtime/common/src/claims.rs index 86550ea8b4eb..68f42914e447 100644 --- a/polkadot/runtime/common/src/claims.rs +++ b/polkadot/runtime/common/src/claims.rs @@ -702,7 +702,6 @@ mod tests { use secp_utils::*; use parity_scale_codec::Encode; - use sp_core::H256; // The testing primitives are very useful for avoiding having to work with signatures // or public keys. `u64` is used as the `AccountId` and no `Signature`s are required. use crate::claims; @@ -715,11 +714,8 @@ mod tests { }; use pallet_balances; use sp_runtime::{ - traits::{BlakeTwo256, Identity, IdentityLookup}, - transaction_validity::TransactionLongevity, - BuildStorage, - DispatchError::BadOrigin, - TokenError, + traits::Identity, transaction_validity::TransactionLongevity, BuildStorage, + DispatchError::BadOrigin, TokenError, }; type Block = frame_system::mocking::MockBlock; @@ -734,34 +730,13 @@ mod tests { } ); - parameter_types! { - pub const BlockHashCount: u32 = 250; - } - #[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] impl frame_system::Config for Test { - type BaseCallFilter = frame_support::traits::Everything; - type BlockWeights = (); - type BlockLength = (); - type DbWeight = (); type RuntimeOrigin = RuntimeOrigin; type RuntimeCall = RuntimeCall; - type Nonce = u64; - type Hash = H256; - type Hashing = BlakeTwo256; - type AccountId = u64; - type Lookup = IdentityLookup; type Block = Block; type RuntimeEvent = RuntimeEvent; - type BlockHashCount = BlockHashCount; - type Version = (); - type PalletInfo = PalletInfo; type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - type OnSetCode = (); type MaxConsumers = frame_support::traits::ConstU32<16>; } diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index bbc79a13d37f..173301e1ad91 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1768,7 +1768,7 @@ sp_api::impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs index b74def5de8aa..6c899c527015 100644 --- a/polkadot/runtime/test-runtime/src/lib.rs +++ b/polkadot/runtime/test-runtime/src/lib.rs @@ -767,7 +767,7 @@ sp_api::impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 574710b3110d..91047d953f54 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -1793,7 +1793,7 @@ sp_api::impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/prdoc/pr_1781.prdoc b/prdoc/pr_1781.prdoc new file mode 100644 index 000000000000..e3560842d15a --- /dev/null +++ b/prdoc/pr_1781.prdoc @@ -0,0 +1,45 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: "Multi-Block-Migrations, `poll` hook and new System Callbacks" + +doc: + - audience: Runtime Dev + description: | + The major things that this MR touches are: + + **Multi-Block-Migrations**: `pallet-migrations` is introduced that can be configured in the + `System` of a runtime to act as multi-block migrator. The `migrations` pallet then in turn + receives the list of MBMs as config parameter. The list of migrations can be an aggregated + tuple of `SteppedMigration` trait implementation. + It is paramount that the `migrations` pallet is configured in `System` once it is deployed. A + test is in place to double check this. + + To integrate this into your runtime, it is only necessary to change the return type of + `initialize_block` to `RuntimeExecutiveMode`. For extended info please see + https://github.com/paritytech/polkadot-sdk/pull/1781. + + **poll**: a new pallet hook named `poll` is added. This can be used for places where the code + that should be executed is not deadline critical. Runtime devs are advised to skim their usage + of `on_initialize` and `on_finalize` to see whether they can be replace with `poll`. `poll` is + not guaranteed to be called each block. In fact it will not be called when MBMs are ongoing. + + **System Callbacks**: The `system` pallet gets five new config items - all of which can be + safely set to `()` as default. They are: + - `SingleBlockMigrations`: replaces the `Executive` now for configuring migrations. + - `MultiBlockMigrator`: the `pallet-migrations` would be set here, if deployed. + - `PreInherents`: a hook that runs before any inherent. + - `PostInherents`: a hook to run between inherents and `poll`/MBM logic. + - `PostTransactions`: a hook to run after all transactions but before `on_idle`. + +crates: + - name: frame-executive + - name: frame-system + - name: frame-support + - name: frame-support-procedural + - name: pallet-migrations + - name: sc-basic-authorship + - name: sc-block-builder + - name: sp-api + - name: sp-api-proc-macro + - name: sp-runtime diff --git a/substrate/bin/minimal/runtime/src/lib.rs b/substrate/bin/minimal/runtime/src/lib.rs index 610289693d91..fb996aef46b8 100644 --- a/substrate/bin/minimal/runtime/src/lib.rs +++ b/substrate/bin/minimal/runtime/src/lib.rs @@ -26,7 +26,8 @@ use frame::{ prelude::*, runtime::{ apis::{ - self, impl_runtime_apis, ApplyExtrinsicResult, CheckInherentsResult, OpaqueMetadata, + self, impl_runtime_apis, ApplyExtrinsicResult, CheckInherentsResult, + ExtrinsicInclusionMode, OpaqueMetadata, }, prelude::*, }, @@ -121,7 +122,7 @@ impl_runtime_apis! { RuntimeExecutive::execute_block(block) } - fn initialize_block(header: &Header) { + fn initialize_block(header: &Header) -> ExtrinsicInclusionMode { RuntimeExecutive::initialize_block(header) } } diff --git a/substrate/bin/node-template/runtime/src/lib.rs b/substrate/bin/node-template/runtime/src/lib.rs index 3b6a74be2512..159697f427f6 100644 --- a/substrate/bin/node-template/runtime/src/lib.rs +++ b/substrate/bin/node-template/runtime/src/lib.rs @@ -331,7 +331,7 @@ impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/substrate/bin/node/runtime/Cargo.toml b/substrate/bin/node/runtime/Cargo.toml index aa13c3d292be..9607d05daf0e 100644 --- a/substrate/bin/node/runtime/Cargo.toml +++ b/substrate/bin/node/runtime/Cargo.toml @@ -88,6 +88,7 @@ pallet-election-provider-support-benchmarking = { path = "../../../frame/electio pallet-elections-phragmen = { path = "../../../frame/elections-phragmen", default-features = false } pallet-example-tasks = { path = "../../../frame/examples/tasks", default-features = false } pallet-fast-unstake = { path = "../../../frame/fast-unstake", default-features = false } +pallet-migrations = { path = "../../../frame/migrations", default-features = false } pallet-nis = { path = "../../../frame/nis", default-features = false } pallet-grandpa = { path = "../../../frame/grandpa", default-features = false } pallet-im-online = { path = "../../../frame/im-online", default-features = false } @@ -198,6 +199,7 @@ std = [ "pallet-lottery/std", "pallet-membership/std", "pallet-message-queue/std", + "pallet-migrations/std", "pallet-mixnet/std", "pallet-mmr/std", "pallet-multisig/std", @@ -302,6 +304,7 @@ runtime-benchmarks = [ "pallet-lottery/runtime-benchmarks", "pallet-membership/runtime-benchmarks", "pallet-message-queue/runtime-benchmarks", + "pallet-migrations/runtime-benchmarks", "pallet-mixnet/runtime-benchmarks", "pallet-mmr/runtime-benchmarks", "pallet-multisig/runtime-benchmarks", @@ -381,6 +384,7 @@ try-runtime = [ "pallet-lottery/try-runtime", "pallet-membership/try-runtime", "pallet-message-queue/try-runtime", + "pallet-migrations/try-runtime", "pallet-mixnet/try-runtime", "pallet-mmr/try-runtime", "pallet-multisig/try-runtime", diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs index b34d8c89e568..5431628c747d 100644 --- a/substrate/bin/node/runtime/src/lib.rs +++ b/substrate/bin/node/runtime/src/lib.rs @@ -310,6 +310,7 @@ impl frame_system::Config for Runtime { type SystemWeightInfo = frame_system::weights::SubstrateWeight; type SS58Prefix = ConstU16<42>; type MaxConsumers = ConstU32<16>; + type MultiBlockMigrator = MultiBlockMigrations; } impl pallet_insecure_randomness_collective_flip::Config for Runtime {} @@ -2007,6 +2008,25 @@ impl pallet_statement::Config for Runtime { type MaxAllowedBytes = MaxAllowedBytes; } +parameter_types! { + pub MbmServiceWeight: Weight = Perbill::from_percent(80) * RuntimeBlockWeights::get().max_block; +} + +impl pallet_migrations::Config for Runtime { + type RuntimeEvent = RuntimeEvent; + #[cfg(not(feature = "runtime-benchmarks"))] + type Migrations = (); + // Benchmarks need mocked migrations to guarantee that they succeed. + #[cfg(feature = "runtime-benchmarks")] + type Migrations = pallet_migrations::mock_helpers::MockedMigrations; + type CursorMaxLen = ConstU32<65_536>; + type IdentifierMaxLen = ConstU32<256>; + type MigrationStatusHandler = (); + type FailedMigrationHandler = frame_support::migrations::FreezeChainOnFailedMigration; + type MaxServiceWeight = MbmServiceWeight; + type WeightInfo = pallet_migrations::weights::SubstrateWeight; +} + parameter_types! { pub const BrokerPalletId: PalletId = PalletId(*b"py/broke"); } @@ -2241,6 +2261,7 @@ construct_runtime!( TxPause: pallet_tx_pause, SafeMode: pallet_safe_mode, Statement: pallet_statement, + MultiBlockMigrations: pallet_migrations, Broker: pallet_broker, TasksExample: pallet_example_tasks, Mixnet: pallet_mixnet, @@ -2372,6 +2393,7 @@ mod benches { [pallet_lottery, Lottery] [pallet_membership, TechnicalMembership] [pallet_message_queue, MessageQueue] + [pallet_migrations, MultiBlockMigrations] [pallet_mmr, Mmr] [pallet_multisig, Multisig] [pallet_nomination_pools, NominationPoolsBench::] @@ -2417,7 +2439,7 @@ impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> sp_runtime::ExtrinsicInclusionMode { Executive::initialize_block(header) } } diff --git a/substrate/client/basic-authorship/src/basic_authorship.rs b/substrate/client/basic-authorship/src/basic_authorship.rs index fdc3d4918de3..932287ac8657 100644 --- a/substrate/client/basic-authorship/src/basic_authorship.rs +++ b/substrate/client/basic-authorship/src/basic_authorship.rs @@ -38,7 +38,7 @@ use sp_core::traits::SpawnNamed; use sp_inherents::InherentData; use sp_runtime::{ traits::{BlakeTwo256, Block as BlockT, Hash as HashT, Header as HeaderT}, - Digest, Percent, SaturatedConversion, + Digest, ExtrinsicInclusionMode, Percent, SaturatedConversion, }; use std::{marker::PhantomData, pin::Pin, sync::Arc, time}; @@ -335,11 +335,12 @@ where self.apply_inherents(&mut block_builder, inherent_data)?; - // TODO call `after_inherents` and check if we should apply extrinsincs here - // - - let end_reason = - self.apply_extrinsics(&mut block_builder, deadline, block_size_limit).await?; + let mode = block_builder.extrinsic_inclusion_mode(); + let end_reason = match mode { + ExtrinsicInclusionMode::AllExtrinsics => + self.apply_extrinsics(&mut block_builder, deadline, block_size_limit).await?, + ExtrinsicInclusionMode::OnlyInherents => EndProposingReason::TransactionForbidden, + }; let (block, storage_changes, proof) = block_builder.build()?.into_inner(); let block_took = block_timer.elapsed(); diff --git a/substrate/client/block-builder/src/lib.rs b/substrate/client/block-builder/src/lib.rs index 258e39d962b2..2f22cd42591f 100644 --- a/substrate/client/block-builder/src/lib.rs +++ b/substrate/client/block-builder/src/lib.rs @@ -37,7 +37,7 @@ use sp_core::traits::CallContext; use sp_runtime::{ legacy, traits::{Block as BlockT, Hash, HashingFor, Header as HeaderT, NumberFor, One}, - Digest, + Digest, ExtrinsicInclusionMode, }; use std::marker::PhantomData; @@ -198,10 +198,12 @@ pub struct BlockBuilder<'a, Block: BlockT, C: ProvideRuntimeApi + 'a> { extrinsics: Vec, api: ApiRef<'a, C::Api>, call_api_at: &'a C, + /// Version of the [`BlockBuilderApi`] runtime API. version: u32, parent_hash: Block::Hash, /// The estimated size of the block header. estimated_header_size: usize, + extrinsic_inclusion_mode: ExtrinsicInclusionMode, } impl<'a, Block, C> BlockBuilder<'a, Block, C> @@ -244,9 +246,19 @@ where api.set_call_context(CallContext::Onchain); - api.initialize_block(parent_hash, &header)?; + let core_version = api + .api_version::>(parent_hash)? + .ok_or_else(|| Error::VersionInvalid("Core".to_string()))?; - let version = api + let extrinsic_inclusion_mode = if core_version >= 5 { + api.initialize_block(parent_hash, &header)? + } else { + #[allow(deprecated)] + api.initialize_block_before_version_5(parent_hash, &header)?; + ExtrinsicInclusionMode::AllExtrinsics + }; + + let bb_version = api .api_version::>(parent_hash)? .ok_or_else(|| Error::VersionInvalid("BlockBuilderApi".to_string()))?; @@ -254,12 +266,18 @@ where parent_hash, extrinsics: Vec::new(), api, - version, + version: bb_version, estimated_header_size, call_api_at, + extrinsic_inclusion_mode, }) } + /// The extrinsic inclusion mode of the runtime for this block. + pub fn extrinsic_inclusion_mode(&self) -> ExtrinsicInclusionMode { + self.extrinsic_inclusion_mode + } + /// Push onto the block's list of extrinsics. /// /// This will ensure the extrinsic can be validly executed (by executing it). diff --git a/substrate/client/proposer-metrics/src/lib.rs b/substrate/client/proposer-metrics/src/lib.rs index 012e8ca769a9..2856300cf802 100644 --- a/substrate/client/proposer-metrics/src/lib.rs +++ b/substrate/client/proposer-metrics/src/lib.rs @@ -44,11 +44,14 @@ impl MetricsLink { } /// The reason why proposing a block ended. +#[derive(Clone, Copy, PartialEq, Eq)] pub enum EndProposingReason { NoMoreTransactions, HitDeadline, HitBlockSizeLimit, HitBlockWeightLimit, + /// No transactions are allowed in the block. + TransactionForbidden, } /// Authorship metrics. @@ -112,6 +115,7 @@ impl Metrics { EndProposingReason::NoMoreTransactions => "no_more_transactions", EndProposingReason::HitBlockSizeLimit => "hit_block_size_limit", EndProposingReason::HitBlockWeightLimit => "hit_block_weight_limit", + EndProposingReason::TransactionForbidden => "transactions_forbidden", }; self.end_proposing_reason.with_label_values(&[reason]).inc(); diff --git a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs index 9544736d84c8..89d8c4ce2713 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs @@ -242,12 +242,12 @@ async fn follow_with_runtime() { let event: FollowEvent = get_next_event(&mut sub).await; // it is basically json-encoded substrate_test_runtime_client::runtime::VERSION - let runtime_str = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":0,\ - \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",4],\ + let runtime_str = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\ + \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",5],\ [\"0x37e397fc7c91f5e4\",2],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\ [\"0xbc9d89904f5b923f\",1],[\"0xc6e9a76309f39b09\",2],[\"0xdd718d5cc53262d4\",1],\ [\"0xcbca25e39f142387\",2],[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],\ - [\"0xed99c5acb25eedf5\",3],[\"0xfbc577b9d747efd6\",1]],\"transactionVersion\":1,\"stateVersion\":0}"; + [\"0xed99c5acb25eedf5\",3],[\"0xfbc577b9d747efd6\",1]],\"transactionVersion\":1,\"stateVersion\":1}"; let runtime: RuntimeVersion = serde_json::from_str(runtime_str).unwrap(); diff --git a/substrate/client/rpc/src/state/tests.rs b/substrate/client/rpc/src/state/tests.rs index 96f4c1be960f..dd866e671c50 100644 --- a/substrate/client/rpc/src/state/tests.rs +++ b/substrate/client/rpc/src/state/tests.rs @@ -475,7 +475,7 @@ async fn should_return_runtime_version() { // it is basically json-encoded substrate_test_runtime_client::runtime::VERSION let result = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\ - \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",4],\ + \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",5],\ [\"0x37e397fc7c91f5e4\",2],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",6],\ [\"0xbc9d89904f5b923f\",1],[\"0xc6e9a76309f39b09\",2],[\"0xdd718d5cc53262d4\",1],\ [\"0xcbca25e39f142387\",2],[\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],\ diff --git a/substrate/frame/executive/Cargo.toml b/substrate/frame/executive/Cargo.toml index a4ca265f6178..63285e4cb493 100644 --- a/substrate/frame/executive/Cargo.toml +++ b/substrate/frame/executive/Cargo.toml @@ -16,6 +16,7 @@ workspace = true targets = ["x86_64-unknown-linux-gnu"] [dependencies] +aquamarine = "0.3.2" codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = [ "derive", ] } @@ -44,6 +45,7 @@ default = ["std"] with-tracing = ["sp-tracing/with-tracing"] std = [ "codec/std", + "frame-support/experimental", "frame-support/std", "frame-system/std", "frame-try-runtime/std", diff --git a/substrate/frame/executive/src/lib.rs b/substrate/frame/executive/src/lib.rs index 48ff675f8082..3028eaf318e0 100644 --- a/substrate/frame/executive/src/lib.rs +++ b/substrate/frame/executive/src/lib.rs @@ -15,6 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![cfg_attr(not(feature = "std"), no_std)] + //! # Executive Module //! //! The Executive module acts as the orchestration layer for the runtime. It dispatches incoming @@ -35,6 +37,8 @@ //! - Finalize a block. //! - Start an off-chain worker. //! +//! The flow of their application in a block is explained in the [block flowchart](block_flowchart). +//! //! ### Implementations //! //! The Executive module provides the following implementations: @@ -114,17 +118,51 @@ //! pub type Executive = executive::Executive; //! ``` -#![cfg_attr(not(feature = "std"), no_std)] +#[cfg(doc)] +#[cfg_attr(doc, aquamarine::aquamarine)] +/// # Block Execution +/// +/// These are the steps of block execution as done by [`Executive::execute_block`]. A block is +/// invalid if any of them fail. +/// +/// ```mermaid +/// flowchart TD +/// Executive::execute_block --> on_runtime_upgrade +/// on_runtime_upgrade --> System::initialize +/// Executive::initialize_block --> System::initialize +/// System::initialize --> on_initialize +/// on_initialize --> PreInherents[System::PreInherents] +/// PreInherents --> Inherents[Apply Inherents] +/// Inherents --> PostInherents[System::PostInherents] +/// PostInherents --> Check{MBM ongoing?} +/// Check -->|No| poll +/// Check -->|Yes| post_transactions_2[System::PostTransaction] +/// post_transactions_2 --> Step[MBMs::step] +/// Step --> on_finalize +/// poll --> transactions[Apply Transactions] +/// transactions --> post_transactions_1[System::PostTransaction] +/// post_transactions_1 --> CheckIdle{Weight remaining?} +/// CheckIdle -->|Yes| on_idle +/// CheckIdle -->|No| on_finalize +/// on_idle --> on_finalize +/// ``` +pub mod block_flowchart {} + +#[cfg(test)] +mod tests; use codec::{Codec, Encode}; use frame_support::{ + defensive_assert, dispatch::{DispatchClass, DispatchInfo, GetDispatchInfo, PostDispatchInfo}, + migrations::MultiStepMigrator, pallet_prelude::InvalidTransaction, traits::{ BeforeAllRuntimeMigrations, EnsureInherentsAreFirst, ExecuteBlock, OffchainWorker, - OnFinalize, OnIdle, OnInitialize, OnRuntimeUpgrade, + OnFinalize, OnIdle, OnInitialize, OnPoll, OnRuntimeUpgrade, PostInherents, + PostTransactions, PreInherents, }, - weights::Weight, + weights::{Weight, WeightMeter}, }; use frame_system::pallet_prelude::BlockNumberFor; use sp_runtime::{ @@ -134,7 +172,7 @@ use sp_runtime::{ ValidateUnsigned, Zero, }, transaction_validity::{TransactionSource, TransactionValidity}, - ApplyExtrinsicResult, + ApplyExtrinsicResult, ExtrinsicInclusionMode, }; use sp_std::{marker::PhantomData, prelude::*}; @@ -198,7 +236,8 @@ impl< + OnInitialize> + OnIdle> + OnFinalize> - + OffchainWorker>, + + OffchainWorker> + + OnPoll>, COnRuntimeUpgrade: OnRuntimeUpgrade, > ExecuteBlock for Executive @@ -237,6 +276,7 @@ impl< + OnIdle> + OnFinalize> + OffchainWorker> + + OnPoll> + TryState> + TryDecodeEntireStorage, COnRuntimeUpgrade: OnRuntimeUpgrade, @@ -272,36 +312,50 @@ where select, ); - Self::initialize_block(block.header()); - Self::initial_checks(&block); - + let mode = Self::initialize_block(block.header()); + let num_inherents = Self::initial_checks(&block) as usize; let (header, extrinsics) = block.deconstruct(); + // Check if there are any forbidden non-inherents in the block. + if mode == ExtrinsicInclusionMode::OnlyInherents && extrinsics.len() > num_inherents { + return Err("Only inherents allowed".into()) + } + let try_apply_extrinsic = |uxt: Block::Extrinsic| -> ApplyExtrinsicResult { sp_io::init_tracing(); let encoded = uxt.encode(); let encoded_len = encoded.len(); + let is_inherent = System::is_inherent(&uxt); // skip signature verification. let xt = if signature_check { uxt.check(&Default::default()) } else { uxt.unchecked_into_checked_i_know_what_i_am_doing(&Default::default()) }?; - >::note_extrinsic(encoded); let dispatch_info = xt.get_dispatch_info(); + if !is_inherent && !>::inherents_applied() { + Self::inherents_applied(); + } + + >::note_extrinsic(encoded); let r = Applyable::apply::(xt, &dispatch_info, encoded_len)?; + if r.is_err() && dispatch_info.class == DispatchClass::Mandatory { + return Err(InvalidTransaction::BadMandatory.into()) + } + >::note_applied_extrinsic(&r, dispatch_info); Ok(r.map(|_| ()).map_err(|e| e.error)) }; - for e in extrinsics { + // Apply extrinsics: + for e in extrinsics.iter() { if let Err(err) = try_apply_extrinsic(e.clone()) { log::error!( - target: LOG_TARGET, "executing transaction {:?} failed due to {:?}. Aborting the rest of the block execution.", + target: LOG_TARGET, "transaction {:?} failed due to {:?}. Aborting the rest of the block execution.", e, err, ); @@ -309,9 +363,17 @@ where } } + // In this case there were no transactions to trigger this state transition: + if !>::inherents_applied() { + Self::inherents_applied(); + } + // post-extrinsics book-keeping >::note_finished_extrinsics(); - Self::idle_and_finalize_hook(*header.number()); + ::PostTransactions::post_transactions(); + + Self::on_idle_hook(*header.number()); + Self::on_finalize_hook(*header.number()); // run the try-state checks of all pallets, ensuring they don't alter any state. let _guard = frame_support::StorageNoopGuard::default(); @@ -449,7 +511,8 @@ impl< + OnInitialize> + OnIdle> + OnFinalize> - + OffchainWorker>, + + OffchainWorker> + + OnPoll>, COnRuntimeUpgrade: OnRuntimeUpgrade, > Executive where @@ -464,16 +527,36 @@ where pub fn execute_on_runtime_upgrade() -> Weight { let before_all_weight = ::before_all_runtime_migrations(); - <(COnRuntimeUpgrade, AllPalletsWithSystem) as OnRuntimeUpgrade>::on_runtime_upgrade() - .saturating_add(before_all_weight) + + let runtime_upgrade_weight = <( + COnRuntimeUpgrade, + ::SingleBlockMigrations, + // We want to run the migrations before we call into the pallets as they may + // access any state that would then not be migrated. + AllPalletsWithSystem, + ) as OnRuntimeUpgrade>::on_runtime_upgrade(); + + before_all_weight.saturating_add(runtime_upgrade_weight) } /// Start the execution of a particular block. - pub fn initialize_block(header: &frame_system::pallet_prelude::HeaderFor) { + pub fn initialize_block( + header: &frame_system::pallet_prelude::HeaderFor, + ) -> ExtrinsicInclusionMode { sp_io::init_tracing(); sp_tracing::enter_span!(sp_tracing::Level::TRACE, "init_block"); let digests = Self::extract_pre_digest(header); Self::initialize_block_impl(header.number(), header.parent_hash(), &digests); + + Self::extrinsic_mode() + } + + fn extrinsic_mode() -> ExtrinsicInclusionMode { + if ::MultiBlockMigrator::ongoing() { + ExtrinsicInclusionMode::OnlyInherents + } else { + ExtrinsicInclusionMode::AllExtrinsics + } } fn extract_pre_digest(header: &frame_system::pallet_prelude::HeaderFor) -> Digest { @@ -519,6 +602,7 @@ where ); frame_system::Pallet::::note_finished_initialize(); + ::PreInherents::pre_inherents(); } /// Returns if the runtime has been upgraded, based on [`frame_system::LastRuntimeUpgrade`]. @@ -529,7 +613,8 @@ where last.map(|v| v.was_upgraded(¤t)).unwrap_or(true) } - fn initial_checks(block: &Block) { + /// Returns the number of inherents in the block. + fn initial_checks(block: &Block) -> u32 { sp_tracing::enter_span!(sp_tracing::Level::TRACE, "initial_checks"); let header = block.header(); @@ -542,8 +627,9 @@ where "Parent hash should be valid.", ); - if let Err(i) = System::ensure_inherents_are_first(block) { - panic!("Invalid inherent position for extrinsic at index {}", i); + match System::ensure_inherents_are_first(block) { + Ok(num) => num, + Err(i) => panic!("Invalid inherent position for extrinsic at index {}", i), } } @@ -552,53 +638,90 @@ where sp_io::init_tracing(); sp_tracing::within_span! { sp_tracing::info_span!("execute_block", ?block); + // Execute `on_runtime_upgrade` and `on_initialize`. + let mode = Self::initialize_block(block.header()); + let num_inherents = Self::initial_checks(&block) as usize; + let (header, extrinsics) = block.deconstruct(); + let num_extrinsics = extrinsics.len(); - Self::initialize_block(block.header()); + if mode == ExtrinsicInclusionMode::OnlyInherents && num_extrinsics > num_inherents { + // Invalid block + panic!("Only inherents are allowed in this block") + } - // any initial checks - Self::initial_checks(&block); + Self::apply_extrinsics(extrinsics.into_iter()); - // execute extrinsics - let (header, extrinsics) = block.deconstruct(); - Self::execute_extrinsics_with_book_keeping(extrinsics, *header.number()); + // In this case there were no transactions to trigger this state transition: + if !>::inherents_applied() { + defensive_assert!(num_inherents == num_extrinsics); + Self::inherents_applied(); + } - // any final checks + >::note_finished_extrinsics(); + ::PostTransactions::post_transactions(); + + Self::on_idle_hook(*header.number()); + Self::on_finalize_hook(*header.number()); Self::final_checks(&header); } } - /// Execute given extrinsics and take care of post-extrinsics book-keeping. - fn execute_extrinsics_with_book_keeping( - extrinsics: Vec, - block_number: NumberFor, - ) { + /// Logic that runs directly after inherent application. + /// + /// It advances the Multi-Block-Migrations or runs the `on_poll` hook. + pub fn inherents_applied() { + >::note_inherents_applied(); + ::PostInherents::post_inherents(); + + if ::MultiBlockMigrator::ongoing() { + let used_weight = ::MultiBlockMigrator::step(); + >::register_extra_weight_unchecked( + used_weight, + DispatchClass::Mandatory, + ); + } else { + let block_number = >::block_number(); + Self::on_poll_hook(block_number); + } + } + + /// Execute given extrinsics. + fn apply_extrinsics(extrinsics: impl Iterator) { extrinsics.into_iter().for_each(|e| { if let Err(e) = Self::apply_extrinsic(e) { let err: &'static str = e.into(); panic!("{}", err) } }); - - // post-extrinsics book-keeping - >::note_finished_extrinsics(); - - Self::idle_and_finalize_hook(block_number); } /// Finalize the block - it is up the caller to ensure that all header fields are valid /// except state-root. + // Note: Only used by the block builder - not Executive itself. pub fn finalize_block() -> frame_system::pallet_prelude::HeaderFor { sp_io::init_tracing(); sp_tracing::enter_span!(sp_tracing::Level::TRACE, "finalize_block"); - >::note_finished_extrinsics(); - let block_number = >::block_number(); - Self::idle_and_finalize_hook(block_number); + // In this case there were no transactions to trigger this state transition: + if !>::inherents_applied() { + Self::inherents_applied(); + } + >::note_finished_extrinsics(); + ::PostTransactions::post_transactions(); + let block_number = >::block_number(); + Self::on_idle_hook(block_number); + Self::on_finalize_hook(block_number); >::finalize() } - fn idle_and_finalize_hook(block_number: NumberFor) { + /// Run the `on_idle` hook of all pallet, but only if there is weight remaining and there are no + /// ongoing MBMs. + fn on_idle_hook(block_number: NumberFor) { + if ::MultiBlockMigrator::ongoing() { + return + } + let weight = >::block_weight(); let max_weight = >::get().max_block; let remaining_weight = max_weight.saturating_sub(weight.total()); @@ -613,7 +736,33 @@ where DispatchClass::Mandatory, ); } + } + + fn on_poll_hook(block_number: NumberFor) { + defensive_assert!( + !::MultiBlockMigrator::ongoing(), + "on_poll should not be called during migrations" + ); + let weight = >::block_weight(); + let max_weight = >::get().max_block; + let remaining = max_weight.saturating_sub(weight.total()); + + if remaining.all_gt(Weight::zero()) { + let mut meter = WeightMeter::with_limit(remaining); + >>::on_poll( + block_number, + &mut meter, + ); + >::register_extra_weight_unchecked( + meter.consumed(), + DispatchClass::Mandatory, + ); + } + } + + /// Run the `on_finalize` hook of all pallet. + fn on_finalize_hook(block_number: NumberFor) { >>::on_finalize(block_number); } @@ -627,8 +776,18 @@ where let encoded_len = encoded.len(); sp_tracing::enter_span!(sp_tracing::info_span!("apply_extrinsic", ext=?sp_core::hexdisplay::HexDisplay::from(&encoded))); + + // We use the dedicated `is_inherent` check here, since just relying on `Mandatory` dispatch + // class does not capture optional inherents. + let is_inherent = System::is_inherent(&uxt); + // Verify that the signature is good. let xt = uxt.check(&Default::default())?; + let dispatch_info = xt.get_dispatch_info(); + + if !is_inherent && !>::inherents_applied() { + Self::inherents_applied(); + } // We don't need to make sure to `note_extrinsic` only after we know it's going to be // executed to prevent it from leaking in storage since at this point, it will either @@ -637,8 +796,6 @@ where // AUDIT: Under no circumstances may this function panic from here onwards. - // Decode parameters and dispatch - let dispatch_info = xt.get_dispatch_info(); let r = Applyable::apply::(xt, &dispatch_info, encoded_len)?; // Mandatory(inherents) are not allowed to fail. @@ -745,956 +902,3 @@ where ) } } - -#[cfg(test)] -mod tests { - use super::*; - - use sp_core::H256; - use sp_runtime::{ - generic::{DigestItem, Era}, - testing::{Block, Digest, Header}, - traits::{BlakeTwo256, Block as BlockT, Header as HeaderT, IdentityLookup}, - transaction_validity::{ - InvalidTransaction, TransactionValidityError, UnknownTransaction, ValidTransaction, - }, - BuildStorage, DispatchError, - }; - - use frame_support::{ - assert_err, derive_impl, parameter_types, - traits::{fungible, ConstU32, ConstU64, ConstU8, Currency}, - weights::{ConstantMultiplier, IdentityFee, RuntimeDbWeight, Weight, WeightToFee}, - }; - use frame_system::{ChainContext, LastRuntimeUpgrade, LastRuntimeUpgradeInfo}; - use pallet_balances::Call as BalancesCall; - use pallet_transaction_payment::CurrencyAdapter; - - const TEST_KEY: &[u8] = b":test:key:"; - - #[frame_support::pallet(dev_mode)] - mod custom { - use frame_support::pallet_prelude::*; - use frame_system::pallet_prelude::*; - - #[pallet::pallet] - pub struct Pallet(_); - - #[pallet::config] - pub trait Config: frame_system::Config {} - - #[pallet::hooks] - impl Hooks> for Pallet { - // module hooks. - // one with block number arg and one without - fn on_initialize(n: BlockNumberFor) -> Weight { - println!("on_initialize({})", n); - Weight::from_parts(175, 0) - } - - fn on_idle(n: BlockNumberFor, remaining_weight: Weight) -> Weight { - println!("on_idle{}, {})", n, remaining_weight); - Weight::from_parts(175, 0) - } - - fn on_finalize(n: BlockNumberFor) { - println!("on_finalize({})", n); - } - - fn on_runtime_upgrade() -> Weight { - sp_io::storage::set(super::TEST_KEY, "module".as_bytes()); - Weight::from_parts(200, 0) - } - - fn offchain_worker(n: BlockNumberFor) { - assert_eq!(BlockNumberFor::::from(1u32), n); - } - } - - #[pallet::call] - impl Pallet { - pub fn some_function(origin: OriginFor) -> DispatchResult { - // NOTE: does not make any different. - frame_system::ensure_signed(origin)?; - Ok(()) - } - - #[pallet::weight((200, DispatchClass::Operational))] - pub fn some_root_operation(origin: OriginFor) -> DispatchResult { - frame_system::ensure_root(origin)?; - Ok(()) - } - - pub fn some_unsigned_message(origin: OriginFor) -> DispatchResult { - frame_system::ensure_none(origin)?; - Ok(()) - } - - pub fn allowed_unsigned(origin: OriginFor) -> DispatchResult { - frame_system::ensure_root(origin)?; - Ok(()) - } - - pub fn unallowed_unsigned(origin: OriginFor) -> DispatchResult { - frame_system::ensure_root(origin)?; - Ok(()) - } - - #[pallet::weight((0, DispatchClass::Mandatory))] - pub fn inherent_call(origin: OriginFor) -> DispatchResult { - frame_system::ensure_none(origin)?; - Ok(()) - } - - pub fn calculate_storage_root(_origin: OriginFor) -> DispatchResult { - let root = sp_io::storage::root(sp_runtime::StateVersion::V1); - sp_io::storage::set("storage_root".as_bytes(), &root); - Ok(()) - } - } - - #[pallet::inherent] - impl ProvideInherent for Pallet { - type Call = Call; - - type Error = sp_inherents::MakeFatalError<()>; - - const INHERENT_IDENTIFIER: [u8; 8] = *b"test1234"; - - fn create_inherent(_data: &InherentData) -> Option { - None - } - - fn is_inherent(call: &Self::Call) -> bool { - *call == Call::::inherent_call {} - } - } - - #[pallet::validate_unsigned] - impl ValidateUnsigned for Pallet { - type Call = Call; - - // Inherent call is accepted for being dispatched - fn pre_dispatch(call: &Self::Call) -> Result<(), TransactionValidityError> { - match call { - Call::allowed_unsigned { .. } => Ok(()), - Call::inherent_call { .. } => Ok(()), - _ => Err(UnknownTransaction::NoUnsignedValidator.into()), - } - } - - // Inherent call is not validated as unsigned - fn validate_unsigned( - _source: TransactionSource, - call: &Self::Call, - ) -> TransactionValidity { - match call { - Call::allowed_unsigned { .. } => Ok(Default::default()), - _ => UnknownTransaction::NoUnsignedValidator.into(), - } - } - } - } - - frame_support::construct_runtime!( - pub enum Runtime { - System: frame_system, - Balances: pallet_balances, - TransactionPayment: pallet_transaction_payment, - Custom: custom, - } - ); - - parameter_types! { - pub BlockWeights: frame_system::limits::BlockWeights = - frame_system::limits::BlockWeights::builder() - .base_block(Weight::from_parts(10, 0)) - .for_class(DispatchClass::all(), |weights| weights.base_extrinsic = Weight::from_parts(5, 0)) - .for_class(DispatchClass::non_mandatory(), |weights| weights.max_total = Weight::from_parts(1024, u64::MAX).into()) - .build_or_panic(); - pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { - read: 10, - write: 100, - }; - } - #[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] - impl frame_system::Config for Runtime { - type BaseCallFilter = frame_support::traits::Everything; - type BlockWeights = BlockWeights; - type BlockLength = (); - type DbWeight = (); - type RuntimeOrigin = RuntimeOrigin; - type Nonce = u64; - type RuntimeCall = RuntimeCall; - type Hash = sp_core::H256; - type Hashing = BlakeTwo256; - type AccountId = u64; - type Lookup = IdentityLookup; - type Block = TestBlock; - type RuntimeEvent = RuntimeEvent; - type BlockHashCount = ConstU64<250>; - type Version = RuntimeVersion; - type PalletInfo = PalletInfo; - type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = (); - type OnSetCode = (); - type MaxConsumers = ConstU32<16>; - } - - type Balance = u64; - impl pallet_balances::Config for Runtime { - type Balance = Balance; - type RuntimeEvent = RuntimeEvent; - type DustRemoval = (); - type ExistentialDeposit = ConstU64<1>; - type AccountStore = System; - type MaxLocks = (); - type MaxReserves = (); - type ReserveIdentifier = [u8; 8]; - type WeightInfo = (); - type FreezeIdentifier = (); - type MaxFreezes = ConstU32<1>; - type RuntimeHoldReason = (); - type RuntimeFreezeReason = (); - } - - parameter_types! { - pub const TransactionByteFee: Balance = 0; - } - impl pallet_transaction_payment::Config for Runtime { - type RuntimeEvent = RuntimeEvent; - type OnChargeTransaction = CurrencyAdapter; - type OperationalFeeMultiplier = ConstU8<5>; - type WeightToFee = IdentityFee; - type LengthToFee = ConstantMultiplier; - type FeeMultiplierUpdate = (); - } - impl custom::Config for Runtime {} - - pub struct RuntimeVersion; - impl frame_support::traits::Get for RuntimeVersion { - fn get() -> sp_version::RuntimeVersion { - RuntimeVersionTestValues::get().clone() - } - } - - parameter_types! { - pub static RuntimeVersionTestValues: sp_version::RuntimeVersion = - Default::default(); - } - - type SignedExtra = ( - frame_system::CheckEra, - frame_system::CheckNonce, - frame_system::CheckWeight, - pallet_transaction_payment::ChargeTransactionPayment, - ); - type TestXt = sp_runtime::testing::TestXt; - type TestBlock = Block; - - // Will contain `true` when the custom runtime logic was called. - const CUSTOM_ON_RUNTIME_KEY: &[u8] = b":custom:on_runtime"; - - struct CustomOnRuntimeUpgrade; - impl OnRuntimeUpgrade for CustomOnRuntimeUpgrade { - fn on_runtime_upgrade() -> Weight { - sp_io::storage::set(TEST_KEY, "custom_upgrade".as_bytes()); - sp_io::storage::set(CUSTOM_ON_RUNTIME_KEY, &true.encode()); - System::deposit_event(frame_system::Event::CodeUpdated); - - assert_eq!(0, System::last_runtime_upgrade_spec_version()); - - Weight::from_parts(100, 0) - } - } - - type Executive = super::Executive< - Runtime, - Block, - ChainContext, - Runtime, - AllPalletsWithSystem, - CustomOnRuntimeUpgrade, - >; - - fn extra(nonce: u64, fee: Balance) -> SignedExtra { - ( - frame_system::CheckEra::from(Era::Immortal), - frame_system::CheckNonce::from(nonce), - frame_system::CheckWeight::new(), - pallet_transaction_payment::ChargeTransactionPayment::from(fee), - ) - } - - fn sign_extra(who: u64, nonce: u64, fee: Balance) -> Option<(u64, SignedExtra)> { - Some((who, extra(nonce, fee))) - } - - fn call_transfer(dest: u64, value: u64) -> RuntimeCall { - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest, value }) - } - - #[test] - fn balance_transfer_dispatch_works() { - let mut t = frame_system::GenesisConfig::::default().build_storage().unwrap(); - pallet_balances::GenesisConfig:: { balances: vec![(1, 211)] } - .assimilate_storage(&mut t) - .unwrap(); - let xt = TestXt::new(call_transfer(2, 69), sign_extra(1, 0, 0)); - let weight = xt.get_dispatch_info().weight + - ::BlockWeights::get() - .get(DispatchClass::Normal) - .base_extrinsic; - let fee: Balance = - ::WeightToFee::weight_to_fee(&weight); - let mut t = sp_io::TestExternalities::new(t); - t.execute_with(|| { - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - let r = Executive::apply_extrinsic(xt); - assert!(r.is_ok()); - assert_eq!(>::total_balance(&1), 142 - fee); - assert_eq!(>::total_balance(&2), 69); - }); - } - - fn new_test_ext(balance_factor: Balance) -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::::default().build_storage().unwrap(); - pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } - .assimilate_storage(&mut t) - .unwrap(); - t.into() - } - - fn new_test_ext_v0(balance_factor: Balance) -> sp_io::TestExternalities { - let mut t = frame_system::GenesisConfig::::default().build_storage().unwrap(); - pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } - .assimilate_storage(&mut t) - .unwrap(); - (t, sp_runtime::StateVersion::V0).into() - } - - #[test] - fn block_import_works() { - block_import_works_inner( - new_test_ext_v0(1), - array_bytes::hex_n_into_unchecked( - "65e953676859e7a33245908af7ad3637d6861eb90416d433d485e95e2dd174a1", - ), - ); - block_import_works_inner( - new_test_ext(1), - array_bytes::hex_n_into_unchecked( - "5a19b3d6fdb7241836349fdcbe2d9df4d4f945b949d979e31ad50bff1cbcd1c2", - ), - ); - } - fn block_import_works_inner(mut ext: sp_io::TestExternalities, state_root: H256) { - ext.execute_with(|| { - Executive::execute_block(Block { - header: Header { - parent_hash: [69u8; 32].into(), - number: 1, - state_root, - extrinsics_root: array_bytes::hex_n_into_unchecked( - "03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314", - ), - digest: Digest { logs: vec![] }, - }, - extrinsics: vec![], - }); - }); - } - - #[test] - #[should_panic] - fn block_import_of_bad_state_root_fails() { - new_test_ext(1).execute_with(|| { - Executive::execute_block(Block { - header: Header { - parent_hash: [69u8; 32].into(), - number: 1, - state_root: [0u8; 32].into(), - extrinsics_root: array_bytes::hex_n_into_unchecked( - "03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314", - ), - digest: Digest { logs: vec![] }, - }, - extrinsics: vec![], - }); - }); - } - - #[test] - #[should_panic] - fn block_import_of_bad_extrinsic_root_fails() { - new_test_ext(1).execute_with(|| { - Executive::execute_block(Block { - header: Header { - parent_hash: [69u8; 32].into(), - number: 1, - state_root: array_bytes::hex_n_into_unchecked( - "75e7d8f360d375bbe91bcf8019c01ab6362448b4a89e3b329717eb9d910340e5", - ), - extrinsics_root: [0u8; 32].into(), - digest: Digest { logs: vec![] }, - }, - extrinsics: vec![], - }); - }); - } - - #[test] - fn bad_extrinsic_not_inserted() { - let mut t = new_test_ext(1); - // bad nonce check! - let xt = TestXt::new(call_transfer(33, 69), sign_extra(1, 30, 0)); - t.execute_with(|| { - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - assert_err!( - Executive::apply_extrinsic(xt), - TransactionValidityError::Invalid(InvalidTransaction::Future) - ); - assert_eq!(>::extrinsic_index(), Some(0)); - }); - } - - #[test] - fn block_weight_limit_enforced() { - let mut t = new_test_ext(10000); - // given: TestXt uses the encoded len as fixed Len: - let xt = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), - sign_extra(1, 0, 0), - ); - let encoded = xt.encode(); - let encoded_len = encoded.len() as u64; - // on_initialize weight + base block execution weight - let block_weights = ::BlockWeights::get(); - let base_block_weight = Weight::from_parts(175, 0) + block_weights.base_block; - let limit = block_weights.get(DispatchClass::Normal).max_total.unwrap() - base_block_weight; - let num_to_exhaust_block = limit.ref_time() / (encoded_len + 5); - t.execute_with(|| { - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - // Base block execution weight + `on_initialize` weight from the custom module. - assert_eq!(>::block_weight().total(), base_block_weight); - - for nonce in 0..=num_to_exhaust_block { - let xt = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { - dest: 33, - value: 0, - }), - sign_extra(1, nonce.into(), 0), - ); - let res = Executive::apply_extrinsic(xt); - if nonce != num_to_exhaust_block { - assert!(res.is_ok()); - assert_eq!( - >::block_weight().total(), - //--------------------- on_initialize + block_execution + extrinsic_base weight - Weight::from_parts((encoded_len + 5) * (nonce + 1), 0) + base_block_weight, - ); - assert_eq!( - >::extrinsic_index(), - Some(nonce as u32 + 1) - ); - } else { - assert_eq!(res, Err(InvalidTransaction::ExhaustsResources.into())); - } - } - }); - } - - #[test] - fn block_weight_and_size_is_stored_per_tx() { - let xt = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), - sign_extra(1, 0, 0), - ); - let x1 = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), - sign_extra(1, 1, 0), - ); - let x2 = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), - sign_extra(1, 2, 0), - ); - let len = xt.clone().encode().len() as u32; - let mut t = new_test_ext(1); - t.execute_with(|| { - // Block execution weight + on_initialize weight from custom module - let base_block_weight = Weight::from_parts(175, 0) + - ::BlockWeights::get().base_block; - - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - assert_eq!(>::block_weight().total(), base_block_weight); - assert_eq!(>::all_extrinsics_len(), 0); - - assert!(Executive::apply_extrinsic(xt.clone()).unwrap().is_ok()); - assert!(Executive::apply_extrinsic(x1.clone()).unwrap().is_ok()); - assert!(Executive::apply_extrinsic(x2.clone()).unwrap().is_ok()); - - // default weight for `TestXt` == encoded length. - let extrinsic_weight = Weight::from_parts(len as u64, 0) + - ::BlockWeights::get() - .get(DispatchClass::Normal) - .base_extrinsic; - assert_eq!( - >::block_weight().total(), - base_block_weight + 3u64 * extrinsic_weight, - ); - assert_eq!(>::all_extrinsics_len(), 3 * len); - - let _ = >::finalize(); - // All extrinsics length cleaned on `System::finalize` - assert_eq!(>::all_extrinsics_len(), 0); - - // New Block - Executive::initialize_block(&Header::new( - 2, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - // Block weight cleaned up on `System::initialize` - assert_eq!(>::block_weight().total(), base_block_weight); - }); - } - - #[test] - fn validate_unsigned() { - let valid = TestXt::new(RuntimeCall::Custom(custom::Call::allowed_unsigned {}), None); - let invalid = TestXt::new(RuntimeCall::Custom(custom::Call::unallowed_unsigned {}), None); - let mut t = new_test_ext(1); - - t.execute_with(|| { - assert_eq!( - Executive::validate_transaction( - TransactionSource::InBlock, - valid.clone(), - Default::default(), - ), - Ok(ValidTransaction::default()), - ); - assert_eq!( - Executive::validate_transaction( - TransactionSource::InBlock, - invalid.clone(), - Default::default(), - ), - Err(TransactionValidityError::Unknown(UnknownTransaction::NoUnsignedValidator)), - ); - assert_eq!(Executive::apply_extrinsic(valid), Ok(Err(DispatchError::BadOrigin))); - assert_eq!( - Executive::apply_extrinsic(invalid), - Err(TransactionValidityError::Unknown(UnknownTransaction::NoUnsignedValidator)) - ); - }); - } - - #[test] - fn can_not_pay_for_tx_fee_on_full_lock() { - let mut t = new_test_ext(1); - t.execute_with(|| { - as fungible::MutateFreeze>::set_freeze( - &(), - &1, - 110, - ) - .unwrap(); - let xt = TestXt::new( - RuntimeCall::System(frame_system::Call::remark { remark: vec![1u8] }), - sign_extra(1, 0, 0), - ); - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - assert_eq!(Executive::apply_extrinsic(xt), Err(InvalidTransaction::Payment.into()),); - assert_eq!(>::total_balance(&1), 111); - }); - } - - #[test] - fn block_hooks_weight_is_stored() { - new_test_ext(1).execute_with(|| { - Executive::initialize_block(&Header::new_from_number(1)); - Executive::finalize_block(); - // NOTE: might need updates over time if new weights are introduced. - // For now it only accounts for the base block execution weight and - // the `on_initialize` weight defined in the custom test module. - assert_eq!( - >::block_weight().total(), - Weight::from_parts(175 + 175 + 10, 0) - ); - }) - } - - #[test] - fn runtime_upgraded_should_work() { - new_test_ext(1).execute_with(|| { - RuntimeVersionTestValues::mutate(|v| *v = Default::default()); - // It should be added at genesis - assert!(LastRuntimeUpgrade::::exists()); - assert!(!Executive::runtime_upgraded()); - - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } - }); - assert!(Executive::runtime_upgraded()); - - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { - spec_version: 1, - spec_name: "test".into(), - ..Default::default() - } - }); - assert!(Executive::runtime_upgraded()); - - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { - spec_version: 0, - impl_version: 2, - ..Default::default() - } - }); - assert!(!Executive::runtime_upgraded()); - - LastRuntimeUpgrade::::take(); - assert!(Executive::runtime_upgraded()); - }) - } - - #[test] - fn last_runtime_upgrade_was_upgraded_works() { - let test_data = vec![ - (0, "", 1, "", true), - (1, "", 1, "", false), - (1, "", 1, "test", true), - (1, "", 0, "", false), - (1, "", 0, "test", true), - ]; - - for (spec_version, spec_name, c_spec_version, c_spec_name, result) in test_data { - let current = sp_version::RuntimeVersion { - spec_version: c_spec_version, - spec_name: c_spec_name.into(), - ..Default::default() - }; - - let last = LastRuntimeUpgradeInfo { - spec_version: spec_version.into(), - spec_name: spec_name.into(), - }; - - assert_eq!(result, last.was_upgraded(¤t)); - } - } - - #[test] - fn custom_runtime_upgrade_is_called_before_modules() { - new_test_ext(1).execute_with(|| { - // Make sure `on_runtime_upgrade` is called. - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } - }); - - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - assert_eq!(&sp_io::storage::get(TEST_KEY).unwrap()[..], *b"module"); - assert_eq!(sp_io::storage::get(CUSTOM_ON_RUNTIME_KEY).unwrap(), true.encode()); - assert_eq!( - Some(RuntimeVersionTestValues::get().into()), - LastRuntimeUpgrade::::get(), - ) - }); - } - - #[test] - fn event_from_runtime_upgrade_is_included() { - new_test_ext(1).execute_with(|| { - // Make sure `on_runtime_upgrade` is called. - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } - }); - - // set block number to non zero so events are not excluded - System::set_block_number(1); - - Executive::initialize_block(&Header::new( - 2, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - System::assert_last_event(frame_system::Event::::CodeUpdated.into()); - }); - } - - /// Regression test that ensures that the custom on runtime upgrade is called when executive is - /// used through the `ExecuteBlock` trait. - #[test] - fn custom_runtime_upgrade_is_called_when_using_execute_block_trait() { - let xt = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), - sign_extra(1, 0, 0), - ); - - let header = new_test_ext(1).execute_with(|| { - // Make sure `on_runtime_upgrade` is called. - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } - }); - - // Let's build some fake block. - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - Executive::apply_extrinsic(xt.clone()).unwrap().unwrap(); - - Executive::finalize_block() - }); - - // Reset to get the correct new genesis below. - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 0, ..Default::default() } - }); - - new_test_ext(1).execute_with(|| { - // Make sure `on_runtime_upgrade` is called. - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } - }); - - >>::execute_block(Block::new(header, vec![xt])); - - assert_eq!(&sp_io::storage::get(TEST_KEY).unwrap()[..], *b"module"); - assert_eq!(sp_io::storage::get(CUSTOM_ON_RUNTIME_KEY).unwrap(), true.encode()); - }); - } - - #[test] - fn all_weights_are_recorded_correctly() { - // Reset to get the correct new genesis below. - RuntimeVersionTestValues::take(); - - new_test_ext(1).execute_with(|| { - // Make sure `on_runtime_upgrade` is called for maximum complexity - RuntimeVersionTestValues::mutate(|v| { - *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } - }); - - let block_number = 1; - - Executive::initialize_block(&Header::new( - block_number, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - // Reset the last runtime upgrade info, to make the second call to `on_runtime_upgrade` - // succeed. - LastRuntimeUpgrade::::take(); - - // All weights that show up in the `initialize_block_impl` - let custom_runtime_upgrade_weight = CustomOnRuntimeUpgrade::on_runtime_upgrade(); - let runtime_upgrade_weight = - ::on_runtime_upgrade(); - let on_initialize_weight = - >::on_initialize(block_number); - let base_block_weight = - ::BlockWeights::get().base_block; - - // Weights are recorded correctly - assert_eq!( - frame_system::Pallet::::block_weight().total(), - custom_runtime_upgrade_weight + - runtime_upgrade_weight + - on_initialize_weight + base_block_weight, - ); - }); - } - - #[test] - fn offchain_worker_works_as_expected() { - new_test_ext(1).execute_with(|| { - let parent_hash = sp_core::H256::from([69u8; 32]); - let mut digest = Digest::default(); - digest.push(DigestItem::Seal([1, 2, 3, 4], vec![5, 6, 7, 8])); - - let header = - Header::new(1, H256::default(), H256::default(), parent_hash, digest.clone()); - - Executive::offchain_worker(&header); - - assert_eq!(digest, System::digest()); - assert_eq!(parent_hash, System::block_hash(0)); - assert_eq!(header.hash(), System::block_hash(1)); - }); - } - - #[test] - fn calculating_storage_root_twice_works() { - let call = RuntimeCall::Custom(custom::Call::calculate_storage_root {}); - let xt = TestXt::new(call, sign_extra(1, 0, 0)); - - let header = new_test_ext(1).execute_with(|| { - // Let's build some fake block. - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - Executive::apply_extrinsic(xt.clone()).unwrap().unwrap(); - - Executive::finalize_block() - }); - - new_test_ext(1).execute_with(|| { - Executive::execute_block(Block::new(header, vec![xt])); - }); - } - - #[test] - #[should_panic(expected = "Invalid inherent position for extrinsic at index 1")] - fn invalid_inherent_position_fail() { - let xt1 = TestXt::new( - RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), - sign_extra(1, 0, 0), - ); - let xt2 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent_call {}), None); - - let header = new_test_ext(1).execute_with(|| { - // Let's build some fake block. - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); - Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); - - Executive::finalize_block() - }); - - new_test_ext(1).execute_with(|| { - Executive::execute_block(Block::new(header, vec![xt1, xt2])); - }); - } - - #[test] - fn valid_inherents_position_works() { - let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent_call {}), None); - let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); - - let header = new_test_ext(1).execute_with(|| { - // Let's build some fake block. - Executive::initialize_block(&Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - )); - - Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); - Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); - - Executive::finalize_block() - }); - - new_test_ext(1).execute_with(|| { - Executive::execute_block(Block::new(header, vec![xt1, xt2])); - }); - } - - #[test] - #[should_panic(expected = "A call was labelled as mandatory, but resulted in an Error.")] - fn invalid_inherents_fail_block_execution() { - let xt1 = - TestXt::new(RuntimeCall::Custom(custom::Call::inherent_call {}), sign_extra(1, 0, 0)); - - new_test_ext(1).execute_with(|| { - Executive::execute_block(Block::new( - Header::new( - 1, - H256::default(), - H256::default(), - [69u8; 32].into(), - Digest::default(), - ), - vec![xt1], - )); - }); - } - - // Inherents are created by the runtime and don't need to be validated. - #[test] - fn inherents_fail_validate_block() { - let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent_call {}), None); - - new_test_ext(1).execute_with(|| { - assert_eq!( - Executive::validate_transaction(TransactionSource::External, xt1, H256::random()) - .unwrap_err(), - InvalidTransaction::MandatoryValidation.into() - ); - }) - } -} diff --git a/substrate/frame/executive/src/tests.rs b/substrate/frame/executive/src/tests.rs new file mode 100644 index 000000000000..70b55f6e8553 --- /dev/null +++ b/substrate/frame/executive/src/tests.rs @@ -0,0 +1,1389 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test the `frame-executive` crate. + +use super::*; + +use sp_core::H256; +use sp_runtime::{ + generic::{DigestItem, Era}, + testing::{Block, Digest, Header}, + traits::{Block as BlockT, Header as HeaderT}, + transaction_validity::{ + InvalidTransaction, TransactionValidityError, UnknownTransaction, ValidTransaction, + }, + BuildStorage, DispatchError, +}; + +use frame_support::{ + assert_err, assert_ok, derive_impl, + migrations::MultiStepMigrator, + pallet_prelude::*, + parameter_types, + traits::{fungible, ConstU8, Currency, IsInherent}, + weights::{ConstantMultiplier, IdentityFee, RuntimeDbWeight, Weight, WeightMeter, WeightToFee}, +}; +use frame_system::{pallet_prelude::*, ChainContext, LastRuntimeUpgrade, LastRuntimeUpgradeInfo}; +use pallet_balances::Call as BalancesCall; +use pallet_transaction_payment::CurrencyAdapter; + +const TEST_KEY: &[u8] = b":test:key:"; + +#[frame_support::pallet(dev_mode)] +mod custom { + use super::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::hooks] + impl Hooks> for Pallet { + // module hooks. + // one with block number arg and one without + fn on_initialize(_: BlockNumberFor) -> Weight { + Weight::from_parts(175, 0) + } + + fn on_idle(_: BlockNumberFor, _: Weight) -> Weight { + Weight::from_parts(175, 0) + } + + fn on_poll(_: BlockNumberFor, _: &mut WeightMeter) {} + + fn on_finalize(_: BlockNumberFor) {} + + fn on_runtime_upgrade() -> Weight { + sp_io::storage::set(super::TEST_KEY, "module".as_bytes()); + Weight::from_parts(200, 0) + } + + fn offchain_worker(n: BlockNumberFor) { + assert_eq!(BlockNumberFor::::from(1u32), n); + } + } + + #[pallet::call] + impl Pallet { + pub fn some_function(origin: OriginFor) -> DispatchResult { + // NOTE: does not make any difference. + frame_system::ensure_signed(origin)?; + Ok(()) + } + + #[pallet::weight((200, DispatchClass::Operational))] + pub fn some_root_operation(origin: OriginFor) -> DispatchResult { + frame_system::ensure_root(origin)?; + Ok(()) + } + + pub fn some_unsigned_message(origin: OriginFor) -> DispatchResult { + frame_system::ensure_none(origin)?; + Ok(()) + } + + pub fn allowed_unsigned(origin: OriginFor) -> DispatchResult { + frame_system::ensure_root(origin)?; + Ok(()) + } + + pub fn unallowed_unsigned(origin: OriginFor) -> DispatchResult { + frame_system::ensure_root(origin)?; + Ok(()) + } + + #[pallet::weight((0, DispatchClass::Mandatory))] + pub fn inherent(origin: OriginFor) -> DispatchResult { + frame_system::ensure_none(origin)?; + Ok(()) + } + + pub fn calculate_storage_root(_origin: OriginFor) -> DispatchResult { + let root = sp_io::storage::root(sp_runtime::StateVersion::V1); + sp_io::storage::set("storage_root".as_bytes(), &root); + Ok(()) + } + } + + #[pallet::inherent] + impl ProvideInherent for Pallet { + type Call = Call; + + type Error = sp_inherents::MakeFatalError<()>; + + const INHERENT_IDENTIFIER: [u8; 8] = *b"test1234"; + + fn create_inherent(_data: &InherentData) -> Option { + None + } + + fn is_inherent(call: &Self::Call) -> bool { + *call == Call::::inherent {} + } + } + + #[pallet::validate_unsigned] + impl ValidateUnsigned for Pallet { + type Call = Call; + + // Inherent call is accepted for being dispatched + fn pre_dispatch(call: &Self::Call) -> Result<(), TransactionValidityError> { + match call { + Call::allowed_unsigned { .. } => Ok(()), + Call::inherent { .. } => Ok(()), + _ => Err(UnknownTransaction::NoUnsignedValidator.into()), + } + } + + // Inherent call is not validated as unsigned + fn validate_unsigned(_source: TransactionSource, call: &Self::Call) -> TransactionValidity { + match call { + Call::allowed_unsigned { .. } => Ok(Default::default()), + _ => UnknownTransaction::NoUnsignedValidator.into(), + } + } + } +} + +#[frame_support::pallet(dev_mode)] +mod custom2 { + use super::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::hooks] + impl Hooks> for Pallet { + // module hooks. + // one with block number arg and one without + fn on_initialize(_: BlockNumberFor) -> Weight { + assert!( + !MockedSystemCallbacks::pre_inherent_called(), + "Pre inherent hook goes after on_initialize" + ); + + Weight::from_parts(0, 0) + } + + fn on_idle(_: BlockNumberFor, _: Weight) -> Weight { + assert!( + MockedSystemCallbacks::post_transactions_called(), + "Post transactions hook goes before after on_idle" + ); + Weight::from_parts(0, 0) + } + + fn on_finalize(_: BlockNumberFor) { + assert!( + MockedSystemCallbacks::post_transactions_called(), + "Post transactions hook goes before after on_finalize" + ); + } + + fn on_runtime_upgrade() -> Weight { + sp_io::storage::set(super::TEST_KEY, "module".as_bytes()); + Weight::from_parts(0, 0) + } + } + + #[pallet::call] + impl Pallet { + pub fn allowed_unsigned(origin: OriginFor) -> DispatchResult { + frame_system::ensure_root(origin)?; + Ok(()) + } + + pub fn some_call(_: OriginFor) -> DispatchResult { + assert!(MockedSystemCallbacks::post_inherent_called()); + assert!(!MockedSystemCallbacks::post_transactions_called()); + assert!(System::inherents_applied()); + + Ok(()) + } + + #[pallet::weight({0})] + pub fn optional_inherent(origin: OriginFor) -> DispatchResult { + frame_system::ensure_none(origin)?; + + assert!(MockedSystemCallbacks::pre_inherent_called()); + assert!(!MockedSystemCallbacks::post_inherent_called(), "Should not already be called"); + assert!(!System::inherents_applied()); + + Ok(()) + } + + #[pallet::weight((0, DispatchClass::Mandatory))] + pub fn inherent(origin: OriginFor) -> DispatchResult { + frame_system::ensure_none(origin)?; + + assert!(MockedSystemCallbacks::pre_inherent_called()); + assert!(!MockedSystemCallbacks::post_inherent_called(), "Should not already be called"); + assert!(!System::inherents_applied()); + + Ok(()) + } + } + + #[pallet::inherent] + impl ProvideInherent for Pallet { + type Call = Call; + + type Error = sp_inherents::MakeFatalError<()>; + + const INHERENT_IDENTIFIER: [u8; 8] = *b"test1235"; + + fn create_inherent(_data: &InherentData) -> Option { + None + } + + fn is_inherent(call: &Self::Call) -> bool { + matches!(call, Call::::inherent {} | Call::::optional_inherent {}) + } + } + + #[pallet::validate_unsigned] + impl ValidateUnsigned for Pallet { + type Call = Call; + + // Inherent call is accepted for being dispatched + fn pre_dispatch(call: &Self::Call) -> Result<(), TransactionValidityError> { + match call { + Call::allowed_unsigned { .. } | + Call::optional_inherent { .. } | + Call::inherent { .. } => Ok(()), + _ => Err(UnknownTransaction::NoUnsignedValidator.into()), + } + } + + // Inherent call is not validated as unsigned + fn validate_unsigned(_source: TransactionSource, call: &Self::Call) -> TransactionValidity { + match call { + Call::allowed_unsigned { .. } => Ok(Default::default()), + _ => UnknownTransaction::NoUnsignedValidator.into(), + } + } + } +} + +frame_support::construct_runtime!( + pub struct Runtime + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + Balances: pallet_balances::{Pallet, Call, Storage, Config, Event}, + TransactionPayment: pallet_transaction_payment::{Pallet, Storage, Event}, + Custom: custom::{Pallet, Call, ValidateUnsigned, Inherent}, + Custom2: custom2::{Pallet, Call, ValidateUnsigned, Inherent}, + } +); + +parameter_types! { + pub BlockWeights: frame_system::limits::BlockWeights = + frame_system::limits::BlockWeights::builder() + .base_block(Weight::from_parts(10, 0)) + .for_class(DispatchClass::all(), |weights| weights.base_extrinsic = Weight::from_parts(5, 0)) + .for_class(DispatchClass::non_mandatory(), |weights| weights.max_total = Weight::from_parts(1024, u64::MAX).into()) + .build_or_panic(); + pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { + read: 10, + write: 100, + }; +} + +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] +impl frame_system::Config for Runtime { + type BlockWeights = BlockWeights; + type RuntimeOrigin = RuntimeOrigin; + type Nonce = u64; + type RuntimeCall = RuntimeCall; + type Block = TestBlock; + type RuntimeEvent = RuntimeEvent; + type Version = RuntimeVersion; + type AccountData = pallet_balances::AccountData; + type PreInherents = MockedSystemCallbacks; + type PostInherents = MockedSystemCallbacks; + type PostTransactions = MockedSystemCallbacks; + type MultiBlockMigrator = MockedModeGetter; +} + +type Balance = u64; + +#[derive_impl(pallet_balances::config_preludes::TestDefaultConfig as pallet_balances::DefaultConfig)] +impl pallet_balances::Config for Runtime { + type Balance = Balance; + type AccountStore = System; +} + +parameter_types! { + pub const TransactionByteFee: Balance = 0; +} +impl pallet_transaction_payment::Config for Runtime { + type RuntimeEvent = RuntimeEvent; + type OnChargeTransaction = CurrencyAdapter; + type OperationalFeeMultiplier = ConstU8<5>; + type WeightToFee = IdentityFee; + type LengthToFee = ConstantMultiplier; + type FeeMultiplierUpdate = (); +} + +impl custom::Config for Runtime {} +impl custom2::Config for Runtime {} + +pub struct RuntimeVersion; +impl frame_support::traits::Get for RuntimeVersion { + fn get() -> sp_version::RuntimeVersion { + RuntimeVersionTestValues::get().clone() + } +} + +parameter_types! { + pub static RuntimeVersionTestValues: sp_version::RuntimeVersion = + Default::default(); +} + +type SignedExtra = ( + frame_system::CheckEra, + frame_system::CheckNonce, + frame_system::CheckWeight, + pallet_transaction_payment::ChargeTransactionPayment, +); +type TestXt = sp_runtime::testing::TestXt; +type TestBlock = Block; + +// Will contain `true` when the custom runtime logic was called. +const CUSTOM_ON_RUNTIME_KEY: &[u8] = b":custom:on_runtime"; + +struct CustomOnRuntimeUpgrade; +impl OnRuntimeUpgrade for CustomOnRuntimeUpgrade { + fn on_runtime_upgrade() -> Weight { + sp_io::storage::set(TEST_KEY, "custom_upgrade".as_bytes()); + sp_io::storage::set(CUSTOM_ON_RUNTIME_KEY, &true.encode()); + System::deposit_event(frame_system::Event::CodeUpdated); + + assert_eq!(0, System::last_runtime_upgrade_spec_version()); + + Weight::from_parts(100, 0) + } +} + +type Executive = super::Executive< + Runtime, + Block, + ChainContext, + Runtime, + AllPalletsWithSystem, + CustomOnRuntimeUpgrade, +>; + +parameter_types! { + pub static SystemCallbacksCalled: u32 = 0; +} + +pub struct MockedSystemCallbacks; +impl PreInherents for MockedSystemCallbacks { + fn pre_inherents() { + assert_eq!(SystemCallbacksCalled::get(), 0); + SystemCallbacksCalled::set(1); + // Change the storage to modify the root hash: + frame_support::storage::unhashed::put(b":pre_inherent", b"0"); + } +} + +impl PostInherents for MockedSystemCallbacks { + fn post_inherents() { + assert_eq!(SystemCallbacksCalled::get(), 1); + SystemCallbacksCalled::set(2); + // Change the storage to modify the root hash: + frame_support::storage::unhashed::put(b":post_inherent", b"0"); + } +} + +impl PostTransactions for MockedSystemCallbacks { + fn post_transactions() { + assert_eq!(SystemCallbacksCalled::get(), 2); + SystemCallbacksCalled::set(3); + // Change the storage to modify the root hash: + frame_support::storage::unhashed::put(b":post_transaction", b"0"); + } +} + +impl MockedSystemCallbacks { + fn pre_inherent_called() -> bool { + SystemCallbacksCalled::get() >= 1 + } + + fn post_inherent_called() -> bool { + SystemCallbacksCalled::get() >= 2 + } + + fn post_transactions_called() -> bool { + SystemCallbacksCalled::get() >= 3 + } + + fn reset() { + SystemCallbacksCalled::set(0); + frame_support::storage::unhashed::kill(b":pre_inherent"); + frame_support::storage::unhashed::kill(b":post_inherent"); + frame_support::storage::unhashed::kill(b":post_transaction"); + } +} + +parameter_types! { + pub static MbmActive: bool = false; +} + +pub struct MockedModeGetter; +impl MultiStepMigrator for MockedModeGetter { + fn ongoing() -> bool { + MbmActive::get() + } + + fn step() -> Weight { + Weight::zero() + } +} + +fn extra(nonce: u64, fee: Balance) -> SignedExtra { + ( + frame_system::CheckEra::from(Era::Immortal), + frame_system::CheckNonce::from(nonce), + frame_system::CheckWeight::new(), + pallet_transaction_payment::ChargeTransactionPayment::from(fee), + ) +} + +fn sign_extra(who: u64, nonce: u64, fee: Balance) -> Option<(u64, SignedExtra)> { + Some((who, extra(nonce, fee))) +} + +fn call_transfer(dest: u64, value: u64) -> RuntimeCall { + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest, value }) +} + +#[test] +fn balance_transfer_dispatch_works() { + let mut t = frame_system::GenesisConfig::::default().build_storage().unwrap(); + pallet_balances::GenesisConfig:: { balances: vec![(1, 211)] } + .assimilate_storage(&mut t) + .unwrap(); + let xt = TestXt::new(call_transfer(2, 69), sign_extra(1, 0, 0)); + let weight = xt.get_dispatch_info().weight + + ::BlockWeights::get() + .get(DispatchClass::Normal) + .base_extrinsic; + let fee: Balance = + ::WeightToFee::weight_to_fee(&weight); + let mut t = sp_io::TestExternalities::new(t); + t.execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + let r = Executive::apply_extrinsic(xt); + assert!(r.is_ok()); + assert_eq!(>::total_balance(&1), 142 - fee); + assert_eq!(>::total_balance(&2), 69); + }); +} + +fn new_test_ext(balance_factor: Balance) -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::::default().build_storage().unwrap(); + pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } + .assimilate_storage(&mut t) + .unwrap(); + let mut ext: sp_io::TestExternalities = t.into(); + ext.execute_with(|| { + SystemCallbacksCalled::set(0); + }); + ext +} + +fn new_test_ext_v0(balance_factor: Balance) -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::::default().build_storage().unwrap(); + pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } + .assimilate_storage(&mut t) + .unwrap(); + (t, sp_runtime::StateVersion::V0).into() +} + +#[test] +fn block_import_works() { + block_import_works_inner( + new_test_ext_v0(1), + array_bytes::hex_n_into_unchecked( + "4826d3bdf87dbbc883d2ab274cbe272f58ed94a904619b59953e48294d1142d2", + ), + ); + block_import_works_inner( + new_test_ext(1), + array_bytes::hex_n_into_unchecked( + "d6b465f5a50c9f8d5a6edc0f01d285a6b19030f097d3aaf1649b7be81649f118", + ), + ); +} +fn block_import_works_inner(mut ext: sp_io::TestExternalities, state_root: H256) { + ext.execute_with(|| { + Executive::execute_block(Block { + header: Header { + parent_hash: [69u8; 32].into(), + number: 1, + state_root, + extrinsics_root: array_bytes::hex_n_into_unchecked( + "03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314", + ), + digest: Digest { logs: vec![] }, + }, + extrinsics: vec![], + }); + }); +} + +#[test] +#[should_panic] +fn block_import_of_bad_state_root_fails() { + new_test_ext(1).execute_with(|| { + Executive::execute_block(Block { + header: Header { + parent_hash: [69u8; 32].into(), + number: 1, + state_root: [0u8; 32].into(), + extrinsics_root: array_bytes::hex_n_into_unchecked( + "03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314", + ), + digest: Digest { logs: vec![] }, + }, + extrinsics: vec![], + }); + }); +} + +#[test] +#[should_panic] +fn block_import_of_bad_extrinsic_root_fails() { + new_test_ext(1).execute_with(|| { + Executive::execute_block(Block { + header: Header { + parent_hash: [69u8; 32].into(), + number: 1, + state_root: array_bytes::hex_n_into_unchecked( + "75e7d8f360d375bbe91bcf8019c01ab6362448b4a89e3b329717eb9d910340e5", + ), + extrinsics_root: [0u8; 32].into(), + digest: Digest { logs: vec![] }, + }, + extrinsics: vec![], + }); + }); +} + +#[test] +fn bad_extrinsic_not_inserted() { + let mut t = new_test_ext(1); + // bad nonce check! + let xt = TestXt::new(call_transfer(33, 69), sign_extra(1, 30, 0)); + t.execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + assert_err!( + Executive::apply_extrinsic(xt), + TransactionValidityError::Invalid(InvalidTransaction::Future) + ); + assert_eq!(>::extrinsic_index(), Some(0)); + }); +} + +#[test] +fn block_weight_limit_enforced() { + let mut t = new_test_ext(10000); + // given: TestXt uses the encoded len as fixed Len: + let xt = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, 0, 0), + ); + let encoded = xt.encode(); + let encoded_len = encoded.len() as u64; + // on_initialize weight + base block execution weight + let block_weights = ::BlockWeights::get(); + let base_block_weight = Weight::from_parts(175, 0) + block_weights.base_block; + let limit = block_weights.get(DispatchClass::Normal).max_total.unwrap() - base_block_weight; + let num_to_exhaust_block = limit.ref_time() / (encoded_len + 5); + t.execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + // Base block execution weight + `on_initialize` weight from the custom module. + assert_eq!(>::block_weight().total(), base_block_weight); + + for nonce in 0..=num_to_exhaust_block { + let xt = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, nonce.into(), 0), + ); + let res = Executive::apply_extrinsic(xt); + if nonce != num_to_exhaust_block { + assert!(res.is_ok()); + assert_eq!( + >::block_weight().total(), + //--------------------- on_initialize + block_execution + extrinsic_base weight + Weight::from_parts((encoded_len + 5) * (nonce + 1), 0) + base_block_weight, + ); + assert_eq!( + >::extrinsic_index(), + Some(nonce as u32 + 1) + ); + } else { + assert_eq!(res, Err(InvalidTransaction::ExhaustsResources.into())); + } + } + }); +} + +#[test] +fn block_weight_and_size_is_stored_per_tx() { + let xt = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, 0, 0), + ); + let x1 = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, 1, 0), + ); + let x2 = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, 2, 0), + ); + let len = xt.clone().encode().len() as u32; + let mut t = new_test_ext(1); + t.execute_with(|| { + // Block execution weight + on_initialize weight from custom module + let base_block_weight = Weight::from_parts(175, 0) + + ::BlockWeights::get().base_block; + + Executive::initialize_block(&Header::new_from_number(1)); + + assert_eq!(>::block_weight().total(), base_block_weight); + assert_eq!(>::all_extrinsics_len(), 0); + + assert!(Executive::apply_extrinsic(xt.clone()).unwrap().is_ok()); + assert!(Executive::apply_extrinsic(x1.clone()).unwrap().is_ok()); + assert!(Executive::apply_extrinsic(x2.clone()).unwrap().is_ok()); + + // default weight for `TestXt` == encoded length. + let extrinsic_weight = Weight::from_parts(len as u64, 0) + + ::BlockWeights::get() + .get(DispatchClass::Normal) + .base_extrinsic; + assert_eq!( + >::block_weight().total(), + base_block_weight + 3u64 * extrinsic_weight, + ); + assert_eq!(>::all_extrinsics_len(), 3 * len); + + let _ = >::finalize(); + // All extrinsics length cleaned on `System::finalize` + assert_eq!(>::all_extrinsics_len(), 0); + + // Reset to a new block. + SystemCallbacksCalled::take(); + Executive::initialize_block(&Header::new_from_number(2)); + + // Block weight cleaned up on `System::initialize` + assert_eq!(>::block_weight().total(), base_block_weight); + }); +} + +#[test] +fn validate_unsigned() { + let valid = TestXt::new(RuntimeCall::Custom(custom::Call::allowed_unsigned {}), None); + let invalid = TestXt::new(RuntimeCall::Custom(custom::Call::unallowed_unsigned {}), None); + let mut t = new_test_ext(1); + + t.execute_with(|| { + assert_eq!( + Executive::validate_transaction( + TransactionSource::InBlock, + valid.clone(), + Default::default(), + ), + Ok(ValidTransaction::default()), + ); + assert_eq!( + Executive::validate_transaction( + TransactionSource::InBlock, + invalid.clone(), + Default::default(), + ), + Err(TransactionValidityError::Unknown(UnknownTransaction::NoUnsignedValidator)), + ); + // Need to initialize the block before applying extrinsics for the `MockedSystemCallbacks` + // check. + Executive::initialize_block(&Header::new_from_number(1)); + assert_eq!(Executive::apply_extrinsic(valid), Ok(Err(DispatchError::BadOrigin))); + assert_eq!( + Executive::apply_extrinsic(invalid), + Err(TransactionValidityError::Unknown(UnknownTransaction::NoUnsignedValidator)) + ); + }); +} + +#[test] +fn can_not_pay_for_tx_fee_on_full_lock() { + let mut t = new_test_ext(1); + t.execute_with(|| { + as fungible::MutateFreeze>::set_freeze(&(), &1, 110) + .unwrap(); + let xt = TestXt::new( + RuntimeCall::System(frame_system::Call::remark { remark: vec![1u8] }), + sign_extra(1, 0, 0), + ); + Executive::initialize_block(&Header::new_from_number(1)); + + assert_eq!(Executive::apply_extrinsic(xt), Err(InvalidTransaction::Payment.into()),); + assert_eq!(>::total_balance(&1), 111); + }); +} + +#[test] +fn block_hooks_weight_is_stored() { + new_test_ext(1).execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + Executive::finalize_block(); + // NOTE: might need updates over time if new weights are introduced. + // For now it only accounts for the base block execution weight and + // the `on_initialize` weight defined in the custom test module. + assert_eq!( + >::block_weight().total(), + Weight::from_parts(175 + 175 + 10, 0) + ); + }) +} + +#[test] +fn runtime_upgraded_should_work() { + new_test_ext(1).execute_with(|| { + RuntimeVersionTestValues::mutate(|v| *v = Default::default()); + // It should be added at genesis + assert!(LastRuntimeUpgrade::::exists()); + assert!(!Executive::runtime_upgraded()); + + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + assert!(Executive::runtime_upgraded()); + + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { + spec_version: 1, + spec_name: "test".into(), + ..Default::default() + } + }); + assert!(Executive::runtime_upgraded()); + + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { + spec_version: 0, + impl_version: 2, + ..Default::default() + } + }); + assert!(!Executive::runtime_upgraded()); + + LastRuntimeUpgrade::::take(); + assert!(Executive::runtime_upgraded()); + }) +} + +#[test] +fn last_runtime_upgrade_was_upgraded_works() { + let test_data = vec![ + (0, "", 1, "", true), + (1, "", 1, "", false), + (1, "", 1, "test", true), + (1, "", 0, "", false), + (1, "", 0, "test", true), + ]; + + for (spec_version, spec_name, c_spec_version, c_spec_name, result) in test_data { + let current = sp_version::RuntimeVersion { + spec_version: c_spec_version, + spec_name: c_spec_name.into(), + ..Default::default() + }; + + let last = LastRuntimeUpgradeInfo { + spec_version: spec_version.into(), + spec_name: spec_name.into(), + }; + + assert_eq!(result, last.was_upgraded(¤t)); + } +} + +#[test] +fn custom_runtime_upgrade_is_called_before_modules() { + new_test_ext(1).execute_with(|| { + // Make sure `on_runtime_upgrade` is called. + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + + Executive::initialize_block(&Header::new_from_number(1)); + + assert_eq!(&sp_io::storage::get(TEST_KEY).unwrap()[..], *b"module"); + assert_eq!(sp_io::storage::get(CUSTOM_ON_RUNTIME_KEY).unwrap(), true.encode()); + assert_eq!( + Some(RuntimeVersionTestValues::get().into()), + LastRuntimeUpgrade::::get(), + ) + }); +} + +#[test] +fn event_from_runtime_upgrade_is_included() { + new_test_ext(1).execute_with(|| { + // Make sure `on_runtime_upgrade` is called. + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + + // set block number to non zero so events are not excluded + System::set_block_number(1); + + Executive::initialize_block(&Header::new_from_number(2)); + System::assert_last_event(frame_system::Event::::CodeUpdated.into()); + }); +} + +/// Regression test that ensures that the custom on runtime upgrade is called when executive is +/// used through the `ExecuteBlock` trait. +#[test] +fn custom_runtime_upgrade_is_called_when_using_execute_block_trait() { + let xt = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, 0, 0), + ); + + let header = new_test_ext(1).execute_with(|| { + // Make sure `on_runtime_upgrade` is called. + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + // Reset to get the correct new genesis below. + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 0, ..Default::default() } + }); + + new_test_ext(1).execute_with(|| { + // Make sure `on_runtime_upgrade` is called. + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + + >>::execute_block(Block::new(header, vec![xt])); + + assert_eq!(&sp_io::storage::get(TEST_KEY).unwrap()[..], *b"module"); + assert_eq!(sp_io::storage::get(CUSTOM_ON_RUNTIME_KEY).unwrap(), true.encode()); + }); +} + +#[test] +fn all_weights_are_recorded_correctly() { + // Reset to get the correct new genesis below. + RuntimeVersionTestValues::take(); + + new_test_ext(1).execute_with(|| { + // Make sure `on_runtime_upgrade` is called for maximum complexity + RuntimeVersionTestValues::mutate(|v| { + *v = sp_version::RuntimeVersion { spec_version: 1, ..Default::default() } + }); + + let block_number = 1; + + Executive::initialize_block(&Header::new_from_number(block_number)); + + // Reset the last runtime upgrade info, to make the second call to `on_runtime_upgrade` + // succeed. + LastRuntimeUpgrade::::take(); + MockedSystemCallbacks::reset(); + + // All weights that show up in the `initialize_block_impl` + let custom_runtime_upgrade_weight = CustomOnRuntimeUpgrade::on_runtime_upgrade(); + let runtime_upgrade_weight = + ::on_runtime_upgrade(); + let on_initialize_weight = + >::on_initialize(block_number); + let base_block_weight = ::BlockWeights::get().base_block; + + // Weights are recorded correctly + assert_eq!( + frame_system::Pallet::::block_weight().total(), + custom_runtime_upgrade_weight + + runtime_upgrade_weight + + on_initialize_weight + + base_block_weight, + ); + }); +} + +#[test] +fn offchain_worker_works_as_expected() { + new_test_ext(1).execute_with(|| { + let parent_hash = sp_core::H256::from([69u8; 32]); + let mut digest = Digest::default(); + digest.push(DigestItem::Seal([1, 2, 3, 4], vec![5, 6, 7, 8])); + + let header = Header::new(1, H256::default(), H256::default(), parent_hash, digest.clone()); + + Executive::offchain_worker(&header); + + assert_eq!(digest, System::digest()); + assert_eq!(parent_hash, System::block_hash(0)); + assert_eq!(header.hash(), System::block_hash(1)); + }); +} + +#[test] +fn calculating_storage_root_twice_works() { + let call = RuntimeCall::Custom(custom::Call::calculate_storage_root {}); + let xt = TestXt::new(call, sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + Executive::execute_block(Block::new(header, vec![xt])); + }); +} + +#[test] +#[should_panic(expected = "Invalid inherent position for extrinsic at index 1")] +fn invalid_inherent_position_fail() { + let xt1 = TestXt::new( + RuntimeCall::Balances(BalancesCall::transfer_allow_death { dest: 33, value: 0 }), + sign_extra(1, 0, 0), + ); + let xt2 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + + let header = new_test_ext(1).execute_with(|| { + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + Executive::execute_block(Block::new(header, vec![xt1, xt2])); + }); +} + +#[test] +fn valid_inherents_position_works() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + Executive::execute_block(Block::new(header, vec![xt1, xt2])); + }); +} + +#[test] +#[should_panic(expected = "A call was labelled as mandatory, but resulted in an Error.")] +fn invalid_inherents_fail_block_execution() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), sign_extra(1, 0, 0)); + + new_test_ext(1).execute_with(|| { + Executive::execute_block(Block::new( + Header::new(1, H256::default(), H256::default(), [69u8; 32].into(), Digest::default()), + vec![xt1], + )); + }); +} + +// Inherents are created by the runtime and don't need to be validated. +#[test] +fn inherents_fail_validate_block() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + + new_test_ext(1).execute_with(|| { + assert_eq!( + Executive::validate_transaction(TransactionSource::External, xt1, H256::random()) + .unwrap_err(), + InvalidTransaction::MandatoryValidation.into() + ); + }) +} + +/// Inherents still work while `initialize_block` forbids transactions. +#[test] +fn inherents_ok_while_exts_forbidden_works() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + + let header = new_test_ext(1).execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + // This is not applied: + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + // Tell `initialize_block` to forbid extrinsics: + Executive::execute_block(Block::new(header, vec![xt1])); + }); +} + +/// Refuses to import blocks with transactions during `OnlyInherents` mode. +#[test] +#[should_panic = "Only inherents are allowed in this block"] +fn transactions_in_only_inherents_block_errors() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + MbmActive::set(true); + Executive::execute_block(Block::new(header, vec![xt1, xt2])); + }); +} + +/// Same as above but no error. +#[test] +fn transactions_in_normal_block_works() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + // Tell `initialize_block` to forbid extrinsics: + Executive::execute_block(Block::new(header, vec![xt1, xt2])); + }); +} + +#[test] +#[cfg(feature = "try-runtime")] +fn try_execute_block_works() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + Executive::try_execute_block( + Block::new(header, vec![xt1, xt2]), + true, + true, + frame_try_runtime::TryStateSelect::All, + ) + .unwrap(); + }); +} + +/// Same as `extrinsic_while_exts_forbidden_errors` but using the try-runtime function. +#[test] +#[cfg(feature = "try-runtime")] +#[should_panic = "Only inherents allowed"] +fn try_execute_tx_forbidden_errors() { + let xt1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt2.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + MbmActive::set(true); + Executive::try_execute_block( + Block::new(header, vec![xt1, xt2]), + true, + true, + frame_try_runtime::TryStateSelect::All, + ) + .unwrap(); + }); +} + +/// Check that `ensure_inherents_are_first` reports the correct indices. +#[test] +fn ensure_inherents_are_first_works() { + let in1 = TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None); + let in2 = TestXt::new(RuntimeCall::Custom2(custom2::Call::inherent {}), None); + let xt2 = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + + // Mocked empty header: + let header = new_test_ext(1).execute_with(|| { + Executive::initialize_block(&Header::new_from_number(1)); + Executive::finalize_block() + }); + + new_test_ext(1).execute_with(|| { + assert_ok!(Runtime::ensure_inherents_are_first(&Block::new(header.clone(), vec![]),), 0); + assert_ok!( + Runtime::ensure_inherents_are_first(&Block::new(header.clone(), vec![xt2.clone()]),), + 0 + ); + assert_ok!( + Runtime::ensure_inherents_are_first(&Block::new(header.clone(), vec![in1.clone()])), + 1 + ); + assert_ok!( + Runtime::ensure_inherents_are_first(&Block::new( + header.clone(), + vec![in1.clone(), xt2.clone()] + ),), + 1 + ); + assert_ok!( + Runtime::ensure_inherents_are_first(&Block::new( + header.clone(), + vec![in2.clone(), in1.clone(), xt2.clone()] + ),), + 2 + ); + + assert_eq!( + Runtime::ensure_inherents_are_first(&Block::new( + header.clone(), + vec![xt2.clone(), in1.clone()] + ),), + Err(1) + ); + assert_eq!( + Runtime::ensure_inherents_are_first(&Block::new( + header.clone(), + vec![xt2.clone(), xt2.clone(), in1.clone()] + ),), + Err(2) + ); + assert_eq!( + Runtime::ensure_inherents_are_first(&Block::new( + header.clone(), + vec![xt2.clone(), xt2.clone(), xt2.clone(), in2.clone()] + ),), + Err(3) + ); + }); +} + +/// Check that block execution rejects blocks with transactions in them while MBMs are active and +/// also that all the system callbacks are called correctly. +#[test] +fn callbacks_in_block_execution_works() { + callbacks_in_block_execution_works_inner(false); + callbacks_in_block_execution_works_inner(true); +} + +fn callbacks_in_block_execution_works_inner(mbms_active: bool) { + MbmActive::set(mbms_active); + + for (n_in, n_tx) in (0..10usize).zip(0..10usize) { + let mut extrinsics = Vec::new(); + + let header = new_test_ext(10).execute_with(|| { + MockedSystemCallbacks::reset(); + Executive::initialize_block(&Header::new_from_number(1)); + assert_eq!(SystemCallbacksCalled::get(), 1); + + for i in 0..n_in { + let xt = if i % 2 == 0 { + TestXt::new(RuntimeCall::Custom(custom::Call::inherent {}), None) + } else { + TestXt::new(RuntimeCall::Custom2(custom2::Call::optional_inherent {}), None) + }; + Executive::apply_extrinsic(xt.clone()).unwrap().unwrap(); + extrinsics.push(xt); + } + + for t in 0..n_tx { + let xt = TestXt::new( + RuntimeCall::Custom2(custom2::Call::some_call {}), + sign_extra(1, t as u64, 0), + ); + // Extrinsics can be applied even when MBMs are active. Only the `execute_block` + // will reject it. + Executive::apply_extrinsic(xt.clone()).unwrap().unwrap(); + extrinsics.push(xt); + } + + Executive::finalize_block() + }); + assert_eq!(SystemCallbacksCalled::get(), 3); + + new_test_ext(10).execute_with(|| { + let header = std::panic::catch_unwind(|| { + Executive::execute_block(Block::new(header, extrinsics)); + }); + + match header { + Err(e) => { + let err = e.downcast::<&str>().unwrap(); + assert_eq!(*err, "Only inherents are allowed in this block"); + assert!( + MbmActive::get() && n_tx > 0, + "Transactions should be rejected when MBMs are active" + ); + }, + Ok(_) => { + assert_eq!(SystemCallbacksCalled::get(), 3); + assert!( + !MbmActive::get() || n_tx == 0, + "MBMs should be deactivated after finalization" + ); + }, + } + }); + } +} + +#[test] +fn post_inherent_called_after_all_inherents() { + let in1 = TestXt::new(RuntimeCall::Custom2(custom2::Call::inherent {}), None); + let xt1 = TestXt::new(RuntimeCall::Custom2(custom2::Call::some_call {}), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(in1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + #[cfg(feature = "try-runtime")] + new_test_ext(1).execute_with(|| { + Executive::try_execute_block( + Block::new(header.clone(), vec![in1.clone(), xt1.clone()]), + true, + true, + frame_try_runtime::TryStateSelect::All, + ) + .unwrap(); + assert!(MockedSystemCallbacks::post_transactions_called()); + }); + + new_test_ext(1).execute_with(|| { + MockedSystemCallbacks::reset(); + Executive::execute_block(Block::new(header, vec![in1, xt1])); + assert!(MockedSystemCallbacks::post_transactions_called()); + }); +} + +/// Regression test for AppSec finding #40. +#[test] +fn post_inherent_called_after_all_optional_inherents() { + let in1 = TestXt::new(RuntimeCall::Custom2(custom2::Call::optional_inherent {}), None); + let xt1 = TestXt::new(RuntimeCall::Custom2(custom2::Call::some_call {}), sign_extra(1, 0, 0)); + + let header = new_test_ext(1).execute_with(|| { + // Let's build some fake block. + Executive::initialize_block(&Header::new_from_number(1)); + + Executive::apply_extrinsic(in1.clone()).unwrap().unwrap(); + Executive::apply_extrinsic(xt1.clone()).unwrap().unwrap(); + + Executive::finalize_block() + }); + + #[cfg(feature = "try-runtime")] + new_test_ext(1).execute_with(|| { + Executive::try_execute_block( + Block::new(header.clone(), vec![in1.clone(), xt1.clone()]), + true, + true, + frame_try_runtime::TryStateSelect::All, + ) + .unwrap(); + assert!(MockedSystemCallbacks::post_transactions_called()); + }); + + new_test_ext(1).execute_with(|| { + MockedSystemCallbacks::reset(); + Executive::execute_block(Block::new(header, vec![in1, xt1])); + assert!(MockedSystemCallbacks::post_transactions_called()); + }); +} + +#[test] +fn is_inherent_works() { + let ext = TestXt::new(RuntimeCall::Custom2(custom2::Call::inherent {}), None); + assert!(Runtime::is_inherent(&ext)); + let ext = TestXt::new(RuntimeCall::Custom2(custom2::Call::optional_inherent {}), None); + assert!(Runtime::is_inherent(&ext)); + + let ext = TestXt::new(call_transfer(33, 0), sign_extra(1, 0, 0)); + assert!(!Runtime::is_inherent(&ext)); + + let ext = TestXt::new(RuntimeCall::Custom2(custom2::Call::allowed_unsigned {}), None); + assert!(!Runtime::is_inherent(&ext), "Unsigned ext are not automatically inherents"); +} diff --git a/substrate/frame/migrations/Cargo.toml b/substrate/frame/migrations/Cargo.toml new file mode 100644 index 000000000000..2914aa9ea973 --- /dev/null +++ b/substrate/frame/migrations/Cargo.toml @@ -0,0 +1,64 @@ +[package] +name = "pallet-migrations" +version = "1.0.0" +description = "FRAME pallet to execute multi-block migrations." +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +docify = "0.1.14" +impl-trait-for-tuples = "0.2.2" +log = "0.4.18" +scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } + +frame-benchmarking = { default-features = false, optional = true, path = "../benchmarking" } +frame-support = { default-features = false, path = "../support" } +frame-system = { default-features = false, path = "../system" } +sp-core = { path = "../../primitives/core", default-features = false } +sp-std = { path = "../../primitives/std", default-features = false } +sp-runtime = { path = "../../primitives/runtime", default-features = false } + +[dev-dependencies] +frame-executive = { path = "../executive" } +sp-api = { path = "../../primitives/api", features = ["std"] } +sp-block-builder = { path = "../../primitives/block-builder", features = ["std"] } +sp-io = { path = "../../primitives/io", features = ["std"] } +sp-tracing = { path = "../../primitives/tracing", features = ["std"] } +sp-version = { path = "../../primitives/version", features = ["std"] } + +pretty_assertions = "1.3.0" + +[features] +default = ["std"] + +std = [ + "codec/std", + "frame-benchmarking?/std", + "frame-support/std", + "frame-system/std", + "log/std", + "scale-info/std", + "sp-core/std", + "sp-runtime/std", + "sp-std/std", +] + +runtime-benchmarks = [ + "frame-benchmarking/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", +] + +try-runtime = [ + "frame-executive/try-runtime", + "frame-support/try-runtime", + "frame-system/try-runtime", + "sp-runtime/try-runtime", +] diff --git a/substrate/frame/migrations/src/benchmarking.rs b/substrate/frame/migrations/src/benchmarking.rs new file mode 100644 index 000000000000..8ad1fa50d149 --- /dev/null +++ b/substrate/frame/migrations/src/benchmarking.rs @@ -0,0 +1,222 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(feature = "runtime-benchmarks")] + +use super::*; + +use frame_benchmarking::{v2::*, BenchmarkError}; +use frame_system::{Pallet as System, RawOrigin}; +use sp_runtime::traits::One; + +fn assert_last_event(generic_event: ::RuntimeEvent) { + frame_system::Pallet::::assert_last_event(generic_event.into()); +} + +#[benchmarks] +mod benches { + use super::*; + use frame_support::traits::Hooks; + + #[benchmark] + fn onboard_new_mbms() { + T::Migrations::set_fail_after(0); // Should not be called anyway. + assert!(!Cursor::::exists()); + + #[block] + { + Pallet::::onboard_new_mbms(); + } + + assert_last_event::(Event::UpgradeStarted { migrations: 1 }.into()); + } + + #[benchmark] + fn progress_mbms_none() { + T::Migrations::set_fail_after(0); // Should not be called anyway. + assert!(!Cursor::::exists()); + + #[block] + { + Pallet::::progress_mbms(One::one()); + } + } + + /// All migrations completed. + #[benchmark] + fn exec_migration_completed() -> Result<(), BenchmarkError> { + T::Migrations::set_fail_after(0); // Should not be called anyway. + assert_eq!(T::Migrations::len(), 1, "Setup failed"); + let c = ActiveCursor { index: 1, inner_cursor: None, started_at: 0u32.into() }; + let mut meter = WeightMeter::with_limit(T::MaxServiceWeight::get()); + System::::set_block_number(1u32.into()); + + #[block] + { + Pallet::::exec_migration(c, false, &mut meter); + } + + assert_last_event::(Event::UpgradeCompleted {}.into()); + + Ok(()) + } + + /// No migration runs since it is skipped as historic. + #[benchmark] + fn exec_migration_skipped_historic() -> Result<(), BenchmarkError> { + T::Migrations::set_fail_after(0); // Should not be called anyway. + assert_eq!(T::Migrations::len(), 1, "Setup failed"); + let c = ActiveCursor { index: 0, inner_cursor: None, started_at: 0u32.into() }; + + let id: IdentifierOf = T::Migrations::nth_id(0).unwrap().try_into().unwrap(); + Historic::::insert(id, ()); + + let mut meter = WeightMeter::with_limit(T::MaxServiceWeight::get()); + System::::set_block_number(1u32.into()); + + #[block] + { + Pallet::::exec_migration(c, false, &mut meter); + } + + assert_last_event::(Event::MigrationSkipped { index: 0 }.into()); + + Ok(()) + } + + /// Advance a migration by one step. + #[benchmark] + fn exec_migration_advance() -> Result<(), BenchmarkError> { + T::Migrations::set_success_after(1); + assert_eq!(T::Migrations::len(), 1, "Setup failed"); + let c = ActiveCursor { index: 0, inner_cursor: None, started_at: 0u32.into() }; + let mut meter = WeightMeter::with_limit(T::MaxServiceWeight::get()); + System::::set_block_number(1u32.into()); + + #[block] + { + Pallet::::exec_migration(c, false, &mut meter); + } + + assert_last_event::(Event::MigrationAdvanced { index: 0, took: One::one() }.into()); + + Ok(()) + } + + /// Successfully complete a migration. + #[benchmark] + fn exec_migration_complete() -> Result<(), BenchmarkError> { + T::Migrations::set_success_after(0); + assert_eq!(T::Migrations::len(), 1, "Setup failed"); + let c = ActiveCursor { index: 0, inner_cursor: None, started_at: 0u32.into() }; + let mut meter = WeightMeter::with_limit(T::MaxServiceWeight::get()); + System::::set_block_number(1u32.into()); + + #[block] + { + Pallet::::exec_migration(c, false, &mut meter); + } + + assert_last_event::(Event::MigrationCompleted { index: 0, took: One::one() }.into()); + + Ok(()) + } + + #[benchmark] + fn exec_migration_fail() -> Result<(), BenchmarkError> { + T::Migrations::set_fail_after(0); + assert_eq!(T::Migrations::len(), 1, "Setup failed"); + let c = ActiveCursor { index: 0, inner_cursor: None, started_at: 0u32.into() }; + let mut meter = WeightMeter::with_limit(T::MaxServiceWeight::get()); + System::::set_block_number(1u32.into()); + + #[block] + { + Pallet::::exec_migration(c, false, &mut meter); + } + + assert_last_event::(Event::UpgradeFailed {}.into()); + + Ok(()) + } + + #[benchmark] + fn on_init_loop() { + T::Migrations::set_fail_after(0); // Should not be called anyway. + System::::set_block_number(1u32.into()); + Pallet::::on_runtime_upgrade(); + + #[block] + { + Pallet::::on_initialize(1u32.into()); + } + } + + #[benchmark] + fn force_set_cursor() { + #[extrinsic_call] + _(RawOrigin::Root, Some(cursor::())); + } + + #[benchmark] + fn force_set_active_cursor() { + #[extrinsic_call] + _(RawOrigin::Root, 0, None, None); + } + + #[benchmark] + fn force_onboard_mbms() { + #[extrinsic_call] + _(RawOrigin::Root); + } + + #[benchmark] + fn clear_historic(n: Linear<0, { DEFAULT_HISTORIC_BATCH_CLEAR_SIZE * 2 }>) { + let id_max_len = ::IdentifierMaxLen::get(); + assert!(id_max_len >= 4, "Precondition violated"); + + for i in 0..DEFAULT_HISTORIC_BATCH_CLEAR_SIZE * 2 { + let id = IdentifierOf::::truncate_from( + i.encode().into_iter().cycle().take(id_max_len as usize).collect::>(), + ); + + Historic::::insert(&id, ()); + } + + #[extrinsic_call] + _( + RawOrigin::Root, + HistoricCleanupSelector::Wildcard { limit: n.into(), previous_cursor: None }, + ); + } + + fn cursor() -> CursorOf { + // Note: The weight of a function can depend on the weight of reading the `inner_cursor`. + // `Cursor` is a user provided type. Now instead of requiring something like `Cursor: + // From`, we instead rely on the fact that it is MEL and the PoV benchmarking will + // therefore already take the MEL bound, even when the cursor in storage is `None`. + MigrationCursor::Active(ActiveCursor { + index: u32::MAX, + inner_cursor: None, + started_at: 0u32.into(), + }) + } + + // Implements a test for each benchmark. Execute with: + // `cargo test -p pallet-migrations --features runtime-benchmarks`. + impl_benchmark_test_suite!(Pallet, crate::mock::new_test_ext(), crate::mock::Test); +} diff --git a/substrate/frame/migrations/src/lib.rs b/substrate/frame/migrations/src/lib.rs new file mode 100644 index 000000000000..cd57d89f440f --- /dev/null +++ b/substrate/frame/migrations/src/lib.rs @@ -0,0 +1,746 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![deny(missing_docs)] +#![deny(rustdoc::broken_intra_doc_links)] + +//! # `pallet-migrations` +//! +//! Provides multi block migrations for FRAME runtimes. +//! +//! ## Overview +//! +//! The pallet takes care of executing a batch of multi-step migrations over multiple blocks. The +//! process starts on each runtime upgrade. Normal and operational transactions are paused while +//! migrations are on-going. +//! +//! ### Example +//! +//! This example demonstrates a simple mocked walk through of a basic success scenario. The pallet +//! is configured with two migrations: one succeeding after just one step, and the second one +//! succeeding after two steps. A runtime upgrade is then enacted and the block number is advanced +//! until all migrations finish executing. Afterwards, the recorded historic migrations are +//! checked and events are asserted. +#![doc = docify::embed!("substrate/frame/migrations/src/tests.rs", simple_works)] +//! +//! ## Pallet API +//! +//! See the [`pallet`] module for more information about the interfaces this pallet exposes, +//! including its configuration trait, dispatchables, storage items, events and errors. +//! +//! Otherwise noteworthy API of this pallet include its implementation of the +//! [`MultiStepMigrator`] trait. This must be plugged into +//! [`frame_system::Config::MultiBlockMigrator`] for proper function. +//! +//! The API contains some calls for emergency management. They are all prefixed with `force_` and +//! should normally not be needed. Pay special attention prior to using them. +//! +//! ### Design Goals +//! +//! 1. Must automatically execute migrations over multiple blocks. +//! 2. Must expose information about whether migrations are ongoing. +//! 3. Must respect pessimistic weight bounds of migrations. +//! 4. Must execute migrations in order. Skipping is not allowed; migrations are run on a +//! all-or-nothing basis. +//! 5. Must prevent re-execution of past migrations. +//! 6. Must provide transactional storage semantics for migrations. +//! 7. Must guarantee progress. +//! +//! ### Design +//! +//! Migrations are provided to the pallet through the associated type [`Config::Migrations`] of type +//! [`SteppedMigrations`]. This allows multiple migrations to be aggregated through a tuple. It +//! simplifies the trait bounds since all associated types of the trait must be provided by the +//! pallet. The actual progress of the pallet is stored in the [`Cursor`] storage item. This can +//! either be [`MigrationCursor::Active`] or [`MigrationCursor::Stuck`]. In the active case it +//! points to the currently active migration and stores its inner cursor. The inner cursor can then +//! be used by the migration to store its inner state and advance. Each time when the migration +//! returns `Some(cursor)`, it signals the pallet that it is not done yet. +//! The cursor is reset on each runtime upgrade. This ensures that it starts to execute at the +//! first migration in the vector. The pallets cursor is only ever incremented or set to `Stuck` +//! once it encounters an error (Goal 4). Once in the stuck state, the pallet will stay stuck until +//! it is fixed through manual governance intervention. +//! As soon as the cursor of the pallet becomes `Some(_)`; [`MultiStepMigrator::ongoing`] returns +//! `true` (Goal 2). This can be used by upstream code to possibly pause transactions. +//! In `on_initialize` the pallet will load the current migration and check whether it was already +//! executed in the past by checking for membership of its ID in the [`Historic`] set. Historic +//! migrations are skipped without causing an error. Each successfully executed migration is added +//! to this set (Goal 5). +//! This proceeds until no more migrations remain. At that point, the event `UpgradeCompleted` is +//! emitted (Goal 1). +//! The execution of each migration happens by calling [`SteppedMigration::transactional_step`]. +//! This function wraps the inner `step` function into a transactional layer to allow rollback in +//! the error case (Goal 6). +//! Weight limits must be checked by the migration itself. The pallet provides a [`WeightMeter`] for +//! that purpose. The pallet may return [`SteppedMigrationError::InsufficientWeight`] at any point. +//! In that scenario, one of two things will happen: if that migration was exclusively executed +//! in this block, and therefore required more than the maximum amount of weight possible, the +//! process becomes `Stuck`. Otherwise, one re-attempt is executed with the same logic in the next +//! block (Goal 3). Progress through the migrations is guaranteed by providing a timeout for each +//! migration via [`SteppedMigration::max_steps`]. The pallet **ONLY** guarantees progress if this +//! is set to sensible limits (Goal 7). +//! +//! ### Scenario: Governance cleanup +//! +//! Every now and then, governance can make use of the [`clear_historic`][Pallet::clear_historic] +//! call. This ensures that no old migrations pile up in the [`Historic`] set. This can be done very +//! rarely, since the storage should not grow quickly and the lookup weight does not suffer much. +//! Another possibility would be to have a synchronous single-block migration perpetually deployed +//! that cleans them up before the MBMs start. +//! +//! ### Scenario: Successful upgrade +//! +//! The standard procedure for a successful runtime upgrade can look like this: +//! 1. Migrations are configured in the `Migrations` config item. All migrations expose +//! [`max_steps`][SteppedMigration::max_steps], are error tolerant, check their weight bounds and +//! have a unique identifier. +//! 2. The runtime upgrade is enacted. An `UpgradeStarted` event is +//! followed by lots of `MigrationAdvanced` and `MigrationCompleted` events. Finally +//! `UpgradeCompleted` is emitted. +//! 3. Cleanup as described in the governance scenario be executed at any time after the migrations +//! completed. +//! +//! ### Advice: Failed upgrades +//! +//! Failed upgrades cannot be recovered from automatically and require governance intervention. Set +//! up monitoring for `UpgradeFailed` events to be made aware of any failures. The hook +//! [`FailedMigrationHandler::failed`] should be setup in a way that it allows governance to act, +//! but still prevent other transactions from interacting with the inconsistent storage state. Note +//! that this is paramount, since the inconsistent state might contain a faulty balance amount or +//! similar that could cause great harm if user transactions don't remain suspended. One way to +//! implement this would be to use the `SafeMode` or `TxPause` pallets that can prevent most user +//! interactions but still allow a whitelisted set of governance calls. +//! +//! ### Remark: Failed migrations +//! +//! Failed migrations are not added to the `Historic` set. This means that an erroneous +//! migration must be removed and fixed manually. This already applies, even before considering the +//! historic set. +//! +//! ### Remark: Transactional processing +//! +//! You can see the transactional semantics for migration steps as mostly useless, since in the +//! stuck case the state is already messed up. This just prevents it from becoming even more messed +//! up, but doesn't prevent it in the first place. + +#![cfg_attr(not(feature = "std"), no_std)] + +mod benchmarking; +mod mock; +pub mod mock_helpers; +mod tests; +pub mod weights; + +pub use pallet::*; +pub use weights::WeightInfo; + +use codec::{Decode, Encode, MaxEncodedLen}; +use core::ops::ControlFlow; +use frame_support::{ + defensive, defensive_assert, + migrations::*, + traits::Get, + weights::{Weight, WeightMeter}, + BoundedVec, +}; +use frame_system::{pallet_prelude::BlockNumberFor, Pallet as System}; +use sp_runtime::Saturating; +use sp_std::vec::Vec; + +/// Points to the next migration to execute. +#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode, scale_info::TypeInfo, MaxEncodedLen)] +pub enum MigrationCursor { + /// Points to the currently active migration and its inner cursor. + Active(ActiveCursor), + + /// Migration got stuck and cannot proceed. This is bad. + Stuck, +} + +impl MigrationCursor { + /// Try to return self as an [`ActiveCursor`]. + pub fn as_active(&self) -> Option<&ActiveCursor> { + match self { + MigrationCursor::Active(active) => Some(active), + MigrationCursor::Stuck => None, + } + } +} + +impl From> + for MigrationCursor +{ + fn from(active: ActiveCursor) -> Self { + MigrationCursor::Active(active) + } +} + +/// Points to the currently active migration and its inner cursor. +#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode, scale_info::TypeInfo, MaxEncodedLen)] +pub struct ActiveCursor { + /// The index of the migration in the MBM tuple. + pub index: u32, + /// The cursor of the migration that is referenced by `index`. + pub inner_cursor: Option, + /// The block number that the migration started at. + /// + /// This is used to calculate how many blocks it took. + pub started_at: BlockNumber, +} + +impl ActiveCursor { + /// Advance the overarching cursor to the next migration. + pub(crate) fn goto_next_migration(&mut self, current_block: BlockNumber) { + self.index.saturating_inc(); + self.inner_cursor = None; + self.started_at = current_block; + } +} + +/// How to clear the records of historic migrations. +#[derive(Debug, Clone, Eq, PartialEq, Encode, Decode, scale_info::TypeInfo)] +pub enum HistoricCleanupSelector { + /// Clear exactly these entries. + /// + /// This is the advised way of doing it. + Specific(Vec), + + /// Clear up to this many entries + Wildcard { + /// How many should be cleared in this call at most. + limit: Option, + /// The cursor that was emitted from any previous `HistoricCleared`. + /// + /// Does not need to be passed when clearing the first batch. + previous_cursor: Option>, + }, +} + +/// The default number of entries that should be cleared by a `HistoricCleanupSelector::Wildcard`. +/// +/// The caller can explicitly specify a higher amount. Benchmarks are run with twice this value. +const DEFAULT_HISTORIC_BATCH_CLEAR_SIZE: u32 = 128; + +impl HistoricCleanupSelector { + /// The maximal number of entries that this will remove. + /// + /// Needed for weight calculation. + pub fn limit(&self) -> u32 { + match self { + Self::Specific(ids) => ids.len() as u32, + Self::Wildcard { limit, .. } => limit.unwrap_or(DEFAULT_HISTORIC_BATCH_CLEAR_SIZE), + } + } +} + +/// Convenience alias for [`MigrationCursor`]. +pub type CursorOf = MigrationCursor, BlockNumberFor>; + +/// Convenience alias for the raw inner cursor of a migration. +pub type RawCursorOf = BoundedVec::CursorMaxLen>; + +/// Convenience alias for the identifier of a migration. +pub type IdentifierOf = BoundedVec::IdentifierMaxLen>; + +/// Convenience alias for [`ActiveCursor`]. +pub type ActiveCursorOf = ActiveCursor, BlockNumberFor>; + +/// Trait for a tuple of No-OP migrations with one element. +pub trait MockedMigrations: SteppedMigrations { + /// The migration should fail after `n` steps. + fn set_fail_after(n: u32); + /// The migration should succeed after `n` steps. + fn set_success_after(n: u32); +} + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config { + /// The overarching event type of the runtime. + type RuntimeEvent: From> + IsType<::RuntimeEvent>; + + /// All the multi-block migrations to run. + /// + /// Should only be updated in a runtime-upgrade once all the old migrations have completed. + /// (Check that [`Cursor`] is `None`). + #[cfg(not(feature = "runtime-benchmarks"))] + type Migrations: SteppedMigrations; + + /// Mocked migrations for benchmarking only. + /// + /// Should be configured to [`crate::mock_helpers::MockedMigrations`] in benchmarks. + #[cfg(feature = "runtime-benchmarks")] + type Migrations: MockedMigrations; + + /// The maximal length of an encoded cursor. + /// + /// A good default needs to selected such that no migration will ever have a cursor with MEL + /// above this limit. This is statically checked in `integrity_test`. + #[pallet::constant] + type CursorMaxLen: Get; + + /// The maximal length of an encoded identifier. + /// + /// A good default needs to selected such that no migration will ever have an identifier + /// with MEL above this limit. This is statically checked in `integrity_test`. + #[pallet::constant] + type IdentifierMaxLen: Get; + + /// Notifications for status updates of a runtime upgrade. + /// + /// Could be used to pause XCM etc. + type MigrationStatusHandler: MigrationStatusHandler; + + /// Handler for failed migrations. + type FailedMigrationHandler: FailedMigrationHandler; + + /// The maximum weight to spend each block to execute migrations. + type MaxServiceWeight: Get; + + /// Weight information for the calls and functions of this pallet. + type WeightInfo: WeightInfo; + } + + /// The currently active migration to run and its cursor. + /// + /// `None` indicates that no migration is running. + #[pallet::storage] + pub type Cursor = StorageValue<_, CursorOf, OptionQuery>; + + /// Set of all successfully executed migrations. + /// + /// This is used as blacklist, to not re-execute migrations that have not been removed from the + /// codebase yet. Governance can regularly clear this out via `clear_historic`. + #[pallet::storage] + pub type Historic = StorageMap<_, Twox64Concat, IdentifierOf, (), OptionQuery>; + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// A Runtime upgrade started. + /// + /// Its end is indicated by `UpgradeCompleted` or `UpgradeFailed`. + UpgradeStarted { + /// The number of migrations that this upgrade contains. + /// + /// This can be used to design a progress indicator in combination with counting the + /// `MigrationCompleted` and `MigrationSkipped` events. + migrations: u32, + }, + /// The current runtime upgrade completed. + /// + /// This implies that all of its migrations completed successfully as well. + UpgradeCompleted, + /// Runtime upgrade failed. + /// + /// This is very bad and will require governance intervention. + UpgradeFailed, + /// A migration was skipped since it was already executed in the past. + MigrationSkipped { + /// The index of the skipped migration within the [`Config::Migrations`] list. + index: u32, + }, + /// A migration progressed. + MigrationAdvanced { + /// The index of the migration within the [`Config::Migrations`] list. + index: u32, + /// The number of blocks that this migration took so far. + took: BlockNumberFor, + }, + /// A Migration completed. + MigrationCompleted { + /// The index of the migration within the [`Config::Migrations`] list. + index: u32, + /// The number of blocks that this migration took so far. + took: BlockNumberFor, + }, + /// A Migration failed. + /// + /// This implies that the whole upgrade failed and governance intervention is required. + MigrationFailed { + /// The index of the migration within the [`Config::Migrations`] list. + index: u32, + /// The number of blocks that this migration took so far. + took: BlockNumberFor, + }, + /// The set of historical migrations has been cleared. + HistoricCleared { + /// Should be passed to `clear_historic` in a successive call. + next_cursor: Option>, + }, + } + + #[pallet::error] + pub enum Error { + /// The operation cannot complete since some MBMs are ongoing. + Ongoing, + } + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_runtime_upgrade() -> Weight { + Self::onboard_new_mbms() + } + + #[cfg(feature = "std")] + fn integrity_test() { + // Check that the migrations tuple is legit. + frame_support::assert_ok!(T::Migrations::integrity_test()); + + // Very important! Ensure that the pallet is configured in `System::Config`. + { + assert!(!Cursor::::exists(), "Externalities storage should be clean"); + assert!(!::MultiBlockMigrator::ongoing()); + + Cursor::::put(MigrationCursor::Stuck); + assert!(::MultiBlockMigrator::ongoing()); + + Cursor::::kill(); + } + + // The per-block service weight is sane. + #[cfg(not(test))] + { + let want = T::MaxServiceWeight::get(); + let max = ::BlockWeights::get().max_block; + + assert!(want.all_lte(max), "Service weight is larger than a block: {want} > {max}",); + } + + // Cursor MEL + { + let mel = T::Migrations::cursor_max_encoded_len(); + let max_mel = T::CursorMaxLen::get() as usize; + assert!( + mel <= max_mel, + "A Cursor is not guaranteed to fit into the storage: {mel} > {max_mel}", + ); + } + + // Identifier MEL + { + let mel = T::Migrations::identifier_max_encoded_len(); + let max_mel = T::IdentifierMaxLen::get() as usize; + assert!( + mel <= max_mel, + "An Identifier is not guaranteed to fit into the storage: {mel} > {max_mel}", + ); + } + } + } + + #[pallet::call(weight = T::WeightInfo)] + impl Pallet { + /// Allows root to set a cursor to forcefully start, stop or forward the migration process. + /// + /// Should normally not be needed and is only in place as emergency measure. Note that + /// restarting the migration process in this manner will not call the + /// [`MigrationStatusHandler::started`] hook or emit an `UpgradeStarted` event. + #[pallet::call_index(0)] + pub fn force_set_cursor( + origin: OriginFor, + cursor: Option>, + ) -> DispatchResult { + ensure_root(origin)?; + + Cursor::::set(cursor); + + Ok(()) + } + + /// Allows root to set an active cursor to forcefully start/forward the migration process. + /// + /// This is an edge-case version of [`Self::force_set_cursor`] that allows to set the + /// `started_at` value to the next block number. Otherwise this would not be possible, since + /// `force_set_cursor` takes an absolute block number. Setting `started_at` to `None` + /// indicates that the current block number plus one should be used. + #[pallet::call_index(1)] + pub fn force_set_active_cursor( + origin: OriginFor, + index: u32, + inner_cursor: Option>, + started_at: Option>, + ) -> DispatchResult { + ensure_root(origin)?; + + let started_at = started_at.unwrap_or( + System::::block_number().saturating_add(sp_runtime::traits::One::one()), + ); + Cursor::::put(MigrationCursor::Active(ActiveCursor { + index, + inner_cursor, + started_at, + })); + + Ok(()) + } + + /// Forces the onboarding of the migrations. + /// + /// This process happens automatically on a runtime upgrade. It is in place as an emergency + /// measurement. The cursor needs to be `None` for this to succeed. + #[pallet::call_index(2)] + pub fn force_onboard_mbms(origin: OriginFor) -> DispatchResult { + ensure_root(origin)?; + + ensure!(!Cursor::::exists(), Error::::Ongoing); + Self::onboard_new_mbms(); + + Ok(()) + } + + /// Clears the `Historic` set. + /// + /// `map_cursor` must be set to the last value that was returned by the + /// `HistoricCleared` event. The first time `None` can be used. `limit` must be chosen in a + /// way that will result in a sensible weight. + #[pallet::call_index(3)] + #[pallet::weight(T::WeightInfo::clear_historic(selector.limit()))] + pub fn clear_historic( + origin: OriginFor, + selector: HistoricCleanupSelector>, + ) -> DispatchResult { + ensure_root(origin)?; + + match &selector { + HistoricCleanupSelector::Specific(ids) => { + for id in ids { + Historic::::remove(id); + } + Self::deposit_event(Event::HistoricCleared { next_cursor: None }); + }, + HistoricCleanupSelector::Wildcard { previous_cursor, .. } => { + let next = Historic::::clear(selector.limit(), previous_cursor.as_deref()); + Self::deposit_event(Event::HistoricCleared { next_cursor: next.maybe_cursor }); + }, + } + + Ok(()) + } + } +} + +impl Pallet { + /// Onboard all new Multi-Block-Migrations and start the process of executing them. + /// + /// Should only be called once all previous migrations completed. + fn onboard_new_mbms() -> Weight { + if let Some(cursor) = Cursor::::get() { + log::error!("Ongoing migrations interrupted - chain stuck"); + + let maybe_index = cursor.as_active().map(|c| c.index); + Self::upgrade_failed(maybe_index); + return T::WeightInfo::onboard_new_mbms() + } + + let migrations = T::Migrations::len(); + log::debug!("Onboarding {migrations} new MBM migrations"); + + if migrations > 0 { + // Set the cursor to the first migration: + Cursor::::set(Some( + ActiveCursor { + index: 0, + inner_cursor: None, + started_at: System::::block_number(), + } + .into(), + )); + Self::deposit_event(Event::UpgradeStarted { migrations }); + T::MigrationStatusHandler::started(); + } + + T::WeightInfo::onboard_new_mbms() + } + + /// Tries to make progress on the Multi-Block-Migrations process. + fn progress_mbms(n: BlockNumberFor) -> Weight { + let mut meter = WeightMeter::with_limit(T::MaxServiceWeight::get()); + meter.consume(T::WeightInfo::progress_mbms_none()); + + let mut cursor = match Cursor::::get() { + None => { + log::trace!("[Block {n:?}] Waiting for cursor to become `Some`."); + return meter.consumed() + }, + Some(MigrationCursor::Active(cursor)) => { + log::debug!("Progressing MBM #{}", cursor.index); + cursor + }, + Some(MigrationCursor::Stuck) => { + log::error!("Migration stuck. Governance intervention required."); + return meter.consumed() + }, + }; + debug_assert!(Self::ongoing()); + + // The limit here is a defensive measure to prevent an infinite loop. It expresses that we + // allow no more than 8 MBMs to finish in a single block. This should be harmless, since we + // generally expect *Multi*-Block-Migrations to take *multiple* blocks. + for i in 0..8 { + match Self::exec_migration(cursor, i == 0, &mut meter) { + None => return meter.consumed(), + Some(ControlFlow::Continue(next_cursor)) => { + cursor = next_cursor; + }, + Some(ControlFlow::Break(last_cursor)) => { + cursor = last_cursor; + break + }, + } + } + + Cursor::::set(Some(cursor.into())); + + meter.consumed() + } + + /// Try to make progress on the current migration. + /// + /// Returns whether processing should continue or break for this block. The return value means: + /// - `None`: The migration process is completely finished. + /// - `ControlFlow::Break`: Continue in the *next* block with the given cursor. + /// - `ControlFlow::Continue`: Continue in the *current* block with the given cursor. + fn exec_migration( + mut cursor: ActiveCursorOf, + is_first: bool, + meter: &mut WeightMeter, + ) -> Option, ActiveCursorOf>> { + // The differences between the single branches' weights is not that big. And since we do + // only one step per block, we can just use the maximum instead of more precise accounting. + if meter.try_consume(Self::exec_migration_max_weight()).is_err() { + defensive_assert!(!is_first, "There should be enough weight to do this at least once"); + return Some(ControlFlow::Break(cursor)) + } + + let Some(id) = T::Migrations::nth_id(cursor.index) else { + // No more migrations in the tuple - we are done. + defensive_assert!(cursor.index == T::Migrations::len(), "Inconsistent MBMs tuple"); + Self::deposit_event(Event::UpgradeCompleted); + Cursor::::kill(); + T::MigrationStatusHandler::completed(); + return None; + }; + + let Ok(bounded_id): Result, _> = id.try_into() else { + defensive!("integrity_test ensures that all identifiers' MEL bounds fit into CursorMaxLen; qed."); + Self::upgrade_failed(Some(cursor.index)); + return None + }; + + if Historic::::contains_key(&bounded_id) { + Self::deposit_event(Event::MigrationSkipped { index: cursor.index }); + cursor.goto_next_migration(System::::block_number()); + return Some(ControlFlow::Continue(cursor)) + } + + let max_steps = T::Migrations::nth_max_steps(cursor.index); + let next_cursor = T::Migrations::nth_transactional_step( + cursor.index, + cursor.inner_cursor.clone().map(|c| c.into_inner()), + meter, + ); + let Some((max_steps, next_cursor)) = max_steps.zip(next_cursor) else { + defensive!("integrity_test ensures that the tuple is valid; qed"); + Self::upgrade_failed(Some(cursor.index)); + return None + }; + + let took = System::::block_number().saturating_sub(cursor.started_at); + match next_cursor { + Ok(Some(next_cursor)) => { + let Ok(bound_next_cursor) = next_cursor.try_into() else { + defensive!("The integrity check ensures that all cursors' MEL bound fits into CursorMaxLen; qed"); + Self::upgrade_failed(Some(cursor.index)); + return None + }; + + Self::deposit_event(Event::MigrationAdvanced { index: cursor.index, took }); + cursor.inner_cursor = Some(bound_next_cursor); + + if max_steps.map_or(false, |max| took > max.into()) { + Self::deposit_event(Event::MigrationFailed { index: cursor.index, took }); + Self::upgrade_failed(Some(cursor.index)); + None + } else { + // A migration cannot progress more than one step per block, we therefore break. + Some(ControlFlow::Break(cursor)) + } + }, + Ok(None) => { + // A migration is done when it returns cursor `None`. + Self::deposit_event(Event::MigrationCompleted { index: cursor.index, took }); + Historic::::insert(&bounded_id, ()); + cursor.goto_next_migration(System::::block_number()); + Some(ControlFlow::Continue(cursor)) + }, + Err(SteppedMigrationError::InsufficientWeight { required }) => { + if is_first || required.any_gt(meter.limit()) { + Self::deposit_event(Event::MigrationFailed { index: cursor.index, took }); + Self::upgrade_failed(Some(cursor.index)); + None + } else { + // Retry and hope that there is more weight in the next block. + Some(ControlFlow::Break(cursor)) + } + }, + Err(SteppedMigrationError::InvalidCursor | SteppedMigrationError::Failed) => { + Self::deposit_event(Event::MigrationFailed { index: cursor.index, took }); + Self::upgrade_failed(Some(cursor.index)); + None + }, + } + } + + /// Fail the current runtime upgrade, caused by `migration`. + fn upgrade_failed(migration: Option) { + use FailedMigrationHandling::*; + Self::deposit_event(Event::UpgradeFailed); + + match T::FailedMigrationHandler::failed(migration) { + KeepStuck => Cursor::::set(Some(MigrationCursor::Stuck)), + ForceUnstuck => Cursor::::kill(), + Ignore => {}, + } + } + + fn exec_migration_max_weight() -> Weight { + T::WeightInfo::exec_migration_complete() + .max(T::WeightInfo::exec_migration_completed()) + .max(T::WeightInfo::exec_migration_skipped_historic()) + .max(T::WeightInfo::exec_migration_advance()) + .max(T::WeightInfo::exec_migration_fail()) + } +} + +impl MultiStepMigrator for Pallet { + fn ongoing() -> bool { + Cursor::::exists() + } + + fn step() -> Weight { + Self::progress_mbms(System::::block_number()) + } +} diff --git a/substrate/frame/migrations/src/mock.rs b/substrate/frame/migrations/src/mock.rs new file mode 100644 index 000000000000..372926475544 --- /dev/null +++ b/substrate/frame/migrations/src/mock.rs @@ -0,0 +1,163 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Mocked runtime for testing the migrations pallet. + +#![cfg(test)] + +use crate::{mock_helpers::*, Event, Historic}; + +use frame_support::{ + derive_impl, + migrations::*, + traits::{OnFinalize, OnInitialize}, + weights::Weight, +}; +use frame_system::EventRecord; +use sp_core::{ConstU32, H256}; + +type Block = frame_system::mocking::MockBlock; + +// Configure a mock runtime to test the pallet. +frame_support::construct_runtime!( + pub enum Test { + System: frame_system, + Migrations: crate, + } +); + +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] +impl frame_system::Config for Test { + type Block = Block; + type PalletInfo = PalletInfo; + type MultiBlockMigrator = Migrations; +} + +frame_support::parameter_types! { + pub const MaxServiceWeight: Weight = Weight::MAX.div(10); +} + +impl crate::Config for Test { + type RuntimeEvent = RuntimeEvent; + type Migrations = MockedMigrations; + type CursorMaxLen = ConstU32<65_536>; + type IdentifierMaxLen = ConstU32<256>; + type MigrationStatusHandler = MockedMigrationStatusHandler; + type FailedMigrationHandler = MockedFailedMigrationHandler; + type MaxServiceWeight = MaxServiceWeight; + type WeightInfo = (); +} + +frame_support::parameter_types! { + /// The number of started upgrades. + pub static UpgradesStarted: u32 = 0; + /// The number of completed upgrades. + pub static UpgradesCompleted: u32 = 0; + /// The migrations that failed. + pub static UpgradesFailed: Vec> = vec![]; + /// Return value of [`MockedFailedMigrationHandler::failed`]. + pub static FailedUpgradeResponse: FailedMigrationHandling = FailedMigrationHandling::KeepStuck; +} + +/// Records all started and completed upgrades in `UpgradesStarted` and `UpgradesCompleted`. +pub struct MockedMigrationStatusHandler; +impl MigrationStatusHandler for MockedMigrationStatusHandler { + fn started() { + log::info!("MigrationStatusHandler started"); + UpgradesStarted::mutate(|v| *v += 1); + } + + fn completed() { + log::info!("MigrationStatusHandler completed"); + UpgradesCompleted::mutate(|v| *v += 1); + } +} + +/// Records all failed upgrades in `UpgradesFailed`. +pub struct MockedFailedMigrationHandler; +impl FailedMigrationHandler for MockedFailedMigrationHandler { + fn failed(migration: Option) -> FailedMigrationHandling { + UpgradesFailed::mutate(|v| v.push(migration)); + let res = FailedUpgradeResponse::get(); + log::error!("FailedMigrationHandler failed at: {migration:?}, handling as {res:?}"); + res + } +} + +/// Returns the number of `(started, completed, failed)` upgrades and resets their numbers. +pub fn upgrades_started_completed_failed() -> (u32, u32, u32) { + (UpgradesStarted::take(), UpgradesCompleted::take(), UpgradesFailed::take().len() as u32) +} + +/// Build genesis storage according to the mock runtime. +pub fn new_test_ext() -> sp_io::TestExternalities { + sp_io::TestExternalities::new(Default::default()) +} + +/// Run this closure in test externalities. +pub fn test_closure(f: impl FnOnce() -> R) -> R { + let mut ext = new_test_ext(); + ext.execute_with(f) +} + +pub fn run_to_block(n: u32) { + while System::block_number() < n as u64 { + log::debug!("Block {}", System::block_number()); + System::set_block_number(System::block_number() + 1); + System::on_initialize(System::block_number()); + Migrations::on_initialize(System::block_number()); + // Executive calls this: + ::step(); + + Migrations::on_finalize(System::block_number()); + System::on_finalize(System::block_number()); + } +} + +/// Returns the historic migrations, sorted by their identifier. +pub fn historic() -> Vec { + let mut historic = Historic::::iter_keys().collect::>(); + historic.sort(); + historic +} + +// Traits to make using events less insufferable: +pub trait IntoRecord { + fn into_record(self) -> EventRecord<::RuntimeEvent, H256>; +} + +impl IntoRecord for Event { + fn into_record(self) -> EventRecord<::RuntimeEvent, H256> { + let re: ::RuntimeEvent = self.into(); + EventRecord { phase: frame_system::Phase::Initialization, event: re, topics: vec![] } + } +} + +pub trait IntoRecords { + fn into_records(self) -> Vec::RuntimeEvent, H256>>; +} + +impl IntoRecords for Vec { + fn into_records(self) -> Vec::RuntimeEvent, H256>> { + self.into_iter().map(|e| e.into_record()).collect() + } +} + +pub fn assert_events(events: Vec) { + pretty_assertions::assert_eq!(events.into_records(), System::events()); + System::reset_events(); +} diff --git a/substrate/frame/migrations/src/mock_helpers.rs b/substrate/frame/migrations/src/mock_helpers.rs new file mode 100644 index 000000000000..c5e23efb4e31 --- /dev/null +++ b/substrate/frame/migrations/src/mock_helpers.rs @@ -0,0 +1,142 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test helpers for internal and external usage. + +#![allow(missing_docs)] + +use codec::{Decode, Encode}; +use frame_support::{ + migrations::*, + weights::{Weight, WeightMeter}, +}; +use sp_core::ConstU32; +use sp_runtime::BoundedVec; +use sp_std::{vec, vec::Vec}; + +/// Opaque identifier of a migration. +pub type MockedIdentifier = BoundedVec>; + +/// How a mocked migration should behave. +#[derive(Debug, Clone, Copy, Encode, Decode)] +pub enum MockedMigrationKind { + /// Succeed after its number of steps elapsed. + SucceedAfter, + /// Fail after its number of steps elapsed. + FailAfter, + /// Never terminate. + TimeoutAfter, + /// Cause an [`SteppedMigrationError::InsufficientWeight`] error after its number of steps + /// elapsed. + HighWeightAfter(Weight), +} +use MockedMigrationKind::*; // C style + +/// Creates a migration identifier with a specific `kind` and `steps`. +pub fn mocked_id(kind: MockedMigrationKind, steps: u32) -> MockedIdentifier { + (b"MockedMigration", kind, steps).encode().try_into().unwrap() +} + +frame_support::parameter_types! { + /// The configs for the migrations to run. + storage MIGRATIONS: Vec<(MockedMigrationKind, u32)> = vec![]; +} + +/// Allows to set the migrations to run at runtime instead of compile-time. +/// +/// It achieves this by using the storage to store the migrations to run. +pub struct MockedMigrations; +impl SteppedMigrations for MockedMigrations { + fn len() -> u32 { + MIGRATIONS::get().len() as u32 + } + + fn nth_id(n: u32) -> Option> { + let k = MIGRATIONS::get().get(n as usize).copied(); + k.map(|(kind, steps)| mocked_id(kind, steps).into_inner()) + } + + fn nth_step( + n: u32, + cursor: Option>, + _meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + let (kind, steps) = MIGRATIONS::get()[n as usize]; + + let mut count: u32 = + cursor.as_ref().and_then(|c| Decode::decode(&mut &c[..]).ok()).unwrap_or(0); + log::debug!("MockedMigration: Step {}", count); + if count != steps || matches!(kind, TimeoutAfter) { + count += 1; + return Some(Ok(Some(count.encode()))) + } + + Some(match kind { + SucceedAfter => { + log::debug!("MockedMigration: Succeeded after {} steps", count); + Ok(None) + }, + HighWeightAfter(required) => { + log::debug!("MockedMigration: Not enough weight after {} steps", count); + Err(SteppedMigrationError::InsufficientWeight { required }) + }, + FailAfter => { + log::debug!("MockedMigration: Failed after {} steps", count); + Err(SteppedMigrationError::Failed) + }, + TimeoutAfter => unreachable!(), + }) + } + + fn nth_transactional_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + // This is a hack but should be fine. We dont need it in testing. + Self::nth_step(n, cursor, meter) + } + + fn nth_max_steps(n: u32) -> Option> { + MIGRATIONS::get().get(n as usize).map(|(_, s)| Some(*s)) + } + + fn cursor_max_encoded_len() -> usize { + 65_536 + } + + fn identifier_max_encoded_len() -> usize { + 256 + } +} + +impl MockedMigrations { + /// Set the migrations to run. + pub fn set(migrations: Vec<(MockedMigrationKind, u32)>) { + MIGRATIONS::set(&migrations); + } +} + +impl crate::MockedMigrations for MockedMigrations { + fn set_fail_after(steps: u32) { + MIGRATIONS::set(&vec![(FailAfter, steps)]); + } + + fn set_success_after(steps: u32) { + MIGRATIONS::set(&vec![(SucceedAfter, steps)]); + } +} diff --git a/substrate/frame/migrations/src/tests.rs b/substrate/frame/migrations/src/tests.rs new file mode 100644 index 000000000000..9c9043d37a62 --- /dev/null +++ b/substrate/frame/migrations/src/tests.rs @@ -0,0 +1,335 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(test)] + +use crate::{ + mock::{Test as T, *}, + mock_helpers::{MockedMigrationKind::*, *}, + Cursor, Event, FailedMigrationHandling, MigrationCursor, +}; +use frame_support::{pallet_prelude::Weight, traits::OnRuntimeUpgrade}; + +#[docify::export] +#[test] +fn simple_works() { + use Event::*; + test_closure(|| { + // Add three migrations, each taking one block longer than the previous. + MockedMigrations::set(vec![(SucceedAfter, 0), (SucceedAfter, 1), (SucceedAfter, 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Check that the executed migrations are recorded in `Historical`. + assert_eq!( + historic(), + vec![ + mocked_id(SucceedAfter, 0), + mocked_id(SucceedAfter, 1), + mocked_id(SucceedAfter, 2), + ] + ); + + // Check that we got all events. + assert_events(vec![ + UpgradeStarted { migrations: 3 }, + MigrationCompleted { index: 0, took: 1 }, + MigrationAdvanced { index: 1, took: 0 }, + MigrationCompleted { index: 1, took: 1 }, + MigrationAdvanced { index: 2, took: 0 }, + MigrationAdvanced { index: 2, took: 1 }, + MigrationCompleted { index: 2, took: 2 }, + UpgradeCompleted, + ]); + }); +} + +#[test] +fn failing_migration_sets_cursor_to_stuck() { + test_closure(|| { + FailedUpgradeResponse::set(FailedMigrationHandling::KeepStuck); + MockedMigrations::set(vec![(FailAfter, 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Failed migrations are not recorded in `Historical`. + assert!(historic().is_empty()); + // Check that we got all events. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, took: 1 }, + Event::MigrationAdvanced { index: 0, took: 2 }, + Event::MigrationFailed { index: 0, took: 3 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(UpgradesStarted::take(), 1); + assert_eq!(UpgradesCompleted::take(), 0); + assert_eq!(UpgradesFailed::take(), vec![Some(0)]); + + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck), "Must stuck the chain"); + }); +} + +#[test] +fn failing_migration_force_unstuck_works() { + test_closure(|| { + FailedUpgradeResponse::set(FailedMigrationHandling::ForceUnstuck); + MockedMigrations::set(vec![(FailAfter, 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Failed migrations are not recorded in `Historical`. + assert!(historic().is_empty()); + // Check that we got all events. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, took: 1 }, + Event::MigrationAdvanced { index: 0, took: 2 }, + Event::MigrationFailed { index: 0, took: 3 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(UpgradesStarted::take(), 1); + assert_eq!(UpgradesCompleted::take(), 0); + assert_eq!(UpgradesFailed::take(), vec![Some(0)]); + + assert!(Cursor::::get().is_none(), "Must unstuck the chain"); + }); +} + +/// A migration that reports not getting enough weight errors if it is the first one to run in that +/// block. +#[test] +fn high_weight_migration_singular_fails() { + test_closure(|| { + MockedMigrations::set(vec![(HighWeightAfter(Weight::zero()), 2)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Failed migrations are not recorded in `Historical`. + assert!(historic().is_empty()); + // Check that we got all events. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, took: 1 }, + Event::MigrationAdvanced { index: 0, took: 2 }, + Event::MigrationFailed { index: 0, took: 3 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(upgrades_started_completed_failed(), (1, 0, 1)); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + }); +} + +/// A migration that reports of not getting enough weight is retried once, if it is not the first +/// one to run in a block. +#[test] +fn high_weight_migration_retries_once() { + test_closure(|| { + MockedMigrations::set(vec![(SucceedAfter, 0), (HighWeightAfter(Weight::zero()), 0)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + assert_eq!(historic(), vec![mocked_id(SucceedAfter, 0)]); + // Check that we got all events. + assert_events::>(vec![ + Event::UpgradeStarted { migrations: 2 }, + Event::MigrationCompleted { index: 0, took: 1 }, + // `took=1` means that it was retried once. + Event::MigrationFailed { index: 1, took: 1 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(upgrades_started_completed_failed(), (1, 0, 1)); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + }); +} + +/// If a migration uses more weight than the limit, then it will not retry but fail even when it is +/// not the first one in the block. +// Note: Same as `high_weight_migration_retries_once` but with different required weight for the +// migration. +#[test] +fn high_weight_migration_permanently_overweight_fails() { + test_closure(|| { + MockedMigrations::set(vec![(SucceedAfter, 0), (HighWeightAfter(Weight::MAX), 0)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + assert_eq!(historic(), vec![mocked_id(SucceedAfter, 0)]); + // Check that we got all events. + assert_events::>(vec![ + Event::UpgradeStarted { migrations: 2 }, + Event::MigrationCompleted { index: 0, took: 1 }, + // `blocks=0` means that it was not retried. + Event::MigrationFailed { index: 1, took: 0 }, + Event::UpgradeFailed, + ]); + + // Check that the handler was called correctly. + assert_eq!(upgrades_started_completed_failed(), (1, 0, 1)); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + }); +} + +#[test] +fn historic_skipping_works() { + test_closure(|| { + MockedMigrations::set(vec![ + (SucceedAfter, 0), + (SucceedAfter, 0), // duplicate + (SucceedAfter, 1), + (SucceedAfter, 2), + (SucceedAfter, 1), // duplicate + ]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(10); + + // Just three historical ones, since two were added twice. + assert_eq!( + historic(), + vec![ + mocked_id(SucceedAfter, 0), + mocked_id(SucceedAfter, 1), + mocked_id(SucceedAfter, 2), + ] + ); + // Events received. + assert_events(vec![ + Event::UpgradeStarted { migrations: 5 }, + Event::MigrationCompleted { index: 0, took: 1 }, + Event::MigrationSkipped { index: 1 }, + Event::MigrationAdvanced { index: 2, took: 0 }, + Event::MigrationCompleted { index: 2, took: 1 }, + Event::MigrationAdvanced { index: 3, took: 0 }, + Event::MigrationAdvanced { index: 3, took: 1 }, + Event::MigrationCompleted { index: 3, took: 2 }, + Event::MigrationSkipped { index: 4 }, + Event::UpgradeCompleted, + ]); + assert_eq!(upgrades_started_completed_failed(), (1, 1, 0)); + + // Now go for another upgrade; just to make sure that it wont execute again. + System::reset_events(); + Migrations::on_runtime_upgrade(); + run_to_block(20); + + // Same historical ones as before. + assert_eq!( + historic(), + vec![ + mocked_id(SucceedAfter, 0), + mocked_id(SucceedAfter, 1), + mocked_id(SucceedAfter, 2), + ] + ); + + // Everything got skipped. + assert_events(vec![ + Event::UpgradeStarted { migrations: 5 }, + Event::MigrationSkipped { index: 0 }, + Event::MigrationSkipped { index: 1 }, + Event::MigrationSkipped { index: 2 }, + Event::MigrationSkipped { index: 3 }, + Event::MigrationSkipped { index: 4 }, + Event::UpgradeCompleted, + ]); + assert_eq!(upgrades_started_completed_failed(), (1, 1, 0)); + }); +} + +/// When another upgrade happens while a migration is still running, it should set the cursor to +/// stuck. +#[test] +fn upgrade_fails_when_migration_active() { + test_closure(|| { + MockedMigrations::set(vec![(SucceedAfter, 10)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(3); + + // Events received. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, took: 1 }, + Event::MigrationAdvanced { index: 0, took: 2 }, + ]); + assert_eq!(upgrades_started_completed_failed(), (1, 0, 0)); + + // Upgrade again. + Migrations::on_runtime_upgrade(); + // -- Defensive path -- + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + assert_events(vec![Event::UpgradeFailed]); + assert_eq!(upgrades_started_completed_failed(), (0, 0, 1)); + }); +} + +#[test] +fn migration_timeout_errors() { + test_closure(|| { + MockedMigrations::set(vec![(TimeoutAfter, 3)]); + + System::set_block_number(1); + Migrations::on_runtime_upgrade(); + run_to_block(5); + + // Times out after taking more than 3 steps. + assert_events(vec![ + Event::UpgradeStarted { migrations: 1 }, + Event::MigrationAdvanced { index: 0, took: 1 }, + Event::MigrationAdvanced { index: 0, took: 2 }, + Event::MigrationAdvanced { index: 0, took: 3 }, + Event::MigrationAdvanced { index: 0, took: 4 }, + Event::MigrationFailed { index: 0, took: 4 }, + Event::UpgradeFailed, + ]); + assert_eq!(upgrades_started_completed_failed(), (1, 0, 1)); + + // Failed migrations are not black-listed. + assert!(historic().is_empty()); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + + Migrations::on_runtime_upgrade(); + run_to_block(6); + + assert_events(vec![Event::UpgradeFailed]); + assert_eq!(Cursor::::get(), Some(MigrationCursor::Stuck)); + assert_eq!(upgrades_started_completed_failed(), (0, 0, 1)); + }); +} diff --git a/substrate/frame/migrations/src/weights.rs b/substrate/frame/migrations/src/weights.rs new file mode 100644 index 000000000000..c9b63258c44b --- /dev/null +++ b/substrate/frame/migrations/src/weights.rs @@ -0,0 +1,358 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated weights for `pallet_migrations` +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2023-12-04, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! WORST CASE MAP SIZE: `1000000` +//! HOSTNAME: `loud1`, CPU: `AMD EPYC 7282 16-Core Processor` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("dev")`, DB CACHE: `1024` + +// Executed Command: +// target/release/substrate-node +// benchmark +// pallet +// --chain +// dev +// --pallet +// pallet-migrations +// --extrinsic +// +// --output +// weight.rs +// --template +// ../../polkadot-sdk/substrate/.maintain/frame-weight-template.hbs + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(missing_docs)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use core::marker::PhantomData; + +/// Weight functions needed for `pallet_migrations`. +pub trait WeightInfo { + fn onboard_new_mbms() -> Weight; + fn progress_mbms_none() -> Weight; + fn exec_migration_completed() -> Weight; + fn exec_migration_skipped_historic() -> Weight; + fn exec_migration_advance() -> Weight; + fn exec_migration_complete() -> Weight; + fn exec_migration_fail() -> Weight; + fn on_init_loop() -> Weight; + fn force_set_cursor() -> Weight; + fn force_set_active_cursor() -> Weight; + fn force_onboard_mbms() -> Weight; + fn clear_historic(n: u32, ) -> Weight; +} + +/// Weights for `pallet_migrations` using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + /// Storage: `MultiBlockMigrations::Cursor` (r:1 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + fn onboard_new_mbms() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `67035` + // Minimum execution time: 13_980_000 picoseconds. + Weight::from_parts(14_290_000, 67035) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn progress_mbms_none() -> Weight { + // Proof Size summary in bytes: + // Measured: `109` + // Estimated: `67035` + // Minimum execution time: 3_770_000 picoseconds. + Weight::from_parts(4_001_000, 67035) + .saturating_add(T::DbWeight::get().reads(1_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn exec_migration_completed() -> Weight { + // Proof Size summary in bytes: + // Measured: `134` + // Estimated: `3599` + // Minimum execution time: 10_900_000 picoseconds. + Weight::from_parts(11_251_000, 3599) + .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + fn exec_migration_skipped_historic() -> Weight { + // Proof Size summary in bytes: + // Measured: `297` + // Estimated: `3762` + // Minimum execution time: 17_891_000 picoseconds. + Weight::from_parts(18_501_000, 3762) + .saturating_add(T::DbWeight::get().reads(2_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + fn exec_migration_advance() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `3731` + // Minimum execution time: 18_271_000 picoseconds. + Weight::from_parts(18_740_000, 3731) + .saturating_add(T::DbWeight::get().reads(2_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:1) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + fn exec_migration_complete() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `3731` + // Minimum execution time: 21_241_000 picoseconds. + Weight::from_parts(21_911_000, 3731) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn exec_migration_fail() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `3731` + // Minimum execution time: 22_740_000 picoseconds. + Weight::from_parts(23_231_000, 3731) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + fn on_init_loop() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 440_000 picoseconds. + Weight::from_parts(500_000, 0) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn force_set_cursor() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 5_751_000 picoseconds. + Weight::from_parts(5_950_000, 0) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn force_set_active_cursor() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 6_350_000 picoseconds. + Weight::from_parts(6_560_000, 0) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + fn force_onboard_mbms() -> Weight { + // Proof Size summary in bytes: + // Measured: `218` + // Estimated: `67035` + // Minimum execution time: 11_121_000 picoseconds. + Weight::from_parts(11_530_000, 67035) + .saturating_add(T::DbWeight::get().reads(2_u64)) + } + /// Storage: `MultiBlockMigrations::Historic` (r:256 w:256) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + /// The range of component `n` is `[0, 256]`. + fn clear_historic(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `1089 + n * (271 ±0)` + // Estimated: `3834 + n * (2740 ±0)` + // Minimum execution time: 21_891_000 picoseconds. + Weight::from_parts(18_572_306, 3834) + // Standard Error: 3_236 + .saturating_add(Weight::from_parts(1_648_429, 0).saturating_mul(n.into())) + .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) + .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2740).saturating_mul(n.into())) + } +} + +// For backwards compatibility and tests. +impl WeightInfo for () { + /// Storage: `MultiBlockMigrations::Cursor` (r:1 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + fn onboard_new_mbms() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `67035` + // Minimum execution time: 13_980_000 picoseconds. + Weight::from_parts(14_290_000, 67035) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn progress_mbms_none() -> Weight { + // Proof Size summary in bytes: + // Measured: `109` + // Estimated: `67035` + // Minimum execution time: 3_770_000 picoseconds. + Weight::from_parts(4_001_000, 67035) + .saturating_add(RocksDbWeight::get().reads(1_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn exec_migration_completed() -> Weight { + // Proof Size summary in bytes: + // Measured: `134` + // Estimated: `3599` + // Minimum execution time: 10_900_000 picoseconds. + Weight::from_parts(11_251_000, 3599) + .saturating_add(RocksDbWeight::get().reads(1_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + fn exec_migration_skipped_historic() -> Weight { + // Proof Size summary in bytes: + // Measured: `297` + // Estimated: `3762` + // Minimum execution time: 17_891_000 picoseconds. + Weight::from_parts(18_501_000, 3762) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + fn exec_migration_advance() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `3731` + // Minimum execution time: 18_271_000 picoseconds. + Weight::from_parts(18_740_000, 3731) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:1) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + fn exec_migration_complete() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `3731` + // Minimum execution time: 21_241_000 picoseconds. + Weight::from_parts(21_911_000, 3731) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Storage: `MultiBlockMigrations::Historic` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn exec_migration_fail() -> Weight { + // Proof Size summary in bytes: + // Measured: `243` + // Estimated: `3731` + // Minimum execution time: 22_740_000 picoseconds. + Weight::from_parts(23_231_000, 3731) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + fn on_init_loop() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 440_000 picoseconds. + Weight::from_parts(500_000, 0) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn force_set_cursor() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 5_751_000 picoseconds. + Weight::from_parts(5_950_000, 0) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:0 w:1) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + fn force_set_active_cursor() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 6_350_000 picoseconds. + Weight::from_parts(6_560_000, 0) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: `MultiBlockMigrations::Cursor` (r:1 w:0) + /// Proof: `MultiBlockMigrations::Cursor` (`max_values`: Some(1), `max_size`: Some(65550), added: 66045, mode: `MaxEncodedLen`) + /// Storage: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + /// Proof: UNKNOWN KEY `0x583359fe0e84d953a9dd84e8addb08a5` (r:1 w:0) + fn force_onboard_mbms() -> Weight { + // Proof Size summary in bytes: + // Measured: `218` + // Estimated: `67035` + // Minimum execution time: 11_121_000 picoseconds. + Weight::from_parts(11_530_000, 67035) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + } + /// Storage: `MultiBlockMigrations::Historic` (r:256 w:256) + /// Proof: `MultiBlockMigrations::Historic` (`max_values`: None, `max_size`: Some(266), added: 2741, mode: `MaxEncodedLen`) + /// The range of component `n` is `[0, 256]`. + fn clear_historic(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `1089 + n * (271 ±0)` + // Estimated: `3834 + n * (2740 ±0)` + // Minimum execution time: 21_891_000 picoseconds. + Weight::from_parts(18_572_306, 3834) + // Standard Error: 3_236 + .saturating_add(Weight::from_parts(1_648_429, 0).saturating_mul(n.into())) + .saturating_add(RocksDbWeight::get().reads(1_u64)) + .saturating_add(RocksDbWeight::get().reads((1_u64).saturating_mul(n.into()))) + .saturating_add(RocksDbWeight::get().writes((1_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2740).saturating_mul(n.into())) + } +} diff --git a/substrate/frame/src/lib.rs b/substrate/frame/src/lib.rs index a2e5d726fdd0..52db7c34bfdc 100644 --- a/substrate/frame/src/lib.rs +++ b/substrate/frame/src/lib.rs @@ -197,7 +197,7 @@ pub mod runtime { // Types often used in the runtime APIs. pub use sp_core::OpaqueMetadata; pub use sp_inherents::{CheckInherentsResult, InherentData}; - pub use sp_runtime::ApplyExtrinsicResult; + pub use sp_runtime::{ApplyExtrinsicResult, ExtrinsicInclusionMode}; pub use frame_system_rpc_runtime_api::*; pub use sp_api::{self, *}; diff --git a/substrate/frame/support/procedural/src/construct_runtime/expand/inherent.rs b/substrate/frame/support/procedural/src/construct_runtime/expand/inherent.rs index 34b9d21d8ce8..da483fa6cf0b 100644 --- a/substrate/frame/support/procedural/src/construct_runtime/expand/inherent.rs +++ b/substrate/frame/support/procedural/src/construct_runtime/expand/inherent.rs @@ -204,47 +204,50 @@ pub fn expand_outer_inherent( } } + impl #scrate::traits::IsInherent<<#block as #scrate::sp_runtime::traits::Block>::Extrinsic> for #runtime { + fn is_inherent(ext: &<#block as #scrate::sp_runtime::traits::Block>::Extrinsic) -> bool { + use #scrate::inherent::ProvideInherent; + use #scrate::traits::{IsSubType, ExtrinsicCall}; + + if #scrate::sp_runtime::traits::Extrinsic::is_signed(ext).unwrap_or(false) { + // Signed extrinsics are never inherents. + return false + } + + #( + #pallet_attrs + { + let call = <#unchecked_extrinsic as ExtrinsicCall>::call(ext); + if let Some(call) = IsSubType::<_>::is_sub_type(call) { + if <#pallet_names as ProvideInherent>::is_inherent(&call) { + return true; + } + } + } + )* + false + } + } + impl #scrate::traits::EnsureInherentsAreFirst<#block> for #runtime { - fn ensure_inherents_are_first(block: &#block) -> Result<(), u32> { + fn ensure_inherents_are_first(block: &#block) -> Result { use #scrate::inherent::ProvideInherent; use #scrate::traits::{IsSubType, ExtrinsicCall}; use #scrate::sp_runtime::traits::Block as _; - let mut first_signed_observed = false; + let mut num_inherents = 0u32; for (i, xt) in block.extrinsics().iter().enumerate() { - let is_signed = #scrate::sp_runtime::traits::Extrinsic::is_signed(xt) - .unwrap_or(false); - - let is_inherent = if is_signed { - // Signed extrinsics are not inherents. - false - } else { - let mut is_inherent = false; - #( - #pallet_attrs - { - let call = <#unchecked_extrinsic as ExtrinsicCall>::call(xt); - if let Some(call) = IsSubType::<_>::is_sub_type(call) { - if #pallet_names::is_inherent(&call) { - is_inherent = true; - } - } - } - )* - is_inherent - }; - - if !is_inherent { - first_signed_observed = true; - } + if >::is_inherent(xt) { + if num_inherents != i as u32 { + return Err(i as u32); + } - if first_signed_observed && is_inherent { - return Err(i as u32) + num_inherents += 1; // Safe since we are in an `enumerate` loop. } } - Ok(()) + Ok(num_inherents) } } } diff --git a/substrate/frame/support/procedural/src/pallet/expand/hooks.rs b/substrate/frame/support/procedural/src/pallet/expand/hooks.rs index 6b25ddcba1a7..3623b595268d 100644 --- a/substrate/frame/support/procedural/src/pallet/expand/hooks.rs +++ b/substrate/frame/support/procedural/src/pallet/expand/hooks.rs @@ -175,6 +175,22 @@ pub fn expand_hooks(def: &mut Def) -> proc_macro2::TokenStream { } } + impl<#type_impl_gen> + #frame_support::traits::OnPoll<#frame_system::pallet_prelude::BlockNumberFor::> + for #pallet_ident<#type_use_gen> #where_clause + { + fn on_poll( + n: #frame_system::pallet_prelude::BlockNumberFor::, + weight: &mut #frame_support::weights::WeightMeter + ) { + < + Self as #frame_support::traits::Hooks< + #frame_system::pallet_prelude::BlockNumberFor:: + > + >::on_poll(n, weight); + } + } + impl<#type_impl_gen> #frame_support::traits::OnInitialize<#frame_system::pallet_prelude::BlockNumberFor::> for #pallet_ident<#type_use_gen> #where_clause diff --git a/substrate/frame/support/src/migrations.rs b/substrate/frame/support/src/migrations.rs index f75410590232..2ceab44cb16b 100644 --- a/substrate/frame/support/src/migrations.rs +++ b/substrate/frame/support/src/migrations.rs @@ -16,13 +16,21 @@ // limitations under the License. use crate::{ - traits::{GetStorageVersion, NoStorageVersionSet, PalletInfoAccess, StorageVersion}, - weights::{RuntimeDbWeight, Weight}, + defensive, + storage::transactional::with_transaction_opaque_err, + traits::{ + Defensive, GetStorageVersion, NoStorageVersionSet, PalletInfoAccess, SafeMode, + StorageVersion, + }, + weights::{RuntimeDbWeight, Weight, WeightMeter}, }; +use codec::{Decode, Encode, MaxEncodedLen}; use impl_trait_for_tuples::impl_for_tuples; +use sp_arithmetic::traits::Bounded; use sp_core::Get; use sp_io::{hashing::twox_128, storage::clear_prefix, KillStorageResult}; -use sp_std::marker::PhantomData; +use sp_runtime::traits::Zero; +use sp_std::{marker::PhantomData, vec::Vec}; /// Handles storage migration pallet versioning. /// @@ -91,7 +99,7 @@ pub struct VersionedMigration), @@ -118,7 +126,6 @@ impl< /// migration ran or not. #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { - use codec::Encode; let on_chain_version = Pallet::on_chain_storage_version(); if on_chain_version == FROM { Ok(VersionedPostUpgradeData::MigrationExecuted(Inner::pre_upgrade()?).encode()) @@ -361,3 +368,622 @@ impl, DbWeight: Get> frame_support::traits Ok(()) } } + +/// A migration that can proceed in multiple steps. +pub trait SteppedMigration { + /// The cursor type that stores the progress (aka. state) of this migration. + type Cursor: codec::FullCodec + codec::MaxEncodedLen; + + /// The unique identifier type of this migration. + type Identifier: codec::FullCodec + codec::MaxEncodedLen; + + /// The unique identifier of this migration. + /// + /// If two migrations have the same identifier, then they are assumed to be identical. + fn id() -> Self::Identifier; + + /// The maximum number of steps that this migration can take. + /// + /// This can be used to enforce progress and prevent migrations becoming stuck forever. A + /// migration that exceeds its max steps is treated as failed. `None` means that there is no + /// limit. + fn max_steps() -> Option { + None + } + + /// Try to migrate as much as possible with the given weight. + /// + /// **ANY STORAGE CHANGES MUST BE ROLLED-BACK BY THE CALLER UPON ERROR.** This is necessary + /// since the caller cannot return a cursor in the error case. [`Self::transactional_step`] is + /// provided as convenience for a caller. A cursor of `None` implies that the migration is at + /// its end. A migration that once returned `Nonce` is guaranteed to never be called again. + fn step( + cursor: Option, + meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError>; + + /// Same as [`Self::step`], but rolls back pending changes in the error case. + fn transactional_step( + mut cursor: Option, + meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + with_transaction_opaque_err(move || match Self::step(cursor, meter) { + Ok(new_cursor) => { + cursor = new_cursor; + sp_runtime::TransactionOutcome::Commit(Ok(cursor)) + }, + Err(err) => sp_runtime::TransactionOutcome::Rollback(Err(err)), + }) + .map_err(|()| SteppedMigrationError::Failed)? + } +} + +/// Error that can occur during a [`SteppedMigration`]. +#[derive(Debug, Encode, Decode, MaxEncodedLen, scale_info::TypeInfo)] +pub enum SteppedMigrationError { + // Transient errors: + /// The remaining weight is not enough to do anything. + /// + /// Can be resolved by calling with at least `required` weight. Note that calling it with + /// exactly `required` weight could cause it to not make any progress. + InsufficientWeight { + /// Amount of weight required to make progress. + required: Weight, + }, + // Permanent errors: + /// The migration cannot decode its cursor and therefore not proceed. + /// + /// This should not happen unless (1) the migration itself returned an invalid cursor in a + /// previous iteration, (2) the storage got corrupted or (3) there is a bug in the caller's + /// code. + InvalidCursor, + /// The migration encountered a permanent error and cannot continue. + Failed, +} + +/// Notification handler for status updates regarding Multi-Block-Migrations. +#[impl_trait_for_tuples::impl_for_tuples(8)] +pub trait MigrationStatusHandler { + /// Notifies of the start of a runtime migration. + fn started() {} + + /// Notifies of the completion of a runtime migration. + fn completed() {} +} + +/// Handles a failed runtime migration. +/// +/// This should never happen, but is here for completeness. +pub trait FailedMigrationHandler { + /// Infallibly handle a failed runtime migration. + /// + /// Gets passed in the optional index of the migration in the batch that caused the failure. + /// Returning `None` means that no automatic handling should take place and the callee decides + /// in the implementation what to do. + fn failed(migration: Option) -> FailedMigrationHandling; +} + +/// Do now allow any transactions to be processed after a runtime upgrade failed. +/// +/// This is **not a sane default**, since it prevents governance intervention. +pub struct FreezeChainOnFailedMigration; + +impl FailedMigrationHandler for FreezeChainOnFailedMigration { + fn failed(_migration: Option) -> FailedMigrationHandling { + FailedMigrationHandling::KeepStuck + } +} + +/// Enter safe mode on a failed runtime upgrade. +/// +/// This can be very useful to manually intervene and fix the chain state. `Else` is used in case +/// that the safe mode could not be entered. +pub struct EnterSafeModeOnFailedMigration( + PhantomData<(SM, Else)>, +); + +impl FailedMigrationHandler + for EnterSafeModeOnFailedMigration +where + ::BlockNumber: Bounded, +{ + fn failed(migration: Option) -> FailedMigrationHandling { + let entered = if SM::is_entered() { + SM::extend(Bounded::max_value()) + } else { + SM::enter(Bounded::max_value()) + }; + + // If we could not enter or extend safe mode (for whatever reason), then we try the next. + if entered.is_err() { + Else::failed(migration) + } else { + FailedMigrationHandling::KeepStuck + } + } +} + +/// How to proceed after a runtime upgrade failed. +/// +/// There is NO SANE DEFAULT HERE. All options are very dangerous and should be used with care. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FailedMigrationHandling { + /// Resume extrinsic processing of the chain. This will not resume the upgrade. + /// + /// This should be supplemented with additional measures to ensure that the broken chain state + /// does not get further messed up by user extrinsics. + ForceUnstuck, + /// Set the cursor to `Stuck` and keep blocking extrinsics. + KeepStuck, + /// Don't do anything with the cursor and let the handler decide. + /// + /// This can be useful in cases where the other two options would overwrite any changes that + /// were done by the handler to the cursor. + Ignore, +} + +/// Something that can do multi step migrations. +pub trait MultiStepMigrator { + /// Hint for whether [`Self::step`] should be called. + fn ongoing() -> bool; + + /// Do the next step in the MBM process. + /// + /// Must gracefully handle the case that it is currently not upgrading. + fn step() -> Weight; +} + +impl MultiStepMigrator for () { + fn ongoing() -> bool { + false + } + + fn step() -> Weight { + Weight::zero() + } +} + +/// Multiple [`SteppedMigration`]. +pub trait SteppedMigrations { + /// The number of migrations that `Self` aggregates. + fn len() -> u32; + + /// The `n`th [`SteppedMigration::id`]. + /// + /// Is guaranteed to return `Some` if `n < Self::len()`. + fn nth_id(n: u32) -> Option>; + + /// The [`SteppedMigration::max_steps`] of the `n`th migration. + /// + /// Is guaranteed to return `Some` if `n < Self::len()`. + fn nth_max_steps(n: u32) -> Option>; + + /// Do a [`SteppedMigration::step`] on the `n`th migration. + /// + /// Is guaranteed to return `Some` if `n < Self::len()`. + fn nth_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>>; + + /// Do a [`SteppedMigration::transactional_step`] on the `n`th migration. + /// + /// Is guaranteed to return `Some` if `n < Self::len()`. + fn nth_transactional_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>>; + + /// The maximal encoded length across all cursors. + fn cursor_max_encoded_len() -> usize; + + /// The maximal encoded length across all identifiers. + fn identifier_max_encoded_len() -> usize; + + /// Assert the integrity of the migrations. + /// + /// Should be executed as part of a test prior to runtime usage. May or may not need + /// externalities. + #[cfg(feature = "std")] + fn integrity_test() -> Result<(), &'static str> { + use crate::ensure; + let l = Self::len(); + + for n in 0..l { + ensure!(Self::nth_id(n).is_some(), "id is None"); + ensure!(Self::nth_max_steps(n).is_some(), "steps is None"); + + // The cursor that we use does not matter. Hence use empty. + ensure!( + Self::nth_step(n, Some(vec![]), &mut WeightMeter::new()).is_some(), + "steps is None" + ); + ensure!( + Self::nth_transactional_step(n, Some(vec![]), &mut WeightMeter::new()).is_some(), + "steps is None" + ); + } + + Ok(()) + } +} + +impl SteppedMigrations for () { + fn len() -> u32 { + 0 + } + + fn nth_id(_n: u32) -> Option> { + None + } + + fn nth_max_steps(_n: u32) -> Option> { + None + } + + fn nth_step( + _n: u32, + _cursor: Option>, + _meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + None + } + + fn nth_transactional_step( + _n: u32, + _cursor: Option>, + _meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + None + } + + fn cursor_max_encoded_len() -> usize { + 0 + } + + fn identifier_max_encoded_len() -> usize { + 0 + } +} + +// A collection consisting of only a single migration. +impl SteppedMigrations for T { + fn len() -> u32 { + 1 + } + + fn nth_id(_n: u32) -> Option> { + Some(T::id().encode()) + } + + fn nth_max_steps(n: u32) -> Option> { + // It should be generally fine to call with n>0, but the code should not attempt to. + n.is_zero() + .then_some(T::max_steps()) + .defensive_proof("nth_max_steps should only be called with n==0") + } + + fn nth_step( + _n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + if !_n.is_zero() { + defensive!("nth_step should only be called with n==0"); + return None + } + + let cursor = match cursor { + Some(cursor) => match T::Cursor::decode(&mut &cursor[..]) { + Ok(cursor) => Some(cursor), + Err(_) => return Some(Err(SteppedMigrationError::InvalidCursor)), + }, + None => None, + }; + + Some(T::step(cursor, meter).map(|cursor| cursor.map(|cursor| cursor.encode()))) + } + + fn nth_transactional_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + if n != 0 { + defensive!("nth_transactional_step should only be called with n==0"); + return None + } + + let cursor = match cursor { + Some(cursor) => match T::Cursor::decode(&mut &cursor[..]) { + Ok(cursor) => Some(cursor), + Err(_) => return Some(Err(SteppedMigrationError::InvalidCursor)), + }, + None => None, + }; + + Some( + T::transactional_step(cursor, meter).map(|cursor| cursor.map(|cursor| cursor.encode())), + ) + } + + fn cursor_max_encoded_len() -> usize { + T::Cursor::max_encoded_len() + } + + fn identifier_max_encoded_len() -> usize { + T::Identifier::max_encoded_len() + } +} + +#[impl_trait_for_tuples::impl_for_tuples(1, 30)] +impl SteppedMigrations for Tuple { + fn len() -> u32 { + for_tuples!( #( Tuple::len() )+* ) + } + + fn nth_id(n: u32) -> Option> { + let mut i = 0; + + for_tuples!( #( + if (i + Tuple::len()) > n { + return Tuple::nth_id(n - i) + } + + i += Tuple::len(); + )* ); + + None + } + + fn nth_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + let mut i = 0; + + for_tuples!( #( + if (i + Tuple::len()) > n { + return Tuple::nth_step(n - i, cursor, meter) + } + + i += Tuple::len(); + )* ); + + None + } + + fn nth_transactional_step( + n: u32, + cursor: Option>, + meter: &mut WeightMeter, + ) -> Option>, SteppedMigrationError>> { + let mut i = 0; + + for_tuples! ( #( + if (i + Tuple::len()) > n { + return Tuple::nth_transactional_step(n - i, cursor, meter) + } + + i += Tuple::len(); + )* ); + + None + } + + fn nth_max_steps(n: u32) -> Option> { + let mut i = 0; + + for_tuples!( #( + if (i + Tuple::len()) > n { + return Tuple::nth_max_steps(n - i) + } + + i += Tuple::len(); + )* ); + + None + } + + fn cursor_max_encoded_len() -> usize { + let mut max_len = 0; + + for_tuples!( #( + max_len = max_len.max(Tuple::cursor_max_encoded_len()); + )* ); + + max_len + } + + fn identifier_max_encoded_len() -> usize { + let mut max_len = 0; + + for_tuples!( #( + max_len = max_len.max(Tuple::identifier_max_encoded_len()); + )* ); + + max_len + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{assert_ok, storage::unhashed}; + + #[derive(Decode, Encode, MaxEncodedLen, Eq, PartialEq)] + pub enum Either { + Left(L), + Right(R), + } + + pub struct M0; + impl SteppedMigration for M0 { + type Cursor = (); + type Identifier = u8; + + fn id() -> Self::Identifier { + 0 + } + + fn step( + _cursor: Option, + _meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + log::info!("M0"); + unhashed::put(&[0], &()); + Ok(None) + } + } + + pub struct M1; + impl SteppedMigration for M1 { + type Cursor = (); + type Identifier = u8; + + fn id() -> Self::Identifier { + 1 + } + + fn step( + _cursor: Option, + _meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + log::info!("M1"); + unhashed::put(&[1], &()); + Ok(None) + } + + fn max_steps() -> Option { + Some(1) + } + } + + pub struct M2; + impl SteppedMigration for M2 { + type Cursor = (); + type Identifier = u8; + + fn id() -> Self::Identifier { + 2 + } + + fn step( + _cursor: Option, + _meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + log::info!("M2"); + unhashed::put(&[2], &()); + Ok(None) + } + + fn max_steps() -> Option { + Some(2) + } + } + + pub struct F0; + impl SteppedMigration for F0 { + type Cursor = (); + type Identifier = u8; + + fn id() -> Self::Identifier { + 3 + } + + fn step( + _cursor: Option, + _meter: &mut WeightMeter, + ) -> Result, SteppedMigrationError> { + log::info!("F0"); + unhashed::put(&[3], &()); + Err(SteppedMigrationError::Failed) + } + } + + // Three migrations combined to execute in order: + type Triple = (M0, (M1, M2)); + // Six migrations, just concatenating the ones from before: + type Hextuple = (Triple, Triple); + + #[test] + fn singular_migrations_work() { + assert_eq!(M0::max_steps(), None); + assert_eq!(M1::max_steps(), Some(1)); + assert_eq!(M2::max_steps(), Some(2)); + + assert_eq!(<(M0, M1)>::nth_max_steps(0), Some(None)); + assert_eq!(<(M0, M1)>::nth_max_steps(1), Some(Some(1))); + assert_eq!(<(M0, M1, M2)>::nth_max_steps(2), Some(Some(2))); + + assert_eq!(<(M0, M1)>::nth_max_steps(2), None); + } + + #[test] + fn tuple_migrations_work() { + assert_eq!(<() as SteppedMigrations>::len(), 0); + assert_eq!(<((), ((), ())) as SteppedMigrations>::len(), 0); + assert_eq!(::len(), 3); + assert_eq!(::len(), 6); + + // Check the IDs. The index specific functions all return an Option, + // to account for the out-of-range case. + assert_eq!(::nth_id(0), Some(0u8.encode())); + assert_eq!(::nth_id(1), Some(1u8.encode())); + assert_eq!(::nth_id(2), Some(2u8.encode())); + + sp_io::TestExternalities::default().execute_with(|| { + for n in 0..3 { + ::nth_step( + n, + Default::default(), + &mut WeightMeter::new(), + ); + } + }); + } + + #[test] + fn integrity_test_works() { + sp_io::TestExternalities::default().execute_with(|| { + assert_ok!(<() as SteppedMigrations>::integrity_test()); + assert_ok!(::integrity_test()); + assert_ok!(::integrity_test()); + assert_ok!(::integrity_test()); + assert_ok!(::integrity_test()); + assert_ok!(::integrity_test()); + }); + } + + #[test] + fn transactional_rollback_works() { + sp_io::TestExternalities::default().execute_with(|| { + assert_ok!(<(M0, F0) as SteppedMigrations>::nth_transactional_step( + 0, + Default::default(), + &mut WeightMeter::new() + ) + .unwrap()); + assert!(unhashed::exists(&[0])); + + let _g = crate::StorageNoopGuard::new(); + assert!(<(M0, F0) as SteppedMigrations>::nth_transactional_step( + 1, + Default::default(), + &mut WeightMeter::new() + ) + .unwrap() + .is_err()); + assert!(<(F0, M1) as SteppedMigrations>::nth_transactional_step( + 0, + Default::default(), + &mut WeightMeter::new() + ) + .unwrap() + .is_err()); + }); + } +} diff --git a/substrate/frame/support/src/storage/transactional.rs b/substrate/frame/support/src/storage/transactional.rs index d42e1809e912..0671db4a3a86 100644 --- a/substrate/frame/support/src/storage/transactional.rs +++ b/substrate/frame/support/src/storage/transactional.rs @@ -127,6 +127,22 @@ where } } +/// Same as [`with_transaction`] but casts any internal error to `()`. +/// +/// This rids `E` of the `From` bound that is required by `with_transaction`. +pub fn with_transaction_opaque_err(f: F) -> Result, ()> +where + F: FnOnce() -> TransactionOutcome>, +{ + with_transaction(move || -> TransactionOutcome, DispatchError>> { + match f() { + TransactionOutcome::Commit(res) => TransactionOutcome::Commit(Ok(res)), + TransactionOutcome::Rollback(res) => TransactionOutcome::Rollback(Ok(res)), + } + }) + .map_err(|_| ()) +} + /// Same as [`with_transaction`] but without a limit check on nested transactional layers. /// /// This is mostly for backwards compatibility before there was a transactional layer limit. diff --git a/substrate/frame/support/src/traits.rs b/substrate/frame/support/src/traits.rs index 3d0429f71b11..1997d8fc223e 100644 --- a/substrate/frame/support/src/traits.rs +++ b/substrate/frame/support/src/traits.rs @@ -59,10 +59,10 @@ pub use misc::{ AccountTouch, Backing, ConstBool, ConstI128, ConstI16, ConstI32, ConstI64, ConstI8, ConstU128, ConstU16, ConstU32, ConstU64, ConstU8, DefensiveMax, DefensiveMin, DefensiveSaturating, DefensiveTruncateFrom, EnsureInherentsAreFirst, EqualPrivilegeOnly, EstimateCallFee, - ExecuteBlock, ExtrinsicCall, Get, GetBacking, GetDefault, HandleLifetime, IsSubType, IsType, - Len, OffchainWorker, OnKilledAccount, OnNewAccount, PrivilegeCmp, SameOrOther, Time, - TryCollect, TryDrop, TypedGet, UnixTime, VariantCount, VariantCountOf, WrapperKeepOpaque, - WrapperOpaque, + ExecuteBlock, ExtrinsicCall, Get, GetBacking, GetDefault, HandleLifetime, IsInherent, + IsSubType, IsType, Len, OffchainWorker, OnKilledAccount, OnNewAccount, PrivilegeCmp, + SameOrOther, Time, TryCollect, TryDrop, TypedGet, UnixTime, VariantCount, VariantCountOf, + WrapperKeepOpaque, WrapperOpaque, }; #[allow(deprecated)] pub use misc::{PreimageProvider, PreimageRecipient}; @@ -86,7 +86,8 @@ mod hooks; pub use hooks::GenesisBuild; pub use hooks::{ BeforeAllRuntimeMigrations, BuildGenesisConfig, Hooks, IntegrityTest, OnFinalize, OnGenesis, - OnIdle, OnInitialize, OnRuntimeUpgrade, OnTimestampSet, + OnIdle, OnInitialize, OnPoll, OnRuntimeUpgrade, OnTimestampSet, PostInherents, + PostTransactions, PreInherents, }; pub mod schedule; diff --git a/substrate/frame/support/src/traits/hooks.rs b/substrate/frame/support/src/traits/hooks.rs index c37fb0f54bcd..7d0e5aa1e89c 100644 --- a/substrate/frame/support/src/traits/hooks.rs +++ b/substrate/frame/support/src/traits/hooks.rs @@ -25,10 +25,74 @@ use crate::weights::Weight; use impl_trait_for_tuples::impl_for_tuples; use sp_runtime::traits::AtLeast32BitUnsigned; use sp_std::prelude::*; +use sp_weights::WeightMeter; #[cfg(feature = "try-runtime")] use sp_runtime::TryRuntimeError; +/// Provides a callback to execute logic before the all inherents. +pub trait PreInherents { + /// Called before all inherents were applied but after `on_initialize`. + fn pre_inherents() {} +} + +#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))] +#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))] +#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))] +impl PreInherents for Tuple { + fn pre_inherents() { + for_tuples!( #( Tuple::pre_inherents(); )* ); + } +} + +/// Provides a callback to execute logic after the all inherents. +pub trait PostInherents { + /// Called after all inherents were applied. + fn post_inherents() {} +} + +#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))] +#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))] +#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))] +impl PostInherents for Tuple { + fn post_inherents() { + for_tuples!( #( Tuple::post_inherents(); )* ); + } +} + +/// Provides a callback to execute logic before the all transactions. +pub trait PostTransactions { + /// Called after all transactions were applied but before `on_finalize`. + fn post_transactions() {} +} + +#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))] +#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))] +#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))] +impl PostTransactions for Tuple { + fn post_transactions() { + for_tuples!( #( Tuple::post_transactions(); )* ); + } +} + +/// Periodically executes logic. Is not guaranteed to run within a specific timeframe and should +/// only be used on logic that has no deadline. +pub trait OnPoll { + /// Code to execute every now and then at the beginning of the block after inherent application. + /// + /// The remaining weight limit must be respected. + fn on_poll(_n: BlockNumber, _weight: &mut WeightMeter) {} +} + +#[cfg_attr(all(not(feature = "tuples-96"), not(feature = "tuples-128")), impl_for_tuples(64))] +#[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))] +#[cfg_attr(feature = "tuples-128", impl_for_tuples(128))] +impl OnPoll for Tuple { + fn on_poll(n: BlockNumber, weight: &mut WeightMeter) { + for_tuples!( #( Tuple::on_poll(n.clone(), weight); )* ); + } +} + /// See [`Hooks::on_initialize`]. pub trait OnInitialize { /// See [`Hooks::on_initialize`]. @@ -374,6 +438,12 @@ pub trait Hooks { Weight::zero() } + /// A hook to run logic after inherent application. + /// + /// Is not guaranteed to execute in a block and should therefore only be used in no-deadline + /// scenarios. + fn on_poll(_n: BlockNumber, _weight: &mut WeightMeter) {} + /// Hook executed when a code change (aka. a "runtime upgrade") is detected by the FRAME /// `Executive` pallet. /// diff --git a/substrate/frame/support/src/traits/misc.rs b/substrate/frame/support/src/traits/misc.rs index eafd9c8abdd2..1f634a642829 100644 --- a/substrate/frame/support/src/traits/misc.rs +++ b/substrate/frame/support/src/traits/misc.rs @@ -23,6 +23,7 @@ use impl_trait_for_tuples::impl_for_tuples; use scale_info::{build::Fields, meta_type, Path, Type, TypeInfo, TypeParameter}; use sp_arithmetic::traits::{CheckedAdd, CheckedMul, CheckedSub, One, Saturating}; use sp_core::bounded::bounded_vec::TruncateFrom; + #[doc(hidden)] pub use sp_runtime::traits::{ ConstBool, ConstI128, ConstI16, ConstI32, ConstI64, ConstI8, ConstU128, ConstU16, ConstU32, @@ -895,11 +896,21 @@ pub trait GetBacking { /// A trait to ensure the inherent are before non-inherent in a block. /// /// This is typically implemented on runtime, through `construct_runtime!`. -pub trait EnsureInherentsAreFirst { +pub trait EnsureInherentsAreFirst: + IsInherent<::Extrinsic> +{ /// Ensure the position of inherent is correct, i.e. they are before non-inherents. /// - /// On error return the index of the inherent with invalid position (counting from 0). - fn ensure_inherents_are_first(block: &Block) -> Result<(), u32>; + /// On error return the index of the inherent with invalid position (counting from 0). On + /// success it returns the index of the last inherent. `0` therefore means that there are no + /// inherents. + fn ensure_inherents_are_first(block: &Block) -> Result; +} + +/// A trait to check if an extrinsic is an inherent. +pub trait IsInherent { + /// Whether this extrinsic is an inherent. + fn is_inherent(ext: &Extrinsic) -> bool; } /// An extrinsic on which we can get access to call. diff --git a/substrate/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.stderr b/substrate/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.stderr index a4c7ecf78658..30005c07cb63 100644 --- a/substrate/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.stderr +++ b/substrate/frame/support/test/tests/construct_runtime_ui/undefined_inherent_part.stderr @@ -53,8 +53,9 @@ error[E0599]: no function or associated item named `is_inherent` found for struc | |_^ function or associated item not found in `Pallet` | = help: items from traits can only be used if the trait is implemented and in scope - = note: the following trait defines an item `is_inherent`, perhaps you need to implement it: + = note: the following traits define an item `is_inherent`, perhaps you need to implement one of them: candidate #1: `ProvideInherent` + candidate #2: `IsInherent` = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0599]: no function or associated item named `check_inherent` found for struct `pallet::Pallet` in the current scope @@ -119,3 +120,23 @@ error[E0599]: no function or associated item named `is_inherent_required` found = note: the following trait defines an item `is_inherent_required`, perhaps you need to implement it: candidate #1: `ProvideInherent` = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: the trait bound `pallet::Pallet: ProvideInherent` is not satisfied + --> tests/construct_runtime_ui/undefined_inherent_part.rs:70:3 + | +70 | Pallet: pallet expanded::{}::{Pallet, Inherent}, + | ^^^^^^ the trait `ProvideInherent` is not implemented for `pallet::Pallet` + +error[E0277]: the trait bound `pallet::Pallet: ProvideInherent` is not satisfied + --> tests/construct_runtime_ui/undefined_inherent_part.rs:66:1 + | +66 | / construct_runtime! { +67 | | pub struct Runtime +68 | | { +69 | | System: frame_system expanded::{}::{Pallet, Call, Storage, Config, Event}, +70 | | Pallet: pallet expanded::{}::{Pallet, Inherent}, +71 | | } +72 | | } + | |_^ the trait `ProvideInherent` is not implemented for `pallet::Pallet` + | + = note: this error originates in the macro `construct_runtime` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/substrate/frame/support/test/tests/pallet_outer_enums_explicit.rs b/substrate/frame/support/test/tests/pallet_outer_enums_explicit.rs index 79e9d6786717..33b96dea9486 100644 --- a/substrate/frame/support/test/tests/pallet_outer_enums_explicit.rs +++ b/substrate/frame/support/test/tests/pallet_outer_enums_explicit.rs @@ -90,6 +90,7 @@ fn module_error_outer_enum_expand_explicit() { frame_system::Error::NonDefaultComposite => (), frame_system::Error::NonZeroRefCount => (), frame_system::Error::CallFiltered => (), + frame_system::Error::MultiBlockMigrationsOngoing => (), #[cfg(feature = "experimental")] frame_system::Error::InvalidTask => (), #[cfg(feature = "experimental")] diff --git a/substrate/frame/support/test/tests/pallet_outer_enums_implicit.rs b/substrate/frame/support/test/tests/pallet_outer_enums_implicit.rs index 4bd8ee0bb39a..db006fe79359 100644 --- a/substrate/frame/support/test/tests/pallet_outer_enums_implicit.rs +++ b/substrate/frame/support/test/tests/pallet_outer_enums_implicit.rs @@ -90,6 +90,7 @@ fn module_error_outer_enum_expand_implicit() { frame_system::Error::NonDefaultComposite => (), frame_system::Error::NonZeroRefCount => (), frame_system::Error::CallFiltered => (), + frame_system::Error::MultiBlockMigrationsOngoing => (), #[cfg(feature = "experimental")] frame_system::Error::InvalidTask => (), #[cfg(feature = "experimental")] diff --git a/substrate/frame/support/test/tests/runtime_metadata.rs b/substrate/frame/support/test/tests/runtime_metadata.rs index bb7f7d2822e7..40e70b219ba9 100644 --- a/substrate/frame/support/test/tests/runtime_metadata.rs +++ b/substrate/frame/support/test/tests/runtime_metadata.rs @@ -101,7 +101,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } @@ -200,8 +200,8 @@ fn runtime_metadata() { name: "header", ty: meta_type::<&::Header>(), }], - output: meta_type::<()>(), - docs: maybe_docs(vec![" Initialize a block with the given header."]), + output: meta_type::(), + docs: maybe_docs(vec![" Initialize a block with the given header and return the runtime executive mode."]), }, ], docs: maybe_docs(vec![ diff --git a/substrate/frame/system/src/lib.rs b/substrate/frame/system/src/lib.rs index 01df09106d99..3b659916379e 100644 --- a/substrate/frame/system/src/lib.rs +++ b/substrate/frame/system/src/lib.rs @@ -130,11 +130,13 @@ use frame_support::{ DispatchResult, DispatchResultWithPostInfo, PerDispatchClass, PostDispatchInfo, }, ensure, impl_ensure_origin_with_arg_ignoring_arg, + migrations::MultiStepMigrator, pallet_prelude::Pays, storage::{self, StorageStreamIter}, traits::{ ConstU32, Contains, EnsureOrigin, EnsureOriginWithArg, Get, HandleLifetime, - OnKilledAccount, OnNewAccount, OriginTrait, PalletInfo, SortedMembers, StoredMap, TypedGet, + OnKilledAccount, OnNewAccount, OnRuntimeUpgrade, OriginTrait, PalletInfo, SortedMembers, + StoredMap, TypedGet, }, Parameter, }; @@ -169,6 +171,7 @@ pub use extensions::{ // Backward compatible re-export. pub use extensions::check_mortality::CheckMortality as CheckEra; pub use frame_support::dispatch::RawOrigin; +use frame_support::traits::{PostInherents, PostTransactions, PreInherents}; pub use weights::WeightInfo; const LOG_TARGET: &str = "runtime::system"; @@ -299,6 +302,11 @@ pub mod pallet { type BaseCallFilter = frame_support::traits::Everything; type BlockHashCount = frame_support::traits::ConstU64<10>; type OnSetCode = (); + type SingleBlockMigrations = (); + type MultiBlockMigrator = (); + type PreInherents = (); + type PostInherents = (); + type PostTransactions = (); } /// Default configurations of this pallet in a solo-chain environment. @@ -393,6 +401,11 @@ pub mod pallet { /// The set code logic, just the default since we're not a parachain. type OnSetCode = (); + type SingleBlockMigrations = (); + type MultiBlockMigrator = (); + type PreInherents = (); + type PostInherents = (); + type PostTransactions = (); } /// Default configurations of this pallet in a relay-chain environment. @@ -572,6 +585,35 @@ pub mod pallet { /// The maximum number of consumers allowed on a single account. type MaxConsumers: ConsumerLimits; + + /// All migrations that should run in the next runtime upgrade. + /// + /// These used to be formerly configured in `Executive`. Parachains need to ensure that + /// running all these migrations in one block will not overflow the weight limit of a block. + /// The migrations are run *before* the pallet `on_runtime_upgrade` hooks, just like the + /// `OnRuntimeUpgrade` migrations. + type SingleBlockMigrations: OnRuntimeUpgrade; + + /// The migrator that is used to run Multi-Block-Migrations. + /// + /// Can be set to [`pallet-migrations`] or an alternative implementation of the interface. + /// The diagram in `frame_executive::block_flowchart` explains when it runs. + type MultiBlockMigrator: MultiStepMigrator; + + /// A callback that executes in *every block* directly before all inherents were applied. + /// + /// See `frame_executive::block_flowchart` for a in-depth explanation when it runs. + type PreInherents: PreInherents; + + /// A callback that executes in *every block* directly after all inherents were applied. + /// + /// See `frame_executive::block_flowchart` for a in-depth explanation when it runs. + type PostInherents: PostInherents; + + /// A callback that executes in *every block* directly after all transactions were applied. + /// + /// See `frame_executive::block_flowchart` for a in-depth explanation when it runs. + type PostTransactions: PostTransactions; } #[pallet::pallet] @@ -619,6 +661,9 @@ pub mod pallet { } /// Set the new runtime code without doing any checks of the given `code`. + /// + /// Note that runtime upgrades will not run if this is called with a not-increasing spec + /// version! #[pallet::call_index(3)] #[pallet::weight((T::SystemWeightInfo::set_code(), DispatchClass::Operational))] pub fn set_code_without_checks( @@ -814,6 +859,8 @@ pub mod pallet { NonZeroRefCount, /// The origin filter prevent the call to be dispatched. CallFiltered, + /// A multi-block migration is ongoing and prevents the current code from being replaced. + MultiBlockMigrationsOngoing, #[cfg(feature = "experimental")] /// The specified [`Task`] is not valid. InvalidTask, @@ -845,6 +892,10 @@ pub mod pallet { #[pallet::storage] pub(super) type ExtrinsicCount = StorageValue<_, u32>; + /// Whether all inherents have been applied. + #[pallet::storage] + pub type InherentsApplied = StorageValue<_, bool, ValueQuery>; + /// The current weight for the block. #[pallet::storage] #[pallet::whitelist_storage] @@ -1373,6 +1424,19 @@ impl Pallet { Self::deposit_event(Event::CodeUpdated); } + /// Whether all inherents have been applied. + pub fn inherents_applied() -> bool { + InherentsApplied::::get() + } + + /// Note that all inherents have been applied. + /// + /// Should be called immediately after all inherents have been applied. Must be called at least + /// once per block. + pub fn note_inherents_applied() { + InherentsApplied::::put(true); + } + /// Increment the reference counter on an account. #[deprecated = "Use `inc_consumers` instead"] pub fn inc_ref(who: &T::AccountId) { @@ -1692,6 +1756,7 @@ impl Pallet { >::put(digest); >::put(parent_hash); >::insert(*number - One::one(), parent_hash); + >::kill(); // Remove previous block data from storage BlockWeight::::kill(); @@ -1738,6 +1803,7 @@ impl Pallet { ExecutionPhase::::kill(); AllExtrinsicsLen::::kill(); storage::unhashed::kill(well_known_keys::INTRABLOCK_ENTROPY); + InherentsApplied::::kill(); // The following fields // @@ -1981,10 +2047,14 @@ impl Pallet { /// Determine whether or not it is possible to update the code. /// - /// Checks the given code if it is a valid runtime wasm blob by instantianting + /// Checks the given code if it is a valid runtime wasm blob by instantiating /// it and extracting the runtime version of it. It checks that the runtime version /// of the old and new runtime has the same spec name and that the spec version is increasing. pub fn can_set_code(code: &[u8]) -> Result<(), sp_runtime::DispatchError> { + if T::MultiBlockMigrator::ongoing() { + return Err(Error::::MultiBlockMigrationsOngoing.into()) + } + let current_version = T::Version::get(); let new_version = sp_io::misc::runtime_version(code) .and_then(|v| RuntimeVersion::decode(&mut &v[..]).ok()) diff --git a/substrate/frame/system/src/mock.rs b/substrate/frame/system/src/mock.rs index c4108099e39f..7059845a7df3 100644 --- a/substrate/frame/system/src/mock.rs +++ b/substrate/frame/system/src/mock.rs @@ -86,6 +86,22 @@ impl Config for Test { type Version = Version; type AccountData = u32; type OnKilledAccount = RecordKilled; + type MultiBlockMigrator = MockedMigrator; +} + +parameter_types! { + pub static Ongoing: bool = false; +} + +pub struct MockedMigrator; +impl frame_support::migrations::MultiStepMigrator for MockedMigrator { + fn ongoing() -> bool { + Ongoing::get() + } + + fn step() -> Weight { + Weight::zero() + } } pub type SysEvent = frame_system::Event; diff --git a/substrate/frame/system/src/tests.rs b/substrate/frame/system/src/tests.rs index e437e7f9f39b..b889b5ca046e 100644 --- a/substrate/frame/system/src/tests.rs +++ b/substrate/frame/system/src/tests.rs @@ -675,6 +675,28 @@ fn set_code_with_real_wasm_blob() { }); } +#[test] +fn set_code_rejects_during_mbm() { + Ongoing::set(true); + + let executor = substrate_test_runtime_client::new_native_or_wasm_executor(); + let mut ext = new_test_ext(); + ext.register_extension(sp_core::traits::ReadRuntimeVersionExt::new(executor)); + ext.execute_with(|| { + System::set_block_number(1); + let res = System::set_code( + RawOrigin::Root.into(), + substrate_test_runtime_client::runtime::wasm_binary_unwrap().to_vec(), + ); + assert_eq!( + res, + Err(DispatchErrorWithPostInfo::from(Error::::MultiBlockMigrationsOngoing)) + ); + + assert!(System::events().is_empty()); + }); +} + #[test] fn set_code_via_authorization_works() { let executor = substrate_test_runtime_client::new_native_or_wasm_executor(); diff --git a/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs b/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs index 2b1e65ec8852..e34e4c0e7672 100644 --- a/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs +++ b/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs @@ -456,6 +456,7 @@ impl<'a> ToClientSideDecl<'a> { |err| #crate_::ApiError::FailedToDecodeReturnValue { function: #function_name, error: err, + raw: r.clone(), } ) ) diff --git a/substrate/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs b/substrate/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs index c1339ff6621b..1761e0ac9dbf 100644 --- a/substrate/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs +++ b/substrate/primitives/api/proc-macro/src/mock_impl_runtime_apis.rs @@ -158,7 +158,7 @@ fn implement_common_api_traits(block_type: TypePath, self_ty: Type) -> Result::Hash, _: &<#block_type as #crate_::BlockT>::Header, - ) -> std::result::Result<(), #crate_::ApiError> { + ) -> std::result::Result<#crate_::__private::ExtrinsicInclusionMode, #crate_::ApiError> { unimplemented!("`Core::initialize_block` not implemented for runtime api mocks") } } diff --git a/substrate/primitives/api/src/lib.rs b/substrate/primitives/api/src/lib.rs index 190de1ab3fde..a945b9f21f3c 100644 --- a/substrate/primitives/api/src/lib.rs +++ b/substrate/primitives/api/src/lib.rs @@ -101,7 +101,7 @@ pub mod __private { generic::BlockId, traits::{Block as BlockT, Hash as HashT, HashingFor, Header as HeaderT, NumberFor}, transaction_validity::TransactionValidity, - RuntimeString, TransactionOutcome, + ExtrinsicInclusionMode, RuntimeString, TransactionOutcome, }; pub use sp_std::{mem, slice, vec}; pub use sp_version::{create_apis_vec, ApiId, ApisVec, RuntimeVersion}; @@ -115,11 +115,11 @@ pub use sp_core::traits::CallContext; use sp_core::OpaqueMetadata; #[cfg(feature = "std")] use sp_externalities::{Extension, Extensions}; -use sp_runtime::traits::Block as BlockT; #[cfg(feature = "std")] use sp_runtime::traits::HashingFor; #[cfg(feature = "std")] pub use sp_runtime::TransactionOutcome; +use sp_runtime::{traits::Block as BlockT, ExtrinsicInclusionMode}; #[cfg(feature = "std")] pub use sp_state_machine::StorageProof; #[cfg(feature = "std")] @@ -280,7 +280,7 @@ pub use sp_api_proc_macro::decl_runtime_apis; /// ```rust /// use sp_version::create_runtime_str; /// # -/// # use sp_runtime::traits::Block as BlockT; +/// # use sp_runtime::{ExtrinsicInclusionMode, traits::Block as BlockT}; /// # use sp_test_primitives::Block; /// # /// # /// The declaration of the `Runtime` type is done by the `construct_runtime!` macro @@ -307,7 +307,9 @@ pub use sp_api_proc_macro::decl_runtime_apis; /// # unimplemented!() /// # } /// # fn execute_block(_block: Block) {} -/// # fn initialize_block(_header: &::Header) {} +/// # fn initialize_block(_header: &::Header) -> ExtrinsicInclusionMode { +/// # unimplemented!() +/// # } /// # } /// /// impl self::Balance for Runtime { @@ -540,11 +542,12 @@ pub fn init_runtime_logger() { #[cfg(feature = "std")] #[derive(Debug, thiserror::Error)] pub enum ApiError { - #[error("Failed to decode return value of {function}")] + #[error("Failed to decode return value of {function}: {error} raw data: {raw:?}")] FailedToDecodeReturnValue { function: &'static str, #[source] error: codec::Error, + raw: Vec, }, #[error("Failed to convert return value from runtime to node of {function}")] FailedToConvertReturnValue { @@ -800,15 +803,18 @@ pub fn deserialize_runtime_api_info(bytes: [u8; RUNTIME_API_INFO_SIZE]) -> ([u8; decl_runtime_apis! { /// The `Core` runtime api that every Substrate runtime needs to implement. #[core_trait] - #[api_version(4)] + #[api_version(5)] pub trait Core { /// Returns the version of the runtime. fn version() -> RuntimeVersion; /// Execute the given block. fn execute_block(block: Block); /// Initialize a block with the given header. + #[changed_in(5)] #[renamed("initialise_block", 2)] fn initialize_block(header: &::Header); + /// Initialize a block with the given header and return the runtime executive mode. + fn initialize_block(header: &::Header) -> ExtrinsicInclusionMode; } /// The `Metadata` api trait that returns metadata for the runtime. diff --git a/substrate/primitives/api/test/tests/decl_and_impl.rs b/substrate/primitives/api/test/tests/decl_and_impl.rs index d68470551d20..211a08561fd4 100644 --- a/substrate/primitives/api/test/tests/decl_and_impl.rs +++ b/substrate/primitives/api/test/tests/decl_and_impl.rs @@ -139,7 +139,7 @@ impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/impl_incorrect_method_signature.rs b/substrate/primitives/api/test/tests/ui/impl_incorrect_method_signature.rs index 43718e4cd04a..262a874213a5 100644 --- a/substrate/primitives/api/test/tests/ui/impl_incorrect_method_signature.rs +++ b/substrate/primitives/api/test/tests/ui/impl_incorrect_method_signature.rs @@ -40,7 +40,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/impl_missing_version.rs b/substrate/primitives/api/test/tests/ui/impl_missing_version.rs index 560257b5168c..58850ab343fb 100644 --- a/substrate/primitives/api/test/tests/ui/impl_missing_version.rs +++ b/substrate/primitives/api/test/tests/ui/impl_missing_version.rs @@ -45,7 +45,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/missing_versioned_method.rs b/substrate/primitives/api/test/tests/ui/missing_versioned_method.rs index 6ead545f85a1..70f75d065154 100644 --- a/substrate/primitives/api/test/tests/ui/missing_versioned_method.rs +++ b/substrate/primitives/api/test/tests/ui/missing_versioned_method.rs @@ -44,7 +44,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/missing_versioned_method_multiple_vers.rs b/substrate/primitives/api/test/tests/ui/missing_versioned_method_multiple_vers.rs index 8eebc1d79bab..63032000040b 100644 --- a/substrate/primitives/api/test/tests/ui/missing_versioned_method_multiple_vers.rs +++ b/substrate/primitives/api/test/tests/ui/missing_versioned_method_multiple_vers.rs @@ -47,7 +47,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/positive_cases/custom_where_bound.rs b/substrate/primitives/api/test/tests/ui/positive_cases/custom_where_bound.rs index 594556d57be5..0858813bc999 100644 --- a/substrate/primitives/api/test/tests/ui/positive_cases/custom_where_bound.rs +++ b/substrate/primitives/api/test/tests/ui/positive_cases/custom_where_bound.rs @@ -51,7 +51,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/positive_cases/default_impls.rs b/substrate/primitives/api/test/tests/ui/positive_cases/default_impls.rs index ae573238ffe1..3e0cb79156c8 100644 --- a/substrate/primitives/api/test/tests/ui/positive_cases/default_impls.rs +++ b/substrate/primitives/api/test/tests/ui/positive_cases/default_impls.rs @@ -46,7 +46,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.rs b/substrate/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.rs index 921bf0d04351..b2caea7ab7e4 100644 --- a/substrate/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.rs +++ b/substrate/primitives/api/test/tests/ui/type_reference_in_impl_runtime_apis_call.rs @@ -42,7 +42,7 @@ sp_api::impl_runtime_apis! { fn execute_block(_: Block) { unimplemented!() } - fn initialize_block(_: &::Header) { + fn initialize_block(_: &::Header) -> sp_runtime::ExtrinsicInclusionMode { unimplemented!() } } diff --git a/substrate/primitives/runtime/src/lib.rs b/substrate/primitives/runtime/src/lib.rs index ddf92554c830..44bf3c969e54 100644 --- a/substrate/primitives/runtime/src/lib.rs +++ b/substrate/primitives/runtime/src/lib.rs @@ -998,6 +998,16 @@ impl TransactionOutcome { } } +/// Confines the kind of extrinsics that can be included in a block. +#[derive(Debug, Default, PartialEq, Eq, Clone, Copy, Encode, Decode, TypeInfo)] +pub enum ExtrinsicInclusionMode { + /// All extrinsics are allowed to be included in this block. + #[default] + AllExtrinsics, + /// Inherents are allowed to be included. + OnlyInherents, +} + #[cfg(test)] mod tests { use crate::traits::BlakeTwo256; diff --git a/substrate/test-utils/runtime/src/lib.rs b/substrate/test-utils/runtime/src/lib.rs index db9ff187b707..63e0aa6e1379 100644 --- a/substrate/test-utils/runtime/src/lib.rs +++ b/substrate/test-utils/runtime/src/lib.rs @@ -61,7 +61,7 @@ use sp_runtime::{ create_runtime_str, impl_opaque_keys, traits::{BlakeTwo256, Block as BlockT, DispatchInfoOf, NumberFor, Verify}, transaction_validity::{TransactionSource, TransactionValidity, TransactionValidityError}, - ApplyExtrinsicResult, Perbill, + ApplyExtrinsicResult, ExtrinsicInclusionMode, Perbill, }; #[cfg(any(feature = "std", test))] use sp_version::NativeVersion; @@ -480,9 +480,9 @@ impl_runtime_apis! { Executive::execute_block(block); } - fn initialize_block(header: &::Header) { + fn initialize_block(header: &::Header) -> ExtrinsicInclusionMode { log::trace!(target: LOG_TARGET, "initialize_block: {header:#?}"); - Executive::initialize_block(header); + Executive::initialize_block(header) } } From 833bafdbf7b178b639f895e0d43c43eb502de30c Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe <49718502+alexggh@users.noreply.github.com> Date: Thu, 29 Feb 2024 01:43:53 +0200 Subject: [PATCH 07/13] Fixup multi-collator parachain transition to async backing (#3510) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixing: ``` Verification failed for block 0x07bbf1e04121d70a4bdb21cc055132b53ac2390fa95c4d05497fc91b1e8bf7f5 received from (12D3KooWJzLd8skcAgA24EcJey7aJAhYctfUxWGjSP5Usk9wbpPZ): "Header 0x07bbf1e04121d70a4bdb21cc055132b53ac2390fa95c4d05497fc91b1e8bf7f5 rejected: too far in the future" ``` --------- Signed-off-by: Alexandru Gheorghe Co-authored-by: Dmitry Sinyavin Co-authored-by: s0me0ne-unkn0wn <48632512+s0me0ne-unkn0wn@users.noreply.github.com> Co-authored-by: Bastian Köcher --- cumulus/client/consensus/aura/src/lib.rs | 5 ++- cumulus/polkadot-parachain/src/service.rs | 33 +++++++++++-------- prdoc/pr_3510.prdoc | 13 ++++++++ .../bin/node-template/node/src/service.rs | 26 +++++++++------ 4 files changed, 52 insertions(+), 25 deletions(-) create mode 100644 prdoc/pr_3510.prdoc diff --git a/cumulus/client/consensus/aura/src/lib.rs b/cumulus/client/consensus/aura/src/lib.rs index 8e4bc658e44b..ed6f5bdd4d69 100644 --- a/cumulus/client/consensus/aura/src/lib.rs +++ b/cumulus/client/consensus/aura/src/lib.rs @@ -54,7 +54,10 @@ use std::{ mod import_queue; pub use import_queue::{build_verifier, import_queue, BuildVerifierParams, ImportQueueParams}; -pub use sc_consensus_aura::{slot_duration, AuraVerifier, BuildAuraWorkerParams, SlotProportion}; +pub use sc_consensus_aura::{ + slot_duration, standalone::slot_duration_at, AuraVerifier, BuildAuraWorkerParams, + SlotProportion, +}; pub use sc_consensus_slots::InherentDataProviderExt; pub mod collator; diff --git a/cumulus/polkadot-parachain/src/service.rs b/cumulus/polkadot-parachain/src/service.rs index 4e06cd38f1d7..386551102e43 100644 --- a/cumulus/polkadot-parachain/src/service.rs +++ b/cumulus/polkadot-parachain/src/service.rs @@ -1251,28 +1251,33 @@ where <::Pair as Pair>::Signature: TryFrom> + std::hash::Hash + sp_runtime::traits::Member + Codec, { - let client2 = client.clone(); + let verifier_client = client.clone(); let aura_verifier = move || { - let slot_duration = cumulus_client_consensus_aura::slot_duration(&*client2).unwrap(); - Box::new(cumulus_client_consensus_aura::build_verifier::< ::Pair, _, _, _, >(cumulus_client_consensus_aura::BuildVerifierParams { - client: client2.clone(), - create_inherent_data_providers: move |_, _| async move { - let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); - - let slot = - sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_slot_duration( - *timestamp, - slot_duration, - ); - - Ok((slot, timestamp)) + client: verifier_client.clone(), + create_inherent_data_providers: move |parent_hash, _| { + let cidp_client = verifier_client.clone(); + async move { + let slot_duration = cumulus_client_consensus_aura::slot_duration_at( + &*cidp_client, + parent_hash, + )?; + let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); + + let slot = + sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_slot_duration( + *timestamp, + slot_duration, + ); + + Ok((slot, timestamp)) + } }, telemetry: telemetry_handle, })) as Box<_> diff --git a/prdoc/pr_3510.prdoc b/prdoc/pr_3510.prdoc new file mode 100644 index 000000000000..6ee2f9a81f75 --- /dev/null +++ b/prdoc/pr_3510.prdoc @@ -0,0 +1,13 @@ +title: "Fix multi-collator parachain transition to async backing" + +doc: + - audience: Node Operator + description: | + The dynamic Aura slot duration, introduced in PR#3211, didn't take the block import pipeline + into account. The result was the parachain backed by multiple collators not being able to + keep producing blocks after its runtime was upgraded to support async backing, requiring to + restart all the collator nodes. This change fixes the issue, introducing the dynamic Aura + slot duration into the block import pipeline. + +crates: + - name: "polkadot-parachain-bin" diff --git a/substrate/bin/node-template/node/src/service.rs b/substrate/bin/node-template/node/src/service.rs index 25cd65117841..125cca139279 100644 --- a/substrate/bin/node-template/node/src/service.rs +++ b/substrate/bin/node-template/node/src/service.rs @@ -80,23 +80,29 @@ pub fn new_partial(config: &Configuration) -> Result { telemetry.as_ref().map(|x| x.handle()), )?; - let slot_duration = sc_consensus_aura::slot_duration(&*client)?; - + let cidp_client = client.clone(); let import_queue = sc_consensus_aura::import_queue::(ImportQueueParams { block_import: grandpa_block_import.clone(), justification_import: Some(Box::new(grandpa_block_import.clone())), client: client.clone(), - create_inherent_data_providers: move |_, ()| async move { - let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); + create_inherent_data_providers: move |parent_hash, _| { + let cidp_client = cidp_client.clone(); + async move { + let slot_duration = sc_consensus_aura::standalone::slot_duration_at( + &*cidp_client, + parent_hash, + )?; + let timestamp = sp_timestamp::InherentDataProvider::from_system_time(); - let slot = - sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_slot_duration( - *timestamp, - slot_duration, - ); + let slot = + sp_consensus_aura::inherents::InherentDataProvider::from_timestamp_and_slot_duration( + *timestamp, + slot_duration, + ); - Ok((slot, timestamp)) + Ok((slot, timestamp)) + } }, spawner: &task_manager.spawn_essential_handle(), registry: config.prometheus_registry(), From a22319cdd5fcdcec3bbedfe41f6d11a6b5446cf9 Mon Sep 17 00:00:00 2001 From: philoniare Date: Thu, 29 Feb 2024 13:17:24 +0800 Subject: [PATCH 08/13] [Deprecation] Remove sp_weights::OldWeight (#3491) # Description *Removes `sp_weights::OldWeight` and its usage* Fixes #144 --------- Co-authored-by: Liam Aharon --- prdoc/pr_3491.prdoc | 12 +++ substrate/frame/contracts/src/wasm/mod.rs | 11 --- substrate/primitives/weights/src/lib.rs | 27 +------ .../primitives/weights/src/weight_meter.rs | 81 ++++++++----------- 4 files changed, 47 insertions(+), 84 deletions(-) create mode 100644 prdoc/pr_3491.prdoc diff --git a/prdoc/pr_3491.prdoc b/prdoc/pr_3491.prdoc new file mode 100644 index 000000000000..e36afb916a62 --- /dev/null +++ b/prdoc/pr_3491.prdoc @@ -0,0 +1,12 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Remove Deprecated OldWeight + +doc: + - audience: Runtime Dev + description: | + Removed deprecated sp_weights::OldWeight type. Use [`weight_v2::Weight`] instead. + +crates: + - name: frame-support diff --git a/substrate/frame/contracts/src/wasm/mod.rs b/substrate/frame/contracts/src/wasm/mod.rs index c96c28565095..34b45cba5d28 100644 --- a/substrate/frame/contracts/src/wasm/mod.rs +++ b/substrate/frame/contracts/src/wasm/mod.rs @@ -1821,17 +1821,6 @@ mod tests { assert!(weight_left.all_gt(actual_left), "gas_left must be greater than final"); } - /// Test that [`frame_support::weights::OldWeight`] en/decodes the same as our - /// [`crate::OldWeight`]. - #[test] - fn old_weight_decode() { - #![allow(deprecated)] - let sp = frame_support::weights::OldWeight(42).encode(); - let our = crate::OldWeight::decode(&mut &*sp).unwrap(); - - assert_eq!(our, 42); - } - const CODE_VALUE_TRANSFERRED: &str = r#" (module (import "seal0" "seal_value_transferred" (func $seal_value_transferred (param i32 i32))) diff --git a/substrate/primitives/weights/src/lib.rs b/substrate/primitives/weights/src/lib.rs index aede9473535a..b2c956266e2c 100644 --- a/substrate/primitives/weights/src/lib.rs +++ b/substrate/primitives/weights/src/lib.rs @@ -18,9 +18,6 @@ //! # Primitives for transaction weighting. #![cfg_attr(not(feature = "std"), no_std)] -// TODO remove once `OldWeight` is gone. I dont know why this is needed, maybe by one of the macros -// of `OldWeight`. -#![allow(deprecated)] extern crate self as sp_weights; @@ -28,7 +25,7 @@ mod weight_meter; mod weight_v2; use bounded_collections::Get; -use codec::{CompactAs, Decode, Encode, MaxEncodedLen}; +use codec::{Decode, Encode}; use scale_info::TypeInfo; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -52,28 +49,6 @@ pub mod constants { pub const WEIGHT_PROOF_SIZE_PER_KB: u64 = 1024; } -/// The old weight type. -/// -/// NOTE: This type exists purely for compatibility purposes! Use [`weight_v2::Weight`] in all other -/// cases. -#[derive( - Decode, - Encode, - CompactAs, - PartialEq, - Eq, - Clone, - Copy, - RuntimeDebug, - Default, - MaxEncodedLen, - TypeInfo, -)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "serde", serde(transparent))] -#[deprecated(note = "Will be removed soon; use `Weight` instead.")] -pub struct OldWeight(pub u64); - /// The weight of database operations that the runtime can invoke. /// /// NOTE: This is currently only measured in computational time, and will probably diff --git a/substrate/primitives/weights/src/weight_meter.rs b/substrate/primitives/weights/src/weight_meter.rs index 1738948e4c3c..cfe8396ae6d6 100644 --- a/substrate/primitives/weights/src/weight_meter.rs +++ b/substrate/primitives/weights/src/weight_meter.rs @@ -118,13 +118,6 @@ impl WeightMeter { debug_assert!(self.consumed.all_lte(self.limit), "Weight counter overflow"); } - /// Consume the given weight after checking that it can be consumed and return `true`. Otherwise - /// do nothing and return `false`. - #[deprecated(note = "Use `try_consume` instead. Will be removed after December 2023.")] - pub fn check_accrue(&mut self, w: Weight) -> bool { - self.try_consume(w).is_ok() - } - /// Consume the given weight after checking that it can be consumed. /// /// Returns `Ok` if the weight can be consumed or otherwise an `Err`. @@ -139,12 +132,6 @@ impl WeightMeter { }) } - /// Check if the given weight can be consumed. - #[deprecated(note = "Use `can_consume` instead. Will be removed after December 2023.")] - pub fn can_accrue(&self, w: Weight) -> bool { - self.can_consume(w) - } - /// Check if the given weight can be consumed. pub fn can_consume(&self, w: Weight) -> bool { self.consumed.checked_add(&w).map_or(false, |t| t.all_lte(self.limit)) @@ -165,80 +152,80 @@ mod tests { fn weight_meter_remaining_works() { let mut meter = WeightMeter::with_limit(Weight::from_parts(10, 20)); - assert!(meter.check_accrue(Weight::from_parts(5, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(5, 0)), Ok(())); assert_eq!(meter.consumed, Weight::from_parts(5, 0)); assert_eq!(meter.remaining(), Weight::from_parts(5, 20)); - assert!(meter.check_accrue(Weight::from_parts(2, 10))); + assert_eq!(meter.try_consume(Weight::from_parts(2, 10)), Ok(())); assert_eq!(meter.consumed, Weight::from_parts(7, 10)); assert_eq!(meter.remaining(), Weight::from_parts(3, 10)); - assert!(meter.check_accrue(Weight::from_parts(3, 10))); + assert_eq!(meter.try_consume(Weight::from_parts(3, 10)), Ok(())); assert_eq!(meter.consumed, Weight::from_parts(10, 20)); assert_eq!(meter.remaining(), Weight::from_parts(0, 0)); } #[test] - fn weight_meter_can_accrue_works() { + fn weight_meter_can_consume_works() { let meter = WeightMeter::with_limit(Weight::from_parts(1, 1)); - assert!(meter.can_accrue(Weight::from_parts(0, 0))); - assert!(meter.can_accrue(Weight::from_parts(1, 1))); - assert!(!meter.can_accrue(Weight::from_parts(0, 2))); - assert!(!meter.can_accrue(Weight::from_parts(2, 0))); - assert!(!meter.can_accrue(Weight::from_parts(2, 2))); + assert!(meter.can_consume(Weight::from_parts(0, 0))); + assert!(meter.can_consume(Weight::from_parts(1, 1))); + assert!(!meter.can_consume(Weight::from_parts(0, 2))); + assert!(!meter.can_consume(Weight::from_parts(2, 0))); + assert!(!meter.can_consume(Weight::from_parts(2, 2))); } #[test] - fn weight_meter_check_accrue_works() { + fn weight_meter_try_consume_works() { let mut meter = WeightMeter::with_limit(Weight::from_parts(2, 2)); - assert!(meter.check_accrue(Weight::from_parts(0, 0))); - assert!(meter.check_accrue(Weight::from_parts(1, 1))); - assert!(!meter.check_accrue(Weight::from_parts(0, 2))); - assert!(!meter.check_accrue(Weight::from_parts(2, 0))); - assert!(!meter.check_accrue(Weight::from_parts(2, 2))); - assert!(meter.check_accrue(Weight::from_parts(0, 1))); - assert!(meter.check_accrue(Weight::from_parts(1, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(0, 0)), Ok(())); + assert_eq!(meter.try_consume(Weight::from_parts(1, 1)), Ok(())); + assert_eq!(meter.try_consume(Weight::from_parts(0, 2)), Err(())); + assert_eq!(meter.try_consume(Weight::from_parts(2, 0)), Err(())); + assert_eq!(meter.try_consume(Weight::from_parts(2, 2)), Err(())); + assert_eq!(meter.try_consume(Weight::from_parts(0, 1)), Ok(())); + assert_eq!(meter.try_consume(Weight::from_parts(1, 0)), Ok(())); } #[test] - fn weight_meter_check_and_can_accrue_works() { + fn weight_meter_check_and_can_consume_works() { let mut meter = WeightMeter::new(); - assert!(meter.can_accrue(Weight::from_parts(u64::MAX, 0))); - assert!(meter.check_accrue(Weight::from_parts(u64::MAX, 0))); + assert!(meter.can_consume(Weight::from_parts(u64::MAX, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(u64::MAX, 0)), Ok(())); - assert!(meter.can_accrue(Weight::from_parts(0, u64::MAX))); - assert!(meter.check_accrue(Weight::from_parts(0, u64::MAX))); + assert!(meter.can_consume(Weight::from_parts(0, u64::MAX))); + assert_eq!(meter.try_consume(Weight::from_parts(0, u64::MAX)), Ok(())); - assert!(!meter.can_accrue(Weight::from_parts(0, 1))); - assert!(!meter.check_accrue(Weight::from_parts(0, 1))); + assert!(!meter.can_consume(Weight::from_parts(0, 1))); + assert_eq!(meter.try_consume(Weight::from_parts(0, 1)), Err(())); - assert!(!meter.can_accrue(Weight::from_parts(1, 0))); - assert!(!meter.check_accrue(Weight::from_parts(1, 0))); + assert!(!meter.can_consume(Weight::from_parts(1, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(1, 0)), Err(())); - assert!(meter.can_accrue(Weight::zero())); - assert!(meter.check_accrue(Weight::zero())); + assert!(meter.can_consume(Weight::zero())); + assert_eq!(meter.try_consume(Weight::zero()), Ok(())); } #[test] fn consumed_ratio_works() { let mut meter = WeightMeter::with_limit(Weight::from_parts(10, 20)); - assert!(meter.check_accrue(Weight::from_parts(5, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(5, 0)), Ok(())); assert_eq!(meter.consumed_ratio(), Perbill::from_percent(50)); - assert!(meter.check_accrue(Weight::from_parts(0, 12))); + assert_eq!(meter.try_consume(Weight::from_parts(0, 12)), Ok(())); assert_eq!(meter.consumed_ratio(), Perbill::from_percent(60)); - assert!(meter.check_accrue(Weight::from_parts(2, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(2, 0)), Ok(())); assert_eq!(meter.consumed_ratio(), Perbill::from_percent(70)); - assert!(meter.check_accrue(Weight::from_parts(0, 4))); + assert_eq!(meter.try_consume(Weight::from_parts(0, 4)), Ok(())); assert_eq!(meter.consumed_ratio(), Perbill::from_percent(80)); - assert!(meter.check_accrue(Weight::from_parts(3, 0))); + assert_eq!(meter.try_consume(Weight::from_parts(3, 0)), Ok(())); assert_eq!(meter.consumed_ratio(), Perbill::from_percent(100)); - assert!(meter.check_accrue(Weight::from_parts(0, 4))); + assert_eq!(meter.try_consume(Weight::from_parts(0, 4)), Ok(())); assert_eq!(meter.consumed_ratio(), Perbill::from_percent(100)); } From a035dc9be718c6327420c87fecb7a752954492f4 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 29 Feb 2024 09:12:02 +0200 Subject: [PATCH 09/13] Remove `AssignmentProviderConfig` and use parameters from `HostConfiguration` instead (#3181) This PR removes `AssignmentProviderConfig` and uses the corresponding ondemand parameters from `HostConfiguration` instead. Additionally `scheduling_lookahead` and all coretime/ondemand related parameters are extracted in a separate struct - `SchedulerParams`. The most relevant commit from the PR is [this one](https://github.com/paritytech/polkadot-sdk/pull/3181/commits/830bc0f5e858944474171bbe33382ad96040b535). Fixes https://github.com/paritytech/polkadot-sdk/issues/2268 --------- Co-authored-by: command-bot <> --- .../zombienet/tests/0002-pov_recovery.toml | 2 +- polkadot/node/service/src/chain_spec.rs | 16 +- polkadot/node/test/service/src/chain_spec.rs | 11 +- polkadot/primitives/src/vstaging/mod.rs | 76 ++++ .../implementers-guide/src/types/runtime.md | 118 +----- .../src/assigner_coretime/mock_helpers.rs | 11 +- .../parachains/src/assigner_coretime/mod.rs | 16 +- .../src/assigner_on_demand/benchmarking.rs | 2 +- .../src/assigner_on_demand/mock_helpers.rs | 11 +- .../parachains/src/assigner_on_demand/mod.rs | 18 +- .../parachains/src/assigner_parachains.rs | 12 +- .../src/assigner_parachains/mock_helpers.rs | 11 +- polkadot/runtime/parachains/src/builder.rs | 19 +- .../runtime/parachains/src/configuration.rs | 133 +++---- .../src/configuration/benchmarking.rs | 2 + .../parachains/src/configuration/migration.rs | 1 + .../src/configuration/migration/v11.rs | 113 +++++- .../src/configuration/migration/v12.rs | 349 ++++++++++++++++++ .../parachains/src/configuration/tests.rs | 51 ++- .../parachains/src/coretime/migration.rs | 19 +- .../runtime/parachains/src/coretime/mod.rs | 4 +- .../runtime/parachains/src/inclusion/tests.rs | 10 +- polkadot/runtime/parachains/src/mock.rs | 23 +- .../runtime/parachains/src/paras/tests.rs | 7 +- .../parachains/src/paras_inherent/tests.rs | 11 +- polkadot/runtime/parachains/src/scheduler.rs | 43 +-- .../parachains/src/scheduler/common.rs | 19 +- .../runtime/parachains/src/scheduler/tests.rs | 81 ++-- .../parachains/src/session_info/tests.rs | 4 +- polkadot/runtime/rococo/src/lib.rs | 1 + .../runtime_parachains_configuration.rs | 50 ++- polkadot/runtime/westend/src/lib.rs | 1 + .../runtime_parachains_configuration.rs | 50 ++- .../functional/0002-parachains-disputes.toml | 4 +- .../0004-parachains-garbage-candidate.toml | 4 +- ...0005-parachains-disputes-past-session.toml | 4 +- .../0006-parachains-max-tranche0.toml | 4 +- .../0007-dispute-freshly-finalized.toml | 4 +- .../0008-dispute-old-finalized.toml | 4 +- .../functional/0010-validator-disabling.toml | 4 +- .../0011-async-backing-6-seconds-rate.toml | 7 +- .../functional/0012-elastic-scaling-mvp.toml | 6 +- .../zombienet_tests/misc/0001-paritydb.toml | 4 +- 43 files changed, 901 insertions(+), 439 deletions(-) create mode 100644 polkadot/runtime/parachains/src/configuration/migration/v12.rs diff --git a/cumulus/zombienet/tests/0002-pov_recovery.toml b/cumulus/zombienet/tests/0002-pov_recovery.toml index fe42fd4b2f66..15a61eba2a03 100644 --- a/cumulus/zombienet/tests/0002-pov_recovery.toml +++ b/cumulus/zombienet/tests/0002-pov_recovery.toml @@ -4,7 +4,7 @@ default_command = "polkadot" chain = "rococo-local" -[relaychain.genesis.runtimeGenesis.patch.configuration.config] +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] # set parameters such that collators only connect to 1 validator as a backing group max_validators_per_core = 1 group_rotation_frequency = 100 # 10 mins diff --git a/polkadot/node/service/src/chain_spec.rs b/polkadot/node/service/src/chain_spec.rs index af241d1cbc55..af05af87a46b 100644 --- a/polkadot/node/service/src/chain_spec.rs +++ b/polkadot/node/service/src/chain_spec.rs @@ -24,6 +24,8 @@ use polkadot_primitives::{AccountId, AccountPublic, AssignmentId, ValidatorId}; use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; use sp_consensus_babe::AuthorityId as BabeId; +#[cfg(any(feature = "rococo-native", feature = "westend-native",))] +use polkadot_primitives::vstaging::SchedulerParams; #[cfg(feature = "rococo-native")] use rococo_runtime as rococo; #[cfg(feature = "rococo-native")] @@ -129,8 +131,6 @@ fn default_parachains_host_configuration( max_code_size: MAX_CODE_SIZE, max_pov_size: MAX_POV_SIZE, max_head_data_size: 32 * 1024, - group_rotation_frequency: 20, - paras_availability_period: 4, max_upward_queue_count: 8, max_upward_queue_size: 1024 * 1024, max_downward_message_size: 1024 * 1024, @@ -151,11 +151,16 @@ fn default_parachains_host_configuration( relay_vrf_modulo_samples: 2, zeroth_delay_tranche_width: 0, minimum_validation_upgrade_delay: 5, - scheduling_lookahead: 2, async_backing_params: AsyncBackingParams { max_candidate_depth: 3, allowed_ancestry_len: 2, }, + scheduler_params: SchedulerParams { + lookahead: 2, + group_rotation_frequency: 20, + paras_availability_period: 4, + ..Default::default() + }, ..Default::default() } } @@ -891,7 +896,10 @@ pub fn rococo_testnet_genesis( "sudo": { "key": Some(root_key.clone()) }, "configuration": { "config": polkadot_runtime_parachains::configuration::HostConfiguration { - max_validators_per_core: Some(1), + scheduler_params: SchedulerParams { + max_validators_per_core: Some(1), + ..default_parachains_host_configuration().scheduler_params + }, ..default_parachains_host_configuration() }, }, diff --git a/polkadot/node/test/service/src/chain_spec.rs b/polkadot/node/test/service/src/chain_spec.rs index 4cc387317c3f..f14fa9fde58b 100644 --- a/polkadot/node/test/service/src/chain_spec.rs +++ b/polkadot/node/test/service/src/chain_spec.rs @@ -19,7 +19,9 @@ use babe_primitives::AuthorityId as BabeId; use grandpa::AuthorityId as GrandpaId; use pallet_staking::Forcing; -use polkadot_primitives::{AccountId, AssignmentId, ValidatorId, MAX_CODE_SIZE, MAX_POV_SIZE}; +use polkadot_primitives::{ + vstaging::SchedulerParams, AccountId, AssignmentId, ValidatorId, MAX_CODE_SIZE, MAX_POV_SIZE, +}; use polkadot_service::chain_spec::{get_account_id_from_seed, get_from_seed, Extensions}; use polkadot_test_runtime::BABE_GENESIS_EPOCH_CONFIG; use sc_chain_spec::{ChainSpec, ChainType}; @@ -165,11 +167,14 @@ fn polkadot_testnet_genesis( max_code_size: MAX_CODE_SIZE, max_pov_size: MAX_POV_SIZE, max_head_data_size: 32 * 1024, - group_rotation_frequency: 20, - paras_availability_period: 4, no_show_slots: 10, minimum_validation_upgrade_delay: 5, max_downward_message_size: 1024, + scheduler_params: SchedulerParams { + group_rotation_frequency: 20, + paras_availability_period: 4, + ..Default::default() + }, ..Default::default() }, } diff --git a/polkadot/primitives/src/vstaging/mod.rs b/polkadot/primitives/src/vstaging/mod.rs index 39d9dfc02c5b..bd2a5106c44d 100644 --- a/polkadot/primitives/src/vstaging/mod.rs +++ b/polkadot/primitives/src/vstaging/mod.rs @@ -23,6 +23,7 @@ use sp_std::prelude::*; use parity_scale_codec::{Decode, Encode}; use primitives::RuntimeDebug; use scale_info::TypeInfo; +use sp_arithmetic::Perbill; /// Approval voting configuration parameters #[derive( @@ -50,6 +51,81 @@ impl Default for ApprovalVotingParams { } } +/// Scheduler configuration parameters. All coretime/ondemand parameters are here. +#[derive( + RuntimeDebug, + Copy, + Clone, + PartialEq, + Encode, + Decode, + TypeInfo, + serde::Serialize, + serde::Deserialize, +)] +pub struct SchedulerParams { + /// How often parachain groups should be rotated across parachains. + /// + /// Must be non-zero. + pub group_rotation_frequency: BlockNumber, + /// Availability timeout for a block on a core, measured in blocks. + /// + /// This is the maximum amount of blocks after a core became occupied that validators have time + /// to make the block available. + /// + /// This value only has effect on group rotations. If backers backed something at the end of + /// their rotation, the occupied core affects the backing group that comes afterwards. We limit + /// the effect one backing group can have on the next to `paras_availability_period` blocks. + /// + /// Within a group rotation there is no timeout as backers are only affecting themselves. + /// + /// Must be at least 1. With a value of 1, the previous group will not be able to negatively + /// affect the following group at the expense of a tight availability timeline at group + /// rotation boundaries. + pub paras_availability_period: BlockNumber, + /// The maximum number of validators to have per core. + /// + /// `None` means no maximum. + pub max_validators_per_core: Option, + /// The amount of blocks ahead to schedule paras. + pub lookahead: u32, + /// How many cores are managed by the coretime chain. + pub num_cores: u32, + /// The max number of times a claim can time out in availability. + pub max_availability_timeouts: u32, + /// The maximum queue size of the pay as you go module. + pub on_demand_queue_max_size: u32, + /// The target utilization of the spot price queue in percentages. + pub on_demand_target_queue_utilization: Perbill, + /// How quickly the fee rises in reaction to increased utilization. + /// The lower the number the slower the increase. + pub on_demand_fee_variability: Perbill, + /// The minimum amount needed to claim a slot in the spot pricing queue. + pub on_demand_base_fee: Balance, + /// The number of blocks a claim stays in the scheduler's claimqueue before getting cleared. + /// This number should go reasonably higher than the number of blocks in the async backing + /// lookahead. + pub ttl: BlockNumber, +} + +impl> Default for SchedulerParams { + fn default() -> Self { + Self { + group_rotation_frequency: 1u32.into(), + paras_availability_period: 1u32.into(), + max_validators_per_core: Default::default(), + lookahead: 1, + num_cores: Default::default(), + max_availability_timeouts: Default::default(), + on_demand_queue_max_size: ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, + on_demand_target_queue_utilization: Perbill::from_percent(25), + on_demand_fee_variability: Perbill::from_percent(3), + on_demand_base_fee: 10_000_000u128, + ttl: 5u32.into(), + } + } +} + use bitvec::vec::BitVec; /// Bit indices in the `HostConfiguration.node_features` that correspond to different node features. diff --git a/polkadot/roadmap/implementers-guide/src/types/runtime.md b/polkadot/roadmap/implementers-guide/src/types/runtime.md index 4b97409f8df3..1dfabd96db7c 100644 --- a/polkadot/roadmap/implementers-guide/src/types/runtime.md +++ b/polkadot/roadmap/implementers-guide/src/types/runtime.md @@ -4,106 +4,24 @@ Types used within the runtime exclusively and pervasively. ## Host Configuration -The internal-to-runtime configuration of the parachain host. This is expected to be altered only by governance procedures. - -```rust -struct HostConfiguration { - /// The minimum period, in blocks, between which parachains can update their validation code. - pub validation_upgrade_cooldown: BlockNumber, - /// The delay, in blocks, before a validation upgrade is applied. - pub validation_upgrade_delay: BlockNumber, - /// How long to keep code on-chain, in blocks. This should be sufficiently long that disputes - /// have concluded. - pub code_retention_period: BlockNumber, - /// The maximum validation code size, in bytes. - pub max_code_size: u32, - /// The maximum head-data size, in bytes. - pub max_head_data_size: u32, - /// The amount of availability cores to dedicate to parathreads (on-demand parachains). - pub parathread_cores: u32, - /// The number of retries that a parathread (on-demand parachain) author has to submit their block. - pub parathread_retries: u32, - /// How often parachain groups should be rotated across parachains. - pub group_rotation_frequency: BlockNumber, - /// The availability period, in blocks, for parachains. This is the amount of blocks - /// after inclusion that validators have to make the block available and signal its availability to - /// the chain. Must be at least 1. - pub chain_availability_period: BlockNumber, - /// The availability period, in blocks, for parathreads (on-demand parachains). Same as the `chain_availability_period`, - /// but a differing timeout due to differing requirements. Must be at least 1. - pub thread_availability_period: BlockNumber, - /// The amount of blocks ahead to schedule on-demand parachains. - pub scheduling_lookahead: u32, - /// The maximum number of validators to have per core. `None` means no maximum. - pub max_validators_per_core: Option, - /// The maximum number of validators to use for parachains, in total. `None` means no maximum. - pub max_validators: Option, - /// The amount of sessions to keep for disputes. - pub dispute_period: SessionIndex, - /// How long after dispute conclusion to accept statements. - pub dispute_post_conclusion_acceptance_period: BlockNumber, - /// The maximum number of dispute spam slots - pub dispute_max_spam_slots: u32, - /// The amount of consensus slots that must pass between submitting an assignment and - /// submitting an approval vote before a validator is considered a no-show. - /// Must be at least 1. - pub no_show_slots: u32, - /// The number of delay tranches in total. - pub n_delay_tranches: u32, - /// The width of the zeroth delay tranche for approval assignments. This many delay tranches - /// beyond 0 are all consolidated to form a wide 0 tranche. - pub zeroth_delay_tranche_width: u32, - /// The number of validators needed to approve a block. - pub needed_approvals: u32, - /// The number of samples to use in `RelayVRFModulo` or `RelayVRFModuloCompact` approval assignment criterions. - pub relay_vrf_modulo_samples: u32, - /// Total number of individual messages allowed in the parachain -> relay-chain message queue. - pub max_upward_queue_count: u32, - /// Total size of messages allowed in the parachain -> relay-chain message queue before which - /// no further messages may be added to it. If it exceeds this then the queue may contain only - /// a single message. - pub max_upward_queue_size: u32, - /// The maximum size of an upward message that can be sent by a candidate. - /// - /// This parameter affects the upper bound of size of `CandidateCommitments`. - pub max_upward_message_size: u32, - /// The maximum number of messages that a candidate can contain. - /// - /// This parameter affects the upper bound of size of `CandidateCommitments`. - pub max_upward_message_num_per_candidate: u32, - /// The maximum size of a message that can be put in a downward message queue. - /// - /// Since we require receiving at least one DMP message the obvious upper bound of the size is - /// the PoV size. Of course, there is a lot of other different things that a parachain may - /// decide to do with its PoV so this value in practice will be picked as a fraction of the PoV - /// size. - pub max_downward_message_size: u32, - /// The deposit that the sender should provide for opening an HRMP channel. - pub hrmp_sender_deposit: u32, - /// The deposit that the recipient should provide for accepting opening an HRMP channel. - pub hrmp_recipient_deposit: u32, - /// The maximum number of messages allowed in an HRMP channel at once. - pub hrmp_channel_max_capacity: u32, - /// The maximum total size of messages in bytes allowed in an HRMP channel at once. - pub hrmp_channel_max_total_size: u32, - /// The maximum number of inbound HRMP channels a parachain is allowed to accept. - pub hrmp_max_parachain_inbound_channels: u32, - /// The maximum number of inbound HRMP channels a parathread (on-demand parachain) is allowed to accept. - pub hrmp_max_parathread_inbound_channels: u32, - /// The maximum size of a message that could ever be put into an HRMP channel. - /// - /// This parameter affects the upper bound of size of `CandidateCommitments`. - pub hrmp_channel_max_message_size: u32, - /// The maximum number of outbound HRMP channels a parachain is allowed to open. - pub hrmp_max_parachain_outbound_channels: u32, - /// The maximum number of outbound HRMP channels a parathread (on-demand parachain) is allowed to open. - pub hrmp_max_parathread_outbound_channels: u32, - /// The maximum number of outbound HRMP messages can be sent by a candidate. - /// - /// This parameter affects the upper bound of size of `CandidateCommitments`. - pub hrmp_max_message_num_per_candidate: u32, -} -``` +The internal-to-runtime configuration of the parachain host is kept in `struct HostConfiguration`. This is expected to +be altered only by governance procedures or via migrations from the Polkadot-SDK codebase. The latest definition of +`HostConfiguration` can be found in the project repo +[here](https://github.com/paritytech/polkadot-sdk/blob/master/polkadot/runtime/parachains/src/configuration.rs). Each +parameter has got a doc comment so for any details please refer to the code. + +Some related parameters in `HostConfiguration` are grouped together so that they can be managed easily. These are: +* `async_backing_params` in `struct AsyncBackingParams` +* `executor_params` in `struct ExecutorParams` +* `approval_voting_params` in `struct ApprovalVotingParams` +* `scheduler_params` in `struct SchedulerParams` + +Check the definitions of these structs for further details. + +### Configuration migrations +Modifying `HostConfiguration` requires a storage migration. These migrations are located in the +[`migrations`](https://github.com/paritytech/polkadot-sdk/blob/master/polkadot/runtime/parachains/src/configuration.rs) +subfolder of Polkadot-SDK repo. ## ParaInherentData diff --git a/polkadot/runtime/parachains/src/assigner_coretime/mock_helpers.rs b/polkadot/runtime/parachains/src/assigner_coretime/mock_helpers.rs index 71c3f1fa39f7..e2ba0b4f7ea5 100644 --- a/polkadot/runtime/parachains/src/assigner_coretime/mock_helpers.rs +++ b/polkadot/runtime/parachains/src/assigner_coretime/mock_helpers.rs @@ -64,11 +64,12 @@ impl GenesisConfigBuilder { pub(super) fn build(self) -> MockGenesisConfig { let mut genesis = default_genesis_config(); let config = &mut genesis.configuration.config; - config.coretime_cores = self.on_demand_cores; - config.on_demand_base_fee = self.on_demand_base_fee; - config.on_demand_fee_variability = self.on_demand_fee_variability; - config.on_demand_queue_max_size = self.on_demand_max_queue_size; - config.on_demand_target_queue_utilization = self.on_demand_target_queue_utilization; + config.scheduler_params.num_cores = self.on_demand_cores; + config.scheduler_params.on_demand_base_fee = self.on_demand_base_fee; + config.scheduler_params.on_demand_fee_variability = self.on_demand_fee_variability; + config.scheduler_params.on_demand_queue_max_size = self.on_demand_max_queue_size; + config.scheduler_params.on_demand_target_queue_utilization = + self.on_demand_target_queue_utilization; let paras = &mut genesis.paras.paras; for para_id in self.onboarded_on_demand_chains { diff --git a/polkadot/runtime/parachains/src/assigner_coretime/mod.rs b/polkadot/runtime/parachains/src/assigner_coretime/mod.rs index 9da81dc816ca..e2da89fadd48 100644 --- a/polkadot/runtime/parachains/src/assigner_coretime/mod.rs +++ b/polkadot/runtime/parachains/src/assigner_coretime/mod.rs @@ -30,7 +30,7 @@ mod tests; use crate::{ assigner_on_demand, configuration, paras::AssignCoretime, - scheduler::common::{Assignment, AssignmentProvider, AssignmentProviderConfig}, + scheduler::common::{Assignment, AssignmentProvider}, ParaId, }; @@ -316,14 +316,6 @@ impl AssignmentProvider> for Pallet { } } - fn get_provider_config(_core_idx: CoreIndex) -> AssignmentProviderConfig> { - let config = >::config(); - AssignmentProviderConfig { - max_availability_timeouts: config.on_demand_retries, - ttl: config.on_demand_ttl, - } - } - #[cfg(any(feature = "runtime-benchmarks", test))] fn get_mock_assignment(_: CoreIndex, para_id: primitives::Id) -> Assignment { // Given that we are not tracking anything in `Bulk` assignments, it is safe to always @@ -333,7 +325,7 @@ impl AssignmentProvider> for Pallet { fn session_core_count() -> u32 { let config = >::config(); - config.coretime_cores + config.scheduler_params.num_cores } } @@ -482,8 +474,8 @@ impl AssignCoretime for Pallet { // Add a new core and assign the para to it. let mut config = >::config(); - let core = config.coretime_cores; - config.coretime_cores.saturating_inc(); + let core = config.scheduler_params.num_cores; + config.scheduler_params.num_cores.saturating_inc(); // `assign_coretime` is only called at genesis or by root, so setting the active // config here is fine. diff --git a/polkadot/runtime/parachains/src/assigner_on_demand/benchmarking.rs b/polkadot/runtime/parachains/src/assigner_on_demand/benchmarking.rs index 5a6060cd2b4e..8360e7a78d0a 100644 --- a/polkadot/runtime/parachains/src/assigner_on_demand/benchmarking.rs +++ b/polkadot/runtime/parachains/src/assigner_on_demand/benchmarking.rs @@ -43,7 +43,7 @@ where { ParasShared::::set_session_index(SESSION_INDEX); let mut config = HostConfiguration::default(); - config.coretime_cores = 1; + config.scheduler_params.num_cores = 1; ConfigurationPallet::::force_set_active_config(config); let mut parachains = ParachainsCache::new(); ParasPallet::::initialize_para_now( diff --git a/polkadot/runtime/parachains/src/assigner_on_demand/mock_helpers.rs b/polkadot/runtime/parachains/src/assigner_on_demand/mock_helpers.rs index de30330ac84e..f8d1a894f0e4 100644 --- a/polkadot/runtime/parachains/src/assigner_on_demand/mock_helpers.rs +++ b/polkadot/runtime/parachains/src/assigner_on_demand/mock_helpers.rs @@ -63,11 +63,12 @@ impl GenesisConfigBuilder { pub(super) fn build(self) -> MockGenesisConfig { let mut genesis = default_genesis_config(); let config = &mut genesis.configuration.config; - config.coretime_cores = self.on_demand_cores; - config.on_demand_base_fee = self.on_demand_base_fee; - config.on_demand_fee_variability = self.on_demand_fee_variability; - config.on_demand_queue_max_size = self.on_demand_max_queue_size; - config.on_demand_target_queue_utilization = self.on_demand_target_queue_utilization; + config.scheduler_params.num_cores = self.on_demand_cores; + config.scheduler_params.on_demand_base_fee = self.on_demand_base_fee; + config.scheduler_params.on_demand_fee_variability = self.on_demand_fee_variability; + config.scheduler_params.on_demand_queue_max_size = self.on_demand_max_queue_size; + config.scheduler_params.on_demand_target_queue_utilization = + self.on_demand_target_queue_utilization; let paras = &mut genesis.paras.paras; for para_id in self.onboarded_on_demand_chains { diff --git a/polkadot/runtime/parachains/src/assigner_on_demand/mod.rs b/polkadot/runtime/parachains/src/assigner_on_demand/mod.rs index 1b746e88694c..bc450dc78129 100644 --- a/polkadot/runtime/parachains/src/assigner_on_demand/mod.rs +++ b/polkadot/runtime/parachains/src/assigner_on_demand/mod.rs @@ -201,10 +201,10 @@ pub mod pallet { let old_traffic = SpotTraffic::::get(); match Self::calculate_spot_traffic( old_traffic, - config.on_demand_queue_max_size, + config.scheduler_params.on_demand_queue_max_size, Self::queue_size(), - config.on_demand_target_queue_utilization, - config.on_demand_fee_variability, + config.scheduler_params.on_demand_target_queue_utilization, + config.scheduler_params.on_demand_fee_variability, ) { Ok(new_traffic) => { // Only update storage on change @@ -330,8 +330,9 @@ where let traffic = SpotTraffic::::get(); // Calculate spot price - let spot_price: BalanceOf = - traffic.saturating_mul_int(config.on_demand_base_fee.saturated_into::>()); + let spot_price: BalanceOf = traffic.saturating_mul_int( + config.scheduler_params.on_demand_base_fee.saturated_into::>(), + ); // Is the current price higher than `max_amount` ensure!(spot_price.le(&max_amount), Error::::SpotPriceHigherThanMaxAmount); @@ -450,7 +451,10 @@ where OnDemandQueue::::try_mutate(|queue| { // Abort transaction if queue is too large - ensure!(Self::queue_size() < config.on_demand_queue_max_size, Error::::QueueFull); + ensure!( + Self::queue_size() < config.scheduler_params.on_demand_queue_max_size, + Error::::QueueFull + ); match location { QueuePushDirection::Back => queue.push_back(order), QueuePushDirection::Front => queue.push_front(order), @@ -472,7 +476,7 @@ where target: LOG_TARGET, "Failed to fetch the on demand queue size, returning the max size." ); - return config.on_demand_queue_max_size + return config.scheduler_params.on_demand_queue_max_size }, } } diff --git a/polkadot/runtime/parachains/src/assigner_parachains.rs b/polkadot/runtime/parachains/src/assigner_parachains.rs index 34b5d3c1ec51..b5f342563e97 100644 --- a/polkadot/runtime/parachains/src/assigner_parachains.rs +++ b/polkadot/runtime/parachains/src/assigner_parachains.rs @@ -27,7 +27,7 @@ use primitives::CoreIndex; use crate::{ configuration, paras, - scheduler::common::{Assignment, AssignmentProvider, AssignmentProviderConfig}, + scheduler::common::{Assignment, AssignmentProvider}, }; pub use pallet::*; @@ -58,16 +58,6 @@ impl AssignmentProvider> for Pallet { /// this is a no-op in the case of a bulk assignment slot. fn push_back_assignment(_: Assignment) {} - fn get_provider_config(_core_idx: CoreIndex) -> AssignmentProviderConfig> { - AssignmentProviderConfig { - // The next assignment already goes to the same [`ParaId`], no timeout tracking needed. - max_availability_timeouts: 0, - // The next assignment already goes to the same [`ParaId`], this can be any number - // that's high enough to clear the time it takes to clear backing/availability. - ttl: 10u32.into(), - } - } - #[cfg(any(feature = "runtime-benchmarks", test))] fn get_mock_assignment(_: CoreIndex, para_id: primitives::Id) -> Assignment { Assignment::Bulk(para_id) diff --git a/polkadot/runtime/parachains/src/assigner_parachains/mock_helpers.rs b/polkadot/runtime/parachains/src/assigner_parachains/mock_helpers.rs index e6e9fb074aa9..a46e114daeaf 100644 --- a/polkadot/runtime/parachains/src/assigner_parachains/mock_helpers.rs +++ b/polkadot/runtime/parachains/src/assigner_parachains/mock_helpers.rs @@ -60,11 +60,12 @@ impl GenesisConfigBuilder { pub(super) fn build(self) -> MockGenesisConfig { let mut genesis = default_genesis_config(); let config = &mut genesis.configuration.config; - config.coretime_cores = self.on_demand_cores; - config.on_demand_base_fee = self.on_demand_base_fee; - config.on_demand_fee_variability = self.on_demand_fee_variability; - config.on_demand_queue_max_size = self.on_demand_max_queue_size; - config.on_demand_target_queue_utilization = self.on_demand_target_queue_utilization; + config.scheduler_params.num_cores = self.on_demand_cores; + config.scheduler_params.on_demand_base_fee = self.on_demand_base_fee; + config.scheduler_params.on_demand_fee_variability = self.on_demand_fee_variability; + config.scheduler_params.on_demand_queue_max_size = self.on_demand_max_queue_size; + config.scheduler_params.on_demand_target_queue_utilization = + self.on_demand_target_queue_utilization; let paras = &mut genesis.paras.paras; for para_id in self.onboarded_on_demand_chains { diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 500bc70cfa75..0e4e659fef27 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -18,11 +18,7 @@ use crate::{ configuration, inclusion, initializer, paras, paras::ParaKind, paras_inherent, - scheduler::{ - self, - common::{AssignmentProvider, AssignmentProviderConfig}, - CoreOccupied, ParasEntry, - }, + scheduler::{self, common::AssignmentProvider, CoreOccupied, ParasEntry}, session_info, shared, }; use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; @@ -197,7 +193,10 @@ impl BenchBuilder { /// Maximum number of validators per core (a.k.a. max validators per group). This value is used /// if none is explicitly set on the builder. pub(crate) fn fallback_max_validators_per_core() -> u32 { - configuration::Pallet::::config().max_validators_per_core.unwrap_or(5) + configuration::Pallet::::config() + .scheduler_params + .max_validators_per_core + .unwrap_or(5) } /// Specify a mapping of core index/ para id to the number of dispute statements for the @@ -684,7 +683,7 @@ impl BenchBuilder { // We are currently in Session 0, so these changes will take effect in Session 2. Self::setup_para_ids(used_cores); configuration::ActiveConfig::::mutate(|c| { - c.coretime_cores = used_cores; + c.scheduler_params.num_cores = used_cores; }); let validator_ids = Self::generate_validator_pairs(self.max_validators()); @@ -715,8 +714,7 @@ impl BenchBuilder { let cores = (0..used_cores) .into_iter() .map(|i| { - let AssignmentProviderConfig { ttl, .. } = - scheduler::Pallet::::assignment_provider_config(CoreIndex(i)); + let ttl = configuration::Pallet::::config().scheduler_params.ttl; // Load an assignment into provider so that one is present to pop let assignment = ::AssignmentProvider::get_mock_assignment( CoreIndex(i), @@ -731,8 +729,7 @@ impl BenchBuilder { let cores = (0..used_cores) .into_iter() .map(|i| { - let AssignmentProviderConfig { ttl, .. } = - scheduler::Pallet::::assignment_provider_config(CoreIndex(i)); + let ttl = configuration::Pallet::::config().scheduler_params.ttl; // Load an assignment into provider so that one is present to pop let assignment = ::AssignmentProvider::get_mock_assignment( diff --git a/polkadot/runtime/parachains/src/configuration.rs b/polkadot/runtime/parachains/src/configuration.rs index b0e9d03df886..364a15215d38 100644 --- a/polkadot/runtime/parachains/src/configuration.rs +++ b/polkadot/runtime/parachains/src/configuration.rs @@ -29,7 +29,6 @@ use primitives::{ vstaging::{ApprovalVotingParams, NodeFeatures}, AsyncBackingParams, Balance, ExecutorParamError, ExecutorParams, SessionIndex, LEGACY_MIN_BACKING_VOTES, MAX_CODE_SIZE, MAX_HEAD_DATA_SIZE, MAX_POV_SIZE, - ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, }; use sp_runtime::{traits::Zero, Perbill}; use sp_std::prelude::*; @@ -43,6 +42,7 @@ mod benchmarking; pub mod migration; pub use pallet::*; +use primitives::vstaging::SchedulerParams; const LOG_TARGET: &str = "runtime::configuration"; @@ -118,9 +118,9 @@ pub struct HostConfiguration { /// been completed. /// /// Note, there are situations in which `expected_at` in the past. For example, if - /// [`paras_availability_period`](Self::paras_availability_period) is less than the delay set - /// by this field or if PVF pre-check took more time than the delay. In such cases, the upgrade - /// is further at the earliest possible time determined by + /// [`paras_availability_period`](SchedulerParams::paras_availability_period) is less than the + /// delay set by this field or if PVF pre-check took more time than the delay. In such cases, + /// the upgrade is further at the earliest possible time determined by /// [`minimum_validation_upgrade_delay`](Self::minimum_validation_upgrade_delay). /// /// The rationale for this delay has to do with relay-chain reversions. In case there is an @@ -172,48 +172,7 @@ pub struct HostConfiguration { /// How long to keep code on-chain, in blocks. This should be sufficiently long that disputes /// have concluded. pub code_retention_period: BlockNumber, - /// How many cores are managed by the coretime chain. - pub coretime_cores: u32, - /// The number of retries that a on demand author has to submit their block. - pub on_demand_retries: u32, - /// The maximum queue size of the pay as you go module. - pub on_demand_queue_max_size: u32, - /// The target utilization of the spot price queue in percentages. - pub on_demand_target_queue_utilization: Perbill, - /// How quickly the fee rises in reaction to increased utilization. - /// The lower the number the slower the increase. - pub on_demand_fee_variability: Perbill, - /// The minimum amount needed to claim a slot in the spot pricing queue. - pub on_demand_base_fee: Balance, - /// The number of blocks an on demand claim stays in the scheduler's claimqueue before getting - /// cleared. This number should go reasonably higher than the number of blocks in the async - /// backing lookahead. - pub on_demand_ttl: BlockNumber, - /// How often parachain groups should be rotated across parachains. - /// - /// Must be non-zero. - pub group_rotation_frequency: BlockNumber, - /// The minimum availability period, in blocks. - /// - /// This is the minimum amount of blocks after a core became occupied that validators have time - /// to make the block available. - /// - /// This value only has effect on group rotations. If backers backed something at the end of - /// their rotation, the occupied core affects the backing group that comes afterwards. We limit - /// the effect one backing group can have on the next to `paras_availability_period` blocks. - /// - /// Within a group rotation there is no timeout as backers are only affecting themselves. - /// - /// Must be at least 1. With a value of 1, the previous group will not be able to negatively - /// affect the following group at the expense of a tight availability timeline at group - /// rotation boundaries. - pub paras_availability_period: BlockNumber, - /// The amount of blocks ahead to schedule paras. - pub scheduling_lookahead: u32, - /// The maximum number of validators to have per core. - /// - /// `None` means no maximum. - pub max_validators_per_core: Option, + /// The maximum number of validators to use for parachain consensus, period. /// /// `None` means no maximum. @@ -257,7 +216,7 @@ pub struct HostConfiguration { /// scheduled. This number is controlled by this field. /// /// This value should be greater than - /// [`paras_availability_period`](Self::paras_availability_period). + /// [`paras_availability_period`](SchedulerParams::paras_availability_period). pub minimum_validation_upgrade_delay: BlockNumber, /// The minimum number of valid backing statements required to consider a parachain candidate /// backable. @@ -266,6 +225,8 @@ pub struct HostConfiguration { pub node_features: NodeFeatures, /// Params used by approval-voting pub approval_voting_params: ApprovalVotingParams, + /// Scheduler parameters + pub scheduler_params: SchedulerParams, } impl> Default for HostConfiguration { @@ -275,8 +236,6 @@ impl> Default for HostConfiguration> Default for HostConfiguration> Default for HostConfiguration { ZeroMinimumBackingVotes, /// `executor_params` are inconsistent. InconsistentExecutorParams { inner: ExecutorParamError }, + /// TTL should be bigger than lookahead + LookaheadExceedsTTL, } impl HostConfiguration @@ -373,11 +326,11 @@ where pub fn check_consistency(&self) -> Result<(), InconsistentError> { use InconsistentError::*; - if self.group_rotation_frequency.is_zero() { + if self.scheduler_params.group_rotation_frequency.is_zero() { return Err(ZeroGroupRotationFrequency) } - if self.paras_availability_period.is_zero() { + if self.scheduler_params.paras_availability_period.is_zero() { return Err(ZeroParasAvailabilityPeriod) } @@ -399,10 +352,11 @@ where return Err(MaxPovSizeExceedHardLimit { max_pov_size: self.max_pov_size }) } - if self.minimum_validation_upgrade_delay <= self.paras_availability_period { + if self.minimum_validation_upgrade_delay <= self.scheduler_params.paras_availability_period + { return Err(MinimumValidationUpgradeDelayLessThanChainAvailabilityPeriod { minimum_validation_upgrade_delay: self.minimum_validation_upgrade_delay.clone(), - paras_availability_period: self.paras_availability_period.clone(), + paras_availability_period: self.scheduler_params.paras_availability_period.clone(), }) } @@ -447,6 +401,10 @@ where return Err(InconsistentExecutorParams { inner }) } + if self.scheduler_params.ttl < self.scheduler_params.lookahead.into() { + return Err(LookaheadExceedsTTL) + } + Ok(()) } @@ -471,6 +429,7 @@ pub trait WeightInfo { fn set_config_with_executor_params() -> Weight; fn set_config_with_perbill() -> Weight; fn set_node_feature() -> Weight; + fn set_config_with_scheduler_params() -> Weight; } pub struct TestWeightInfo; @@ -499,6 +458,9 @@ impl WeightInfo for TestWeightInfo { fn set_node_feature() -> Weight { Weight::MAX } + fn set_config_with_scheduler_params() -> Weight { + Weight::MAX + } } #[frame_support::pallet] @@ -520,7 +482,8 @@ pub mod pallet { /// v8-v9: /// v9-v10: /// v10-11: - const STORAGE_VERSION: StorageVersion = StorageVersion::new(11); + /// v11-12: + const STORAGE_VERSION: StorageVersion = StorageVersion::new(12); #[pallet::pallet] #[pallet::storage_version(STORAGE_VERSION)] @@ -679,16 +642,16 @@ pub mod pallet { Self::set_coretime_cores_unchecked(new) } - /// Set the number of retries for a particular on demand. + /// Set the max number of times a claim may timeout on a core before it is abandoned #[pallet::call_index(7)] #[pallet::weight(( T::WeightInfo::set_config_with_u32(), DispatchClass::Operational, ))] - pub fn set_on_demand_retries(origin: OriginFor, new: u32) -> DispatchResult { + pub fn set_max_availability_timeouts(origin: OriginFor, new: u32) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.on_demand_retries = new; + config.scheduler_params.max_availability_timeouts = new; }) } @@ -704,7 +667,7 @@ pub mod pallet { ) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.group_rotation_frequency = new; + config.scheduler_params.group_rotation_frequency = new; }) } @@ -720,7 +683,7 @@ pub mod pallet { ) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.paras_availability_period = new; + config.scheduler_params.paras_availability_period = new; }) } @@ -733,7 +696,7 @@ pub mod pallet { pub fn set_scheduling_lookahead(origin: OriginFor, new: u32) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.scheduling_lookahead = new; + config.scheduler_params.lookahead = new; }) } @@ -749,7 +712,7 @@ pub mod pallet { ) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.max_validators_per_core = new; + config.scheduler_params.max_validators_per_core = new; }) } @@ -1141,7 +1104,7 @@ pub mod pallet { pub fn set_on_demand_base_fee(origin: OriginFor, new: Balance) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.on_demand_base_fee = new; + config.scheduler_params.on_demand_base_fee = new; }) } @@ -1154,7 +1117,7 @@ pub mod pallet { pub fn set_on_demand_fee_variability(origin: OriginFor, new: Perbill) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.on_demand_fee_variability = new; + config.scheduler_params.on_demand_fee_variability = new; }) } @@ -1167,7 +1130,7 @@ pub mod pallet { pub fn set_on_demand_queue_max_size(origin: OriginFor, new: u32) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.on_demand_queue_max_size = new; + config.scheduler_params.on_demand_queue_max_size = new; }) } /// Set the on demand (parathreads) fee variability. @@ -1182,7 +1145,7 @@ pub mod pallet { ) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.on_demand_target_queue_utilization = new; + config.scheduler_params.on_demand_target_queue_utilization = new; }) } /// Set the on demand (parathreads) ttl in the claimqueue. @@ -1194,7 +1157,7 @@ pub mod pallet { pub fn set_on_demand_ttl(origin: OriginFor, new: BlockNumberFor) -> DispatchResult { ensure_root(origin)?; Self::schedule_config_update(|config| { - config.on_demand_ttl = new; + config.scheduler_params.ttl = new; }) } @@ -1244,6 +1207,22 @@ pub mod pallet { config.approval_voting_params = new; }) } + + /// Set scheduler-params. + #[pallet::call_index(55)] + #[pallet::weight(( + T::WeightInfo::set_config_with_scheduler_params(), + DispatchClass::Operational, + ))] + pub fn set_scheduler_params( + origin: OriginFor, + new: SchedulerParams>, + ) -> DispatchResult { + ensure_root(origin)?; + Self::schedule_config_update(|config| { + config.scheduler_params = new; + }) + } } impl Pallet { @@ -1252,7 +1231,7 @@ pub mod pallet { /// To be used if authorization is checked otherwise. pub fn set_coretime_cores_unchecked(new: u32) -> DispatchResult { Self::schedule_config_update(|config| { - config.coretime_cores = new; + config.scheduler_params.num_cores = new; }) } } @@ -1393,7 +1372,7 @@ impl Pallet { let base_config_consistent = base_config.check_consistency().is_ok(); // Now, we need to decide what the new configuration should be. - // We also move the `base_config` to `new_config` to empahsize that the base config was + // We also move the `base_config` to `new_config` to emphasize that the base config was // destroyed by the `updater`. updater(&mut base_config); let new_config = base_config; diff --git a/polkadot/runtime/parachains/src/configuration/benchmarking.rs b/polkadot/runtime/parachains/src/configuration/benchmarking.rs index 67daf1c45988..882b5aab096a 100644 --- a/polkadot/runtime/parachains/src/configuration/benchmarking.rs +++ b/polkadot/runtime/parachains/src/configuration/benchmarking.rs @@ -51,6 +51,8 @@ benchmarks! { set_node_feature{}: set_node_feature(RawOrigin::Root, 255, true) + set_config_with_scheduler_params {} : set_scheduler_params(RawOrigin::Root, SchedulerParams::default()) + impl_benchmark_test_suite!( Pallet, crate::mock::new_test_ext(Default::default()), diff --git a/polkadot/runtime/parachains/src/configuration/migration.rs b/polkadot/runtime/parachains/src/configuration/migration.rs index 2838b73092db..87b30b177e73 100644 --- a/polkadot/runtime/parachains/src/configuration/migration.rs +++ b/polkadot/runtime/parachains/src/configuration/migration.rs @@ -18,6 +18,7 @@ pub mod v10; pub mod v11; +pub mod v12; pub mod v6; pub mod v7; pub mod v8; diff --git a/polkadot/runtime/parachains/src/configuration/migration/v11.rs b/polkadot/runtime/parachains/src/configuration/migration/v11.rs index a69061ac1381..f6e0e0431640 100644 --- a/polkadot/runtime/parachains/src/configuration/migration/v11.rs +++ b/polkadot/runtime/parachains/src/configuration/migration/v11.rs @@ -21,13 +21,122 @@ use frame_support::{ migrations::VersionedMigration, pallet_prelude::*, traits::Defensive, weights::Weight, }; use frame_system::pallet_prelude::BlockNumberFor; -use primitives::{vstaging::ApprovalVotingParams, SessionIndex}; +use primitives::{ + vstaging::ApprovalVotingParams, AsyncBackingParams, ExecutorParams, SessionIndex, + LEGACY_MIN_BACKING_VOTES, ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, +}; use sp_std::vec::Vec; use frame_support::traits::OnRuntimeUpgrade; +use polkadot_core_primitives::Balance; +use primitives::vstaging::NodeFeatures; +use sp_arithmetic::Perbill; use super::v10::V10HostConfiguration; -type V11HostConfiguration = configuration::HostConfiguration; + +#[derive(Clone, Encode, PartialEq, Decode, Debug)] +pub struct V11HostConfiguration { + pub max_code_size: u32, + pub max_head_data_size: u32, + pub max_upward_queue_count: u32, + pub max_upward_queue_size: u32, + pub max_upward_message_size: u32, + pub max_upward_message_num_per_candidate: u32, + pub hrmp_max_message_num_per_candidate: u32, + pub validation_upgrade_cooldown: BlockNumber, + pub validation_upgrade_delay: BlockNumber, + pub async_backing_params: AsyncBackingParams, + pub max_pov_size: u32, + pub max_downward_message_size: u32, + pub hrmp_max_parachain_outbound_channels: u32, + pub hrmp_sender_deposit: Balance, + pub hrmp_recipient_deposit: Balance, + pub hrmp_channel_max_capacity: u32, + pub hrmp_channel_max_total_size: u32, + pub hrmp_max_parachain_inbound_channels: u32, + pub hrmp_channel_max_message_size: u32, + pub executor_params: ExecutorParams, + pub code_retention_period: BlockNumber, + pub coretime_cores: u32, + pub on_demand_retries: u32, + pub on_demand_queue_max_size: u32, + pub on_demand_target_queue_utilization: Perbill, + pub on_demand_fee_variability: Perbill, + pub on_demand_base_fee: Balance, + pub on_demand_ttl: BlockNumber, + pub group_rotation_frequency: BlockNumber, + pub paras_availability_period: BlockNumber, + pub scheduling_lookahead: u32, + pub max_validators_per_core: Option, + pub max_validators: Option, + pub dispute_period: SessionIndex, + pub dispute_post_conclusion_acceptance_period: BlockNumber, + pub no_show_slots: u32, + pub n_delay_tranches: u32, + pub zeroth_delay_tranche_width: u32, + pub needed_approvals: u32, + pub relay_vrf_modulo_samples: u32, + pub pvf_voting_ttl: SessionIndex, + pub minimum_validation_upgrade_delay: BlockNumber, + pub minimum_backing_votes: u32, + pub node_features: NodeFeatures, + pub approval_voting_params: ApprovalVotingParams, +} + +impl> Default for V11HostConfiguration { + fn default() -> Self { + Self { + async_backing_params: AsyncBackingParams { + max_candidate_depth: 0, + allowed_ancestry_len: 0, + }, + group_rotation_frequency: 1u32.into(), + paras_availability_period: 1u32.into(), + no_show_slots: 1u32.into(), + validation_upgrade_cooldown: Default::default(), + validation_upgrade_delay: 2u32.into(), + code_retention_period: Default::default(), + max_code_size: Default::default(), + max_pov_size: Default::default(), + max_head_data_size: Default::default(), + coretime_cores: Default::default(), + on_demand_retries: Default::default(), + scheduling_lookahead: 1, + max_validators_per_core: Default::default(), + max_validators: None, + dispute_period: 6, + dispute_post_conclusion_acceptance_period: 100.into(), + n_delay_tranches: Default::default(), + zeroth_delay_tranche_width: Default::default(), + needed_approvals: Default::default(), + relay_vrf_modulo_samples: Default::default(), + max_upward_queue_count: Default::default(), + max_upward_queue_size: Default::default(), + max_downward_message_size: Default::default(), + max_upward_message_size: Default::default(), + max_upward_message_num_per_candidate: Default::default(), + hrmp_sender_deposit: Default::default(), + hrmp_recipient_deposit: Default::default(), + hrmp_channel_max_capacity: Default::default(), + hrmp_channel_max_total_size: Default::default(), + hrmp_max_parachain_inbound_channels: Default::default(), + hrmp_channel_max_message_size: Default::default(), + hrmp_max_parachain_outbound_channels: Default::default(), + hrmp_max_message_num_per_candidate: Default::default(), + pvf_voting_ttl: 2u32.into(), + minimum_validation_upgrade_delay: 2.into(), + executor_params: Default::default(), + approval_voting_params: ApprovalVotingParams { max_approval_coalesce_count: 1 }, + on_demand_queue_max_size: ON_DEMAND_DEFAULT_QUEUE_MAX_SIZE, + on_demand_base_fee: 10_000_000u128, + on_demand_fee_variability: Perbill::from_percent(3), + on_demand_target_queue_utilization: Perbill::from_percent(25), + on_demand_ttl: 5u32.into(), + minimum_backing_votes: LEGACY_MIN_BACKING_VOTES, + node_features: NodeFeatures::EMPTY, + } + } +} mod v10 { use super::*; diff --git a/polkadot/runtime/parachains/src/configuration/migration/v12.rs b/polkadot/runtime/parachains/src/configuration/migration/v12.rs new file mode 100644 index 000000000000..4295a79893e8 --- /dev/null +++ b/polkadot/runtime/parachains/src/configuration/migration/v12.rs @@ -0,0 +1,349 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! A module that is responsible for migration of storage. + +use crate::configuration::{self, migration::v11::V11HostConfiguration, Config, Pallet}; +use frame_support::{ + migrations::VersionedMigration, + pallet_prelude::*, + traits::{Defensive, OnRuntimeUpgrade}, +}; +use frame_system::pallet_prelude::BlockNumberFor; +use primitives::vstaging::SchedulerParams; +use sp_core::Get; +use sp_staking::SessionIndex; +use sp_std::vec::Vec; + +type V12HostConfiguration = configuration::HostConfiguration; + +mod v11 { + use super::*; + + #[frame_support::storage_alias] + pub(crate) type ActiveConfig = + StorageValue, V11HostConfiguration>, OptionQuery>; + + #[frame_support::storage_alias] + pub(crate) type PendingConfigs = StorageValue< + Pallet, + Vec<(SessionIndex, V11HostConfiguration>)>, + OptionQuery, + >; +} + +mod v12 { + use super::*; + + #[frame_support::storage_alias] + pub(crate) type ActiveConfig = + StorageValue, V12HostConfiguration>, OptionQuery>; + + #[frame_support::storage_alias] + pub(crate) type PendingConfigs = StorageValue< + Pallet, + Vec<(SessionIndex, V12HostConfiguration>)>, + OptionQuery, + >; +} + +pub type MigrateToV12 = VersionedMigration< + 11, + 12, + UncheckedMigrateToV12, + Pallet, + ::DbWeight, +>; + +pub struct UncheckedMigrateToV12(sp_std::marker::PhantomData); + +impl OnRuntimeUpgrade for UncheckedMigrateToV12 { + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { + log::trace!(target: crate::configuration::LOG_TARGET, "Running pre_upgrade() for HostConfiguration MigrateToV12"); + Ok(Vec::new()) + } + + fn on_runtime_upgrade() -> Weight { + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV12 started"); + let weight_consumed = migrate_to_v12::(); + + log::info!(target: configuration::LOG_TARGET, "HostConfiguration MigrateToV12 executed successfully"); + + weight_consumed + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(_state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { + log::trace!(target: crate::configuration::LOG_TARGET, "Running post_upgrade() for HostConfiguration MigrateToV12"); + ensure!( + StorageVersion::get::>() >= 12, + "Storage version should be >= 12 after the migration" + ); + + Ok(()) + } +} + +fn migrate_to_v12() -> Weight { + // Unusual formatting is justified: + // - make it easier to verify that fields assign what they supposed to assign. + // - this code is transient and will be removed after all migrations are done. + // - this code is important enough to optimize for legibility sacrificing consistency. + #[rustfmt::skip] + let translate = + |pre: V11HostConfiguration>| -> + V12HostConfiguration> + { + V12HostConfiguration { + max_code_size : pre.max_code_size, + max_head_data_size : pre.max_head_data_size, + max_upward_queue_count : pre.max_upward_queue_count, + max_upward_queue_size : pre.max_upward_queue_size, + max_upward_message_size : pre.max_upward_message_size, + max_upward_message_num_per_candidate : pre.max_upward_message_num_per_candidate, + hrmp_max_message_num_per_candidate : pre.hrmp_max_message_num_per_candidate, + validation_upgrade_cooldown : pre.validation_upgrade_cooldown, + validation_upgrade_delay : pre.validation_upgrade_delay, + max_pov_size : pre.max_pov_size, + max_downward_message_size : pre.max_downward_message_size, + hrmp_sender_deposit : pre.hrmp_sender_deposit, + hrmp_recipient_deposit : pre.hrmp_recipient_deposit, + hrmp_channel_max_capacity : pre.hrmp_channel_max_capacity, + hrmp_channel_max_total_size : pre.hrmp_channel_max_total_size, + hrmp_max_parachain_inbound_channels : pre.hrmp_max_parachain_inbound_channels, + hrmp_max_parachain_outbound_channels : pre.hrmp_max_parachain_outbound_channels, + hrmp_channel_max_message_size : pre.hrmp_channel_max_message_size, + code_retention_period : pre.code_retention_period, + max_validators : pre.max_validators, + dispute_period : pre.dispute_period, + dispute_post_conclusion_acceptance_period: pre.dispute_post_conclusion_acceptance_period, + no_show_slots : pre.no_show_slots, + n_delay_tranches : pre.n_delay_tranches, + zeroth_delay_tranche_width : pre.zeroth_delay_tranche_width, + needed_approvals : pre.needed_approvals, + relay_vrf_modulo_samples : pre.relay_vrf_modulo_samples, + pvf_voting_ttl : pre.pvf_voting_ttl, + minimum_validation_upgrade_delay : pre.minimum_validation_upgrade_delay, + async_backing_params : pre.async_backing_params, + executor_params : pre.executor_params, + minimum_backing_votes : pre.minimum_backing_votes, + node_features : pre.node_features, + approval_voting_params : pre.approval_voting_params, + scheduler_params: SchedulerParams { + group_rotation_frequency : pre.group_rotation_frequency, + paras_availability_period : pre.paras_availability_period, + max_validators_per_core : pre.max_validators_per_core, + lookahead : pre.scheduling_lookahead, + num_cores : pre.coretime_cores, + max_availability_timeouts : pre.on_demand_retries, + on_demand_queue_max_size : pre.on_demand_queue_max_size, + on_demand_target_queue_utilization : pre.on_demand_target_queue_utilization, + on_demand_fee_variability : pre.on_demand_fee_variability, + on_demand_base_fee : pre.on_demand_base_fee, + ttl : pre.on_demand_ttl, + } + } + }; + + let v11 = v11::ActiveConfig::::get() + .defensive_proof("Could not decode old config") + .unwrap_or_default(); + let v12 = translate(v11); + v12::ActiveConfig::::set(Some(v12)); + + // Allowed to be empty. + let pending_v11 = v11::PendingConfigs::::get().unwrap_or_default(); + let mut pending_v12 = Vec::new(); + + for (session, v11) in pending_v11.into_iter() { + let v12 = translate(v11); + pending_v12.push((session, v12)); + } + v12::PendingConfigs::::set(Some(pending_v12.clone())); + + let num_configs = (pending_v12.len() + 1) as u64; + T::DbWeight::get().reads_writes(num_configs, num_configs) +} + +#[cfg(test)] +mod tests { + use primitives::LEGACY_MIN_BACKING_VOTES; + use sp_arithmetic::Perbill; + + use super::*; + use crate::mock::{new_test_ext, Test}; + + #[test] + fn v12_deserialized_from_actual_data() { + // Example how to get new `raw_config`: + // We'll obtain the raw_config at a specified a block + // Steps: + // 1. Go to Polkadot.js -> Developer -> Chain state -> Storage: https://polkadot.js.org/apps/#/chainstate + // 2. Set these parameters: + // 2.1. selected state query: configuration; activeConfig(): + // PolkadotRuntimeParachainsConfigurationHostConfiguration + // 2.2. blockhash to query at: + // 0xf89d3ab5312c5f70d396dc59612f0aa65806c798346f9db4b35278baed2e0e53 (the hash of + // the block) + // 2.3. Note the value of encoded storage key -> + // 0x06de3d8a54d27e44a9d5ce189618f22db4b49d95320d9021994c850f25b8e385 for the + // referenced block. + // 2.4. You'll also need the decoded values to update the test. + // 3. Go to Polkadot.js -> Developer -> Chain state -> Raw storage + // 3.1 Enter the encoded storage key and you get the raw config. + + // This exceeds the maximal line width length, but that's fine, since this is not code and + // doesn't need to be read and also leaving it as one line allows to easily copy it. + let raw_config = + hex_literal::hex![ + "0000300000800000080000000000100000c8000005000000050000000200000002000000000000000000000000005000000010000400000000000000000000000000000000000000000000000000000000000000000000000800000000200000040000000000100000b004000000060000006400000002000000190000000000000002000000020000000200000005000000020000000001000000140000000400000001010000000100000001000000000000001027000080b2e60e80c3c9018096980000000000000000000000000005000000" + ]; + + let v12 = + V12HostConfiguration::::decode(&mut &raw_config[..]).unwrap(); + + // We check only a sample of the values here. If we missed any fields or messed up data + // types that would skew all the fields coming after. + assert_eq!(v12.max_code_size, 3_145_728); + assert_eq!(v12.validation_upgrade_cooldown, 2); + assert_eq!(v12.max_pov_size, 5_242_880); + assert_eq!(v12.hrmp_channel_max_message_size, 1_048_576); + assert_eq!(v12.n_delay_tranches, 25); + assert_eq!(v12.minimum_validation_upgrade_delay, 5); + assert_eq!(v12.minimum_backing_votes, LEGACY_MIN_BACKING_VOTES); + assert_eq!(v12.approval_voting_params.max_approval_coalesce_count, 1); + assert_eq!(v12.scheduler_params.group_rotation_frequency, 20); + assert_eq!(v12.scheduler_params.paras_availability_period, 4); + assert_eq!(v12.scheduler_params.lookahead, 1); + assert_eq!(v12.scheduler_params.num_cores, 1); + assert_eq!(v12.scheduler_params.max_availability_timeouts, 0); + assert_eq!(v12.scheduler_params.on_demand_queue_max_size, 10_000); + assert_eq!( + v12.scheduler_params.on_demand_target_queue_utilization, + Perbill::from_percent(25) + ); + assert_eq!(v12.scheduler_params.on_demand_fee_variability, Perbill::from_percent(3)); + assert_eq!(v12.scheduler_params.on_demand_base_fee, 10_000_000); + assert_eq!(v12.scheduler_params.ttl, 5); + } + + #[test] + fn test_migrate_to_v12() { + // Host configuration has lots of fields. However, in this migration we only add one + // field. The most important part to check are a couple of the last fields. We also pick + // extra fields to check arbitrarily, e.g. depending on their position (i.e. the middle) and + // also their type. + // + // We specify only the picked fields and the rest should be provided by the `Default` + // implementation. That implementation is copied over between the two types and should work + // fine. + let v11 = V11HostConfiguration:: { + needed_approvals: 69, + paras_availability_period: 55, + hrmp_recipient_deposit: 1337, + max_pov_size: 1111, + minimum_validation_upgrade_delay: 20, + on_demand_ttl: 3, + on_demand_retries: 10, + ..Default::default() + }; + + let mut pending_configs = Vec::new(); + pending_configs.push((100, v11.clone())); + pending_configs.push((300, v11.clone())); + + new_test_ext(Default::default()).execute_with(|| { + // Implant the v10 version in the state. + v11::ActiveConfig::::set(Some(v11.clone())); + v11::PendingConfigs::::set(Some(pending_configs)); + + migrate_to_v12::(); + + let v12 = v12::ActiveConfig::::get().unwrap(); + assert_eq!(v12.approval_voting_params.max_approval_coalesce_count, 1); + + let mut configs_to_check = v12::PendingConfigs::::get().unwrap(); + configs_to_check.push((0, v12.clone())); + + for (_, v12) in configs_to_check { + #[rustfmt::skip] + { + assert_eq!(v11.max_code_size , v12.max_code_size); + assert_eq!(v11.max_head_data_size , v12.max_head_data_size); + assert_eq!(v11.max_upward_queue_count , v12.max_upward_queue_count); + assert_eq!(v11.max_upward_queue_size , v12.max_upward_queue_size); + assert_eq!(v11.max_upward_message_size , v12.max_upward_message_size); + assert_eq!(v11.max_upward_message_num_per_candidate , v12.max_upward_message_num_per_candidate); + assert_eq!(v11.hrmp_max_message_num_per_candidate , v12.hrmp_max_message_num_per_candidate); + assert_eq!(v11.validation_upgrade_cooldown , v12.validation_upgrade_cooldown); + assert_eq!(v11.validation_upgrade_delay , v12.validation_upgrade_delay); + assert_eq!(v11.max_pov_size , v12.max_pov_size); + assert_eq!(v11.max_downward_message_size , v12.max_downward_message_size); + assert_eq!(v11.hrmp_max_parachain_outbound_channels , v12.hrmp_max_parachain_outbound_channels); + assert_eq!(v11.hrmp_sender_deposit , v12.hrmp_sender_deposit); + assert_eq!(v11.hrmp_recipient_deposit , v12.hrmp_recipient_deposit); + assert_eq!(v11.hrmp_channel_max_capacity , v12.hrmp_channel_max_capacity); + assert_eq!(v11.hrmp_channel_max_total_size , v12.hrmp_channel_max_total_size); + assert_eq!(v11.hrmp_max_parachain_inbound_channels , v12.hrmp_max_parachain_inbound_channels); + assert_eq!(v11.hrmp_channel_max_message_size , v12.hrmp_channel_max_message_size); + assert_eq!(v11.code_retention_period , v12.code_retention_period); + assert_eq!(v11.max_validators , v12.max_validators); + assert_eq!(v11.dispute_period , v12.dispute_period); + assert_eq!(v11.no_show_slots , v12.no_show_slots); + assert_eq!(v11.n_delay_tranches , v12.n_delay_tranches); + assert_eq!(v11.zeroth_delay_tranche_width , v12.zeroth_delay_tranche_width); + assert_eq!(v11.needed_approvals , v12.needed_approvals); + assert_eq!(v11.relay_vrf_modulo_samples , v12.relay_vrf_modulo_samples); + assert_eq!(v11.pvf_voting_ttl , v12.pvf_voting_ttl); + assert_eq!(v11.minimum_validation_upgrade_delay , v12.minimum_validation_upgrade_delay); + assert_eq!(v11.async_backing_params.allowed_ancestry_len, v12.async_backing_params.allowed_ancestry_len); + assert_eq!(v11.async_backing_params.max_candidate_depth , v12.async_backing_params.max_candidate_depth); + assert_eq!(v11.executor_params , v12.executor_params); + assert_eq!(v11.minimum_backing_votes , v12.minimum_backing_votes); + assert_eq!(v11.group_rotation_frequency , v12.scheduler_params.group_rotation_frequency); + assert_eq!(v11.paras_availability_period , v12.scheduler_params.paras_availability_period); + assert_eq!(v11.max_validators_per_core , v12.scheduler_params.max_validators_per_core); + assert_eq!(v11.scheduling_lookahead , v12.scheduler_params.lookahead); + assert_eq!(v11.coretime_cores , v12.scheduler_params.num_cores); + assert_eq!(v11.on_demand_retries , v12.scheduler_params.max_availability_timeouts); + assert_eq!(v11.on_demand_queue_max_size , v12.scheduler_params.on_demand_queue_max_size); + assert_eq!(v11.on_demand_target_queue_utilization , v12.scheduler_params.on_demand_target_queue_utilization); + assert_eq!(v11.on_demand_fee_variability , v12.scheduler_params.on_demand_fee_variability); + assert_eq!(v11.on_demand_base_fee , v12.scheduler_params.on_demand_base_fee); + assert_eq!(v11.on_demand_ttl , v12.scheduler_params.ttl); + }; // ; makes this a statement. `rustfmt::skip` cannot be put on an expression. + } + }); + } + + // Test that migration doesn't panic in case there are no pending configurations upgrades in + // pallet's storage. + #[test] + fn test_migrate_to_v12_no_pending() { + let v11 = V11HostConfiguration::::default(); + + new_test_ext(Default::default()).execute_with(|| { + // Implant the v10 version in the state. + v11::ActiveConfig::::set(Some(v11)); + // Ensure there are no pending configs. + v12::PendingConfigs::::set(None); + + // Shouldn't fail. + migrate_to_v12::(); + }); + } +} diff --git a/polkadot/runtime/parachains/src/configuration/tests.rs b/polkadot/runtime/parachains/src/configuration/tests.rs index f1570017dd7b..254511231cac 100644 --- a/polkadot/runtime/parachains/src/configuration/tests.rs +++ b/polkadot/runtime/parachains/src/configuration/tests.rs @@ -226,8 +226,11 @@ fn invariants() { ); ActiveConfig::::put(HostConfiguration { - paras_availability_period: 10, minimum_validation_upgrade_delay: 11, + scheduler_params: SchedulerParams { + paras_availability_period: 10, + ..Default::default() + }, ..Default::default() }); assert_err!( @@ -283,12 +286,6 @@ fn setting_pending_config_members() { max_code_size: 100_000, max_pov_size: 1024, max_head_data_size: 1_000, - coretime_cores: 2, - on_demand_retries: 5, - group_rotation_frequency: 20, - paras_availability_period: 10, - scheduling_lookahead: 3, - max_validators_per_core: None, max_validators: None, dispute_period: 239, dispute_post_conclusion_acceptance_period: 10, @@ -314,13 +311,21 @@ fn setting_pending_config_members() { minimum_validation_upgrade_delay: 20, executor_params: Default::default(), approval_voting_params: ApprovalVotingParams { max_approval_coalesce_count: 1 }, - on_demand_queue_max_size: 10_000u32, - on_demand_base_fee: 10_000_000u128, - on_demand_fee_variability: Perbill::from_percent(3), - on_demand_target_queue_utilization: Perbill::from_percent(25), - on_demand_ttl: 5u32, minimum_backing_votes: 5, node_features: bitvec![u8, Lsb0; 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1], + scheduler_params: SchedulerParams { + group_rotation_frequency: 20, + paras_availability_period: 10, + max_validators_per_core: None, + lookahead: 3, + num_cores: 2, + max_availability_timeouts: 5, + on_demand_queue_max_size: 10_000u32, + on_demand_base_fee: 10_000_000u128, + on_demand_fee_variability: Perbill::from_percent(3), + on_demand_target_queue_utilization: Perbill::from_percent(25), + ttl: 5u32, + }, }; Configuration::set_validation_upgrade_cooldown( @@ -342,13 +347,19 @@ fn setting_pending_config_members() { Configuration::set_max_pov_size(RuntimeOrigin::root(), new_config.max_pov_size).unwrap(); Configuration::set_max_head_data_size(RuntimeOrigin::root(), new_config.max_head_data_size) .unwrap(); - Configuration::set_coretime_cores(RuntimeOrigin::root(), new_config.coretime_cores) - .unwrap(); - Configuration::set_on_demand_retries(RuntimeOrigin::root(), new_config.on_demand_retries) - .unwrap(); + Configuration::set_coretime_cores( + RuntimeOrigin::root(), + new_config.scheduler_params.num_cores, + ) + .unwrap(); + Configuration::set_max_availability_timeouts( + RuntimeOrigin::root(), + new_config.scheduler_params.max_availability_timeouts, + ) + .unwrap(); Configuration::set_group_rotation_frequency( RuntimeOrigin::root(), - new_config.group_rotation_frequency, + new_config.scheduler_params.group_rotation_frequency, ) .unwrap(); // This comes out of order to satisfy the validity criteria for the chain and thread @@ -360,17 +371,17 @@ fn setting_pending_config_members() { .unwrap(); Configuration::set_paras_availability_period( RuntimeOrigin::root(), - new_config.paras_availability_period, + new_config.scheduler_params.paras_availability_period, ) .unwrap(); Configuration::set_scheduling_lookahead( RuntimeOrigin::root(), - new_config.scheduling_lookahead, + new_config.scheduler_params.lookahead, ) .unwrap(); Configuration::set_max_validators_per_core( RuntimeOrigin::root(), - new_config.max_validators_per_core, + new_config.scheduler_params.max_validators_per_core, ) .unwrap(); Configuration::set_max_validators(RuntimeOrigin::root(), new_config.max_validators) diff --git a/polkadot/runtime/parachains/src/coretime/migration.rs b/polkadot/runtime/parachains/src/coretime/migration.rs index 9bc0a20ef5b4..03fecc58570f 100644 --- a/polkadot/runtime/parachains/src/coretime/migration.rs +++ b/polkadot/runtime/parachains/src/coretime/migration.rs @@ -114,7 +114,7 @@ mod v_coretime { let legacy_paras = paras::Parachains::::get(); let config = >::config(); - let total_core_count = config.coretime_cores + legacy_paras.len() as u32; + let total_core_count = config.scheduler_params.num_cores + legacy_paras.len() as u32; let dmp_queue_size = crate::dmp::Pallet::::dmq_contents(T::BrokerId::get().into()).len() as u32; @@ -150,7 +150,7 @@ mod v_coretime { // Migrate to Coretime. // - // NOTE: Also migrates coretime_cores config value in configuration::ActiveConfig. + // NOTE: Also migrates `num_cores` config value in configuration::ActiveConfig. fn migrate_to_coretime< T: Config, SendXcm: xcm::v4::SendXcm, @@ -176,8 +176,8 @@ mod v_coretime { } let config = >::config(); - // coretime_cores was on_demand_cores until now: - for on_demand in 0..config.coretime_cores { + // num_cores was on_demand_cores until now: + for on_demand in 0..config.scheduler_params.num_cores { let core = CoreIndex(legacy_count.saturating_add(on_demand as _)); let r = assigner_coretime::Pallet::::assign_core( core, @@ -189,9 +189,9 @@ mod v_coretime { log::error!("Creating assignment for existing on-demand core, failed: {:?}", err); } } - let total_cores = config.coretime_cores + legacy_count; + let total_cores = config.scheduler_params.num_cores + legacy_count; configuration::ActiveConfig::::mutate(|c| { - c.coretime_cores = total_cores; + c.scheduler_params.num_cores = total_cores; }); if let Err(err) = migrate_send_assignments_to_coretime_chain::() { @@ -200,7 +200,9 @@ mod v_coretime { let single_weight = ::WeightInfo::assign_core(1); single_weight - .saturating_mul(u64::from(legacy_count.saturating_add(config.coretime_cores))) + .saturating_mul(u64::from( + legacy_count.saturating_add(config.scheduler_params.num_cores), + )) // Second read from sending assignments to the coretime chain. .saturating_add(T::DbWeight::get().reads_writes(2, 1)) } @@ -244,7 +246,8 @@ mod v_coretime { Some(mk_coretime_call(crate::coretime::CoretimeCalls::SetLease(p.into(), time_slice))) }); - let core_count: u16 = configuration::Pallet::::config().coretime_cores.saturated_into(); + let core_count: u16 = + configuration::Pallet::::config().scheduler_params.num_cores.saturated_into(); let set_core_count = iter::once(mk_coretime_call( crate::coretime::CoretimeCalls::NotifyCoreCount(core_count), )); diff --git a/polkadot/runtime/parachains/src/coretime/mod.rs b/polkadot/runtime/parachains/src/coretime/mod.rs index 531f5c2e4e47..eb9646d7e869 100644 --- a/polkadot/runtime/parachains/src/coretime/mod.rs +++ b/polkadot/runtime/parachains/src/coretime/mod.rs @@ -214,8 +214,8 @@ impl Pallet { } pub fn initializer_on_new_session(notification: &SessionChangeNotification>) { - let old_core_count = notification.prev_config.coretime_cores; - let new_core_count = notification.new_config.coretime_cores; + let old_core_count = notification.prev_config.scheduler_params.num_cores; + let new_core_count = notification.new_config.scheduler_params.num_cores; if new_core_count != old_core_count { let core_count: u16 = new_core_count.saturated_into(); let message = Xcm(vec![ diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index d2b5a67c3e45..3fe7d7f0c7d4 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -47,10 +47,10 @@ use test_helpers::{dummy_collator, dummy_collator_signature, dummy_validation_co fn default_config() -> HostConfiguration { let mut config = HostConfiguration::default(); - config.coretime_cores = 1; + config.scheduler_params.num_cores = 1; config.max_code_size = 0b100000; config.max_head_data_size = 0b100000; - config.group_rotation_frequency = u32::MAX; + config.scheduler_params.group_rotation_frequency = u32::MAX; config } @@ -224,7 +224,7 @@ pub(crate) fn run_to_block( } pub(crate) fn expected_bits() -> usize { - Paras::parachains().len() + Configuration::config().coretime_cores as usize + Paras::parachains().len() + Configuration::config().scheduler_params.num_cores as usize } fn default_bitfield() -> AvailabilityBitfield { @@ -386,7 +386,7 @@ fn collect_pending_cleans_up_pending() { (thread_a, ParaKind::Parathread), ]; let mut config = genesis_config(paras); - config.configuration.config.group_rotation_frequency = 3; + config.configuration.config.scheduler_params.group_rotation_frequency = 3; new_test_ext(config).execute_with(|| { let default_candidate = TestCandidateBuilder::default().build(); >::insert( @@ -2062,7 +2062,7 @@ fn check_allowed_relay_parents() { } let validator_public = validator_pubkeys(&validators); let mut config = genesis_config(paras); - config.configuration.config.group_rotation_frequency = 1; + config.configuration.config.scheduler_params.group_rotation_frequency = 1; new_test_ext(config).execute_with(|| { shared::Pallet::::set_active_validators_ascending(validator_public.clone()); diff --git a/polkadot/runtime/parachains/src/mock.rs b/polkadot/runtime/parachains/src/mock.rs index 1925ca19501a..e18be03bdeb2 100644 --- a/polkadot/runtime/parachains/src/mock.rs +++ b/polkadot/runtime/parachains/src/mock.rs @@ -23,7 +23,7 @@ use crate::{ initializer, origin, paras, paras::ParaKind, paras_inherent, scheduler, - scheduler::common::{AssignmentProvider, AssignmentProviderConfig}, + scheduler::common::AssignmentProvider, session_info, shared, ParaId, }; use frame_support::pallet_prelude::*; @@ -463,10 +463,6 @@ pub mod mock_assigner { pub(super) type MockAssignmentQueue = StorageValue<_, VecDeque, ValueQuery>; - #[pallet::storage] - pub(super) type MockProviderConfig = - StorageValue<_, AssignmentProviderConfig, OptionQuery>; - #[pallet::storage] pub(super) type MockCoreCount = StorageValue<_, u32, OptionQuery>; } @@ -478,12 +474,6 @@ pub mod mock_assigner { MockAssignmentQueue::::mutate(|queue| queue.push_back(assignment)); } - // This configuration needs to be customized to service `get_provider_config` in - // scheduler tests. - pub fn set_assignment_provider_config(config: AssignmentProviderConfig) { - MockProviderConfig::::set(Some(config)); - } - // Allows for customized core count in scheduler tests, rather than a core count // derived from on-demand config + parachain count. pub fn set_core_count(count: u32) { @@ -512,17 +502,6 @@ pub mod mock_assigner { // in the mock assigner. fn push_back_assignment(_assignment: Assignment) {} - // Gets the provider config we set earlier using `set_assignment_provider_config`, falling - // back to the on demand parachain configuration if none was set. - fn get_provider_config(_core_idx: CoreIndex) -> AssignmentProviderConfig { - match MockProviderConfig::::get() { - Some(config) => config, - None => AssignmentProviderConfig { - max_availability_timeouts: 1, - ttl: BlockNumber::from(5u32), - }, - } - } #[cfg(any(feature = "runtime-benchmarks", test))] fn get_mock_assignment(_: CoreIndex, para_id: ParaId) -> Assignment { Assignment::Bulk(para_id) diff --git a/polkadot/runtime/parachains/src/paras/tests.rs b/polkadot/runtime/parachains/src/paras/tests.rs index 24ea919ec875..262ec9d3fdba 100644 --- a/polkadot/runtime/parachains/src/paras/tests.rs +++ b/polkadot/runtime/parachains/src/paras/tests.rs @@ -17,7 +17,7 @@ use super::*; use frame_support::{assert_err, assert_ok, assert_storage_noop}; use keyring::Sr25519Keyring; -use primitives::{BlockNumber, PARACHAIN_KEY_TYPE_ID}; +use primitives::{vstaging::SchedulerParams, BlockNumber, PARACHAIN_KEY_TYPE_ID}; use sc_keystore::LocalKeystore; use sp_keystore::{Keystore, KeystorePtr}; use std::sync::Arc; @@ -909,7 +909,10 @@ fn full_parachain_cleanup_storage() { minimum_validation_upgrade_delay: 2, // Those are not relevant to this test. However, HostConfiguration is still a // subject for the consistency check. - paras_availability_period: 1, + scheduler_params: SchedulerParams { + paras_availability_period: 1, + ..Default::default() + }, ..Default::default() }, }, diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index defb2f4404f5..b7285ec884ad 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -27,13 +27,14 @@ mod enter { builder::{Bench, BenchBuilder}, mock::{mock_assigner, new_test_ext, BlockLength, BlockWeights, MockGenesisConfig, Test}, scheduler::{ - common::{Assignment, AssignmentProvider, AssignmentProviderConfig}, + common::{Assignment, AssignmentProvider}, ParasEntry, }, }; use assert_matches::assert_matches; use frame_support::assert_ok; use frame_system::limits; + use primitives::vstaging::SchedulerParams; use sp_runtime::Perbill; use sp_std::collections::btree_map::BTreeMap; @@ -87,7 +88,7 @@ mod enter { // `create_inherent` and will not cause `enter` to early. fn include_backed_candidates() { let config = MockGenesisConfig::default(); - assert!(config.configuration.config.scheduling_lookahead > 0); + assert!(config.configuration.config.scheduler_params.lookahead > 0); new_test_ext(config).execute_with(|| { let dispute_statements = BTreeMap::new(); @@ -625,7 +626,7 @@ mod enter { #[test] fn limit_candidates_over_weight_1() { let config = MockGenesisConfig::default(); - assert!(config.configuration.config.scheduling_lookahead > 0); + assert!(config.configuration.config.scheduler_params.lookahead > 0); new_test_ext(config).execute_with(|| { // Create the inherent data for this block @@ -706,8 +707,8 @@ mod enter { let cores = (0..used_cores) .into_iter() .map(|i| { - let AssignmentProviderConfig { ttl, .. } = - scheduler::Pallet::::assignment_provider_config(CoreIndex(i)); + let SchedulerParams { ttl, .. } = + >::config().scheduler_params; // Load an assignment into provider so that one is present to pop let assignment = ::AssignmentProvider::get_mock_assignment( diff --git a/polkadot/runtime/parachains/src/scheduler.rs b/polkadot/runtime/parachains/src/scheduler.rs index 3dbb050fb4e5..61ab36dbe96c 100644 --- a/polkadot/runtime/parachains/src/scheduler.rs +++ b/polkadot/runtime/parachains/src/scheduler.rs @@ -51,7 +51,7 @@ use sp_std::{ pub mod common; -use common::{Assignment, AssignmentProvider, AssignmentProviderConfig}; +use common::{Assignment, AssignmentProvider}; pub use pallet::*; @@ -222,7 +222,7 @@ impl Pallet { let n_cores = core::cmp::max( T::AssignmentProvider::session_core_count(), - match config.max_validators_per_core { + match config.scheduler_params.max_validators_per_core { Some(x) if x != 0 => validators.len() as u32 / x, _ => 0, }, @@ -350,6 +350,7 @@ impl Pallet { fn drop_expired_claims_from_claimqueue() { let now = >::block_number(); let availability_cores = AvailabilityCores::::get(); + let ttl = >::config().scheduler_params.ttl; ClaimQueue::::mutate(|cq| { for (idx, _) in (0u32..).zip(availability_cores) { @@ -382,8 +383,6 @@ impl Pallet { if let Some(assignment) = T::AssignmentProvider::pop_assignment_for_core(core_idx) { - let AssignmentProviderConfig { ttl, .. } = - T::AssignmentProvider::get_provider_config(core_idx); core_claimqueue.push_back(ParasEntry::new(assignment, now + ttl)); } } @@ -428,7 +427,7 @@ impl Pallet { } let rotations_since_session_start: BlockNumberFor = - (at - session_start_block) / config.group_rotation_frequency; + (at - session_start_block) / config.scheduler_params.group_rotation_frequency; let rotations_since_session_start = as TryInto>::try_into(rotations_since_session_start) @@ -460,9 +459,9 @@ impl Pallet { // Note: blocks backed in this rotation will never time out here as backed_in + // config.paras_availability_period will always be > now for these blocks, as // otherwise above condition would not be true. - pending_since + config.paras_availability_period + pending_since + config.scheduler_params.paras_availability_period } else { - next_rotation + config.paras_availability_period + next_rotation + config.scheduler_params.paras_availability_period }; AvailabilityTimeoutStatus { timed_out: time_out_at <= now, live_until: time_out_at } @@ -478,7 +477,8 @@ impl Pallet { let now = >::block_number() + One::one(); let rotation_info = Self::group_rotation_info(now); - let current_window = rotation_info.last_rotation_at() + config.paras_availability_period; + let current_window = + rotation_info.last_rotation_at() + config.scheduler_params.paras_availability_period; now < current_window } @@ -488,7 +488,7 @@ impl Pallet { ) -> GroupRotationInfo> { let session_start_block = Self::session_start_block(); let group_rotation_frequency = - >::config().group_rotation_frequency; + >::config().scheduler_params.group_rotation_frequency; GroupRotationInfo { session_start_block, now, group_rotation_frequency } } @@ -508,6 +508,8 @@ impl Pallet { /// Return the next thing that will be scheduled on this core assuming it is currently /// occupied and the candidate occupying it times out. pub(crate) fn next_up_on_time_out(core: CoreIndex) -> Option { + let max_availability_timeouts = + >::config().scheduler_params.max_availability_timeouts; Self::next_up_on_available(core).or_else(|| { // Or, if none, the claim currently occupying the core, // as it would be put back on the queue after timing out if number of retries is not at @@ -515,16 +517,12 @@ impl Pallet { let cores = AvailabilityCores::::get(); cores.get(core.0 as usize).and_then(|c| match c { CoreOccupied::Free => None, - CoreOccupied::Paras(pe) => { - let AssignmentProviderConfig { max_availability_timeouts, .. } = - T::AssignmentProvider::get_provider_config(core); - + CoreOccupied::Paras(pe) => if pe.availability_timeouts < max_availability_timeouts { Some(Self::paras_entry_to_scheduled_core(pe)) } else { None - } - }, + }, }) }) } @@ -566,7 +564,7 @@ impl Pallet { // ClaimQueue related functions // fn claimqueue_lookahead() -> u32 { - >::config().scheduling_lookahead + >::config().scheduler_params.lookahead } /// Frees cores and fills the free claimqueue spots by popping from the `AssignmentProvider`. @@ -585,15 +583,15 @@ impl Pallet { let n_lookahead = Self::claimqueue_lookahead().max(1); let n_session_cores = T::AssignmentProvider::session_core_count(); let cq = ClaimQueue::::get(); - let ttl = >::config().on_demand_ttl; + let config = >::config(); + let max_availability_timeouts = config.scheduler_params.max_availability_timeouts; + let ttl = config.scheduler_params.ttl; for core_idx in 0..n_session_cores { let core_idx = CoreIndex::from(core_idx); // add previously timedout paras back into the queue if let Some(mut entry) = timedout_paras.remove(&core_idx) { - let AssignmentProviderConfig { max_availability_timeouts, .. } = - T::AssignmentProvider::get_provider_config(core_idx); if entry.availability_timeouts < max_availability_timeouts { // Increment the timeout counter. entry.availability_timeouts += 1; @@ -668,13 +666,6 @@ impl Pallet { .filter_map(|(core_idx, v)| v.front().map(|e| (core_idx, e.assignment.para_id()))) } - #[cfg(any(feature = "runtime-benchmarks", test))] - pub(crate) fn assignment_provider_config( - core_idx: CoreIndex, - ) -> AssignmentProviderConfig> { - T::AssignmentProvider::get_provider_config(core_idx) - } - #[cfg(any(feature = "try-runtime", test))] fn claimqueue_len() -> usize { ClaimQueue::::get().iter().map(|la_vec| la_vec.1.len()).sum() diff --git a/polkadot/runtime/parachains/src/scheduler/common.rs b/polkadot/runtime/parachains/src/scheduler/common.rs index 2eb73385803c..66a4e6d30be0 100644 --- a/polkadot/runtime/parachains/src/scheduler/common.rs +++ b/polkadot/runtime/parachains/src/scheduler/common.rs @@ -48,22 +48,10 @@ impl Assignment { } } -#[derive(Encode, Decode, TypeInfo)] -/// A set of variables required by the scheduler in order to operate. -pub struct AssignmentProviderConfig { - /// How many times a collation can time out on availability. - /// Zero timeouts still means that a collation can be provided as per the slot auction - /// assignment provider. - pub max_availability_timeouts: u32, - - /// How long the collator has to provide a collation to the backing group before being dropped. - pub ttl: BlockNumber, -} - pub trait AssignmentProvider { /// Pops an [`Assignment`] from the provider for a specified [`CoreIndex`]. /// - /// This is where assignments come into existance. + /// This is where assignments come into existence. fn pop_assignment_for_core(core_idx: CoreIndex) -> Option; /// A previously popped `Assignment` has been fully processed. @@ -77,14 +65,11 @@ pub trait AssignmentProvider { /// Push back a previously popped assignment. /// /// If the assignment could not be processed within the current session, it can be pushed back - /// to the assignment provider in order to be poppped again later. + /// to the assignment provider in order to be popped again later. /// /// This is the second way the life of an assignment can come to an end. fn push_back_assignment(assignment: Assignment); - /// Returns a set of variables needed by the scheduler - fn get_provider_config(core_idx: CoreIndex) -> AssignmentProviderConfig; - /// Push some assignment for mocking/benchmarks purposes. /// /// Useful for benchmarks and testing. The returned assignment is "valid" and can if need be diff --git a/polkadot/runtime/parachains/src/scheduler/tests.rs b/polkadot/runtime/parachains/src/scheduler/tests.rs index 9af23ce64bd6..28b3a6b4f5d6 100644 --- a/polkadot/runtime/parachains/src/scheduler/tests.rs +++ b/polkadot/runtime/parachains/src/scheduler/tests.rs @@ -18,7 +18,9 @@ use super::*; use frame_support::assert_ok; use keyring::Sr25519Keyring; -use primitives::{BlockNumber, SessionIndex, ValidationCode, ValidatorId}; +use primitives::{ + vstaging::SchedulerParams, BlockNumber, SessionIndex, ValidationCode, ValidatorId, +}; use sp_std::collections::{btree_map::BTreeMap, btree_set::BTreeSet}; use crate::{ @@ -103,15 +105,19 @@ fn run_to_end_of_block( fn default_config() -> HostConfiguration { HostConfiguration { - coretime_cores: 3, - group_rotation_frequency: 10, - paras_availability_period: 3, - scheduling_lookahead: 2, // This field does not affect anything that scheduler does. However, `HostConfiguration` // is still a subject to consistency test. It requires that // `minimum_validation_upgrade_delay` is greater than `chain_availability_period` and // `thread_availability_period`. minimum_validation_upgrade_delay: 6, + scheduler_params: SchedulerParams { + group_rotation_frequency: 10, + paras_availability_period: 3, + lookahead: 2, + num_cores: 3, + max_availability_timeouts: 1, + ..Default::default() + }, ..Default::default() } } @@ -155,7 +161,7 @@ fn scheduled_entries() -> impl Iterator(vec![para_a])); } else { assert!(!claimqueue_contains_para_ids::(vec![para_a])); @@ -822,7 +825,7 @@ fn on_demand_claims_are_pruned_after_timing_out() { assert!(!availability_cores_contains_para_ids::(vec![para_a])); // #25 - now += max_retries + 2; + now += max_timeouts + 2; // Add assignment back to the mix. MockAssigner::add_test_assignment(assignment_a.clone()); @@ -833,7 +836,7 @@ fn on_demand_claims_are_pruned_after_timing_out() { // #26 now += 1; // Run to block #n but this time have group 1 conclude the availabilty. - for n in now..=(now + max_retries + 1) { + for n in now..=(now + max_timeouts + 1) { // #n run_to_block(n, |_| None); // Time out core 0 if group 0 is assigned to it, if group 1 is assigned, conclude. @@ -874,10 +877,8 @@ fn on_demand_claims_are_pruned_after_timing_out() { fn availability_predicate_works() { let genesis_config = genesis_config(&default_config()); - let HostConfiguration { group_rotation_frequency, paras_availability_period, .. } = - default_config(); - - assert!(paras_availability_period < group_rotation_frequency); + let SchedulerParams { group_rotation_frequency, paras_availability_period, .. } = + default_config().scheduler_params; new_test_ext(genesis_config).execute_with(|| { run_to_block(1 + paras_availability_period, |_| None); @@ -1044,7 +1045,7 @@ fn next_up_on_time_out_reuses_claim_if_nothing_queued() { #[test] fn session_change_requires_reschedule_dropping_removed_paras() { let mut config = default_config(); - config.scheduling_lookahead = 1; + config.scheduler_params.lookahead = 1; let genesis_config = genesis_config(&config); let para_a = ParaId::from(1_u32); @@ -1056,7 +1057,7 @@ fn session_change_requires_reschedule_dropping_removed_paras() { new_test_ext(genesis_config).execute_with(|| { // Setting explicit core count MockAssigner::set_core_count(5); - let assignment_provider_ttl = MockAssigner::get_provider_config(CoreIndex::from(0)).ttl; + let coretime_ttl = >::config().scheduler_params.ttl; schedule_blank_para(para_a); schedule_blank_para(para_b); @@ -1120,7 +1121,7 @@ fn session_change_requires_reschedule_dropping_removed_paras() { vec![ParasEntry::new( Assignment::Bulk(para_a), // At end of block 2 - assignment_provider_ttl + 2 + coretime_ttl + 2 )] .into_iter() .collect() @@ -1169,7 +1170,7 @@ fn session_change_requires_reschedule_dropping_removed_paras() { vec![ParasEntry::new( Assignment::Bulk(para_a), // At block 3 - assignment_provider_ttl + 3 + coretime_ttl + 3 )] .into_iter() .collect() @@ -1179,7 +1180,7 @@ fn session_change_requires_reschedule_dropping_removed_paras() { vec![ParasEntry::new( Assignment::Bulk(para_b), // At block 3 - assignment_provider_ttl + 3 + coretime_ttl + 3 )] .into_iter() .collect() diff --git a/polkadot/runtime/parachains/src/session_info/tests.rs b/polkadot/runtime/parachains/src/session_info/tests.rs index 92a50575deda..a5bfeae07455 100644 --- a/polkadot/runtime/parachains/src/session_info/tests.rs +++ b/polkadot/runtime/parachains/src/session_info/tests.rs @@ -25,7 +25,7 @@ use crate::{ util::take_active_subset, }; use keyring::Sr25519Keyring; -use primitives::{BlockNumber, ValidatorId, ValidatorIndex}; +use primitives::{vstaging::SchedulerParams, BlockNumber, ValidatorId, ValidatorIndex}; fn run_to_block( to: BlockNumber, @@ -62,9 +62,9 @@ fn run_to_block( fn default_config() -> HostConfiguration { HostConfiguration { - coretime_cores: 1, dispute_period: 2, needed_approvals: 3, + scheduler_params: SchedulerParams { num_cores: 1, ..Default::default() }, ..Default::default() } } diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 173301e1ad91..f3c0c3d6bd8c 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1658,6 +1658,7 @@ pub mod migrations { parachains_configuration::migration::v11::MigrateToV11, // This needs to come after the `parachains_configuration` above as we are reading the configuration. coretime::migration::MigrateToCoretime, + parachains_configuration::migration::v12::MigrateToV12, // permanent pallet_xcm::migration::MigrateToLatestXcmVersion, diff --git a/polkadot/runtime/rococo/src/weights/runtime_parachains_configuration.rs b/polkadot/runtime/rococo/src/weights/runtime_parachains_configuration.rs index 34541b83597e..ca0575cb1b64 100644 --- a/polkadot/runtime/rococo/src/weights/runtime_parachains_configuration.rs +++ b/polkadot/runtime/rococo/src/weights/runtime_parachains_configuration.rs @@ -16,10 +16,10 @@ //! Autogenerated weights for `runtime_parachains::configuration` //! -//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-11-10, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 32.0.0 +//! DATE: 2024-02-08, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `runner-yprdrvc7-project-674-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` +//! HOSTNAME: `runner-bn-ce5rx-project-674-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` //! WASM-EXECUTION: `Compiled`, CHAIN: `Some("rococo-dev")`, DB CACHE: 1024 // Executed Command: @@ -58,8 +58,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_793_000 picoseconds. - Weight::from_parts(8_192_000, 0) + // Minimum execution time: 7_789_000 picoseconds. + Weight::from_parts(8_269_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -74,8 +74,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_819_000 picoseconds. - Weight::from_parts(8_004_000, 0) + // Minimum execution time: 7_851_000 picoseconds. + Weight::from_parts(8_152_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -90,8 +90,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_760_000 picoseconds. - Weight::from_parts(8_174_000, 0) + // Minimum execution time: 7_960_000 picoseconds. + Weight::from_parts(8_276_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -116,8 +116,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_814_000 picoseconds. - Weight::from_parts(8_098_000, 0) + // Minimum execution time: 7_912_000 picoseconds. + Weight::from_parts(8_164_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -132,8 +132,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 10_028_000 picoseconds. - Weight::from_parts(10_386_000, 0) + // Minimum execution time: 9_782_000 picoseconds. + Weight::from_parts(10_373_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -148,8 +148,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_867_000 picoseconds. - Weight::from_parts(8_191_000, 0) + // Minimum execution time: 7_870_000 picoseconds. + Weight::from_parts(8_274_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -164,8 +164,24 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 10_158_000 picoseconds. - Weight::from_parts(10_430_000, 0) + // Minimum execution time: 9_960_000 picoseconds. + Weight::from_parts(10_514_000, 0) + .saturating_add(Weight::from_parts(0, 1636)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `Configuration::PendingConfigs` (r:1 w:1) + /// Proof: `Configuration::PendingConfigs` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `Configuration::BypassConsistencyCheck` (r:1 w:0) + /// Proof: `Configuration::BypassConsistencyCheck` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `ParasShared::CurrentSessionIndex` (r:1 w:0) + /// Proof: `ParasShared::CurrentSessionIndex` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + fn set_config_with_scheduler_params() -> Weight { + // Proof Size summary in bytes: + // Measured: `151` + // Estimated: `1636` + // Minimum execution time: 7_913_000 picoseconds. + Weight::from_parts(8_338_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 91047d953f54..a7772303a95d 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -1699,6 +1699,7 @@ pub mod migrations { // Migrate Identity pallet for Usernames pallet_identity::migration::versioned::V0ToV1, parachains_configuration::migration::v11::MigrateToV11, + parachains_configuration::migration::v12::MigrateToV12, // permanent pallet_xcm::migration::MigrateToLatestXcmVersion, // Migrate from legacy lease to coretime. Needs to run after configuration v11 diff --git a/polkadot/runtime/westend/src/weights/runtime_parachains_configuration.rs b/polkadot/runtime/westend/src/weights/runtime_parachains_configuration.rs index 3a4813b667c6..8fa3207c6446 100644 --- a/polkadot/runtime/westend/src/weights/runtime_parachains_configuration.rs +++ b/polkadot/runtime/westend/src/weights/runtime_parachains_configuration.rs @@ -16,10 +16,10 @@ //! Autogenerated weights for `runtime_parachains::configuration` //! -//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-11-10, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 32.0.0 +//! DATE: 2024-02-08, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `runner-yprdrvc7-project-674-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` +//! HOSTNAME: `runner-bn-ce5rx-project-674-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` //! WASM-EXECUTION: `Compiled`, CHAIN: `Some("westend-dev")`, DB CACHE: 1024 // Executed Command: @@ -58,8 +58,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 8_065_000 picoseconds. - Weight::from_parts(8_389_000, 0) + // Minimum execution time: 7_775_000 picoseconds. + Weight::from_parts(8_036_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -74,8 +74,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 8_038_000 picoseconds. - Weight::from_parts(8_463_000, 0) + // Minimum execution time: 7_708_000 picoseconds. + Weight::from_parts(7_971_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -90,8 +90,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_843_000 picoseconds. - Weight::from_parts(8_216_000, 0) + // Minimum execution time: 7_746_000 picoseconds. + Weight::from_parts(8_028_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -116,8 +116,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_969_000 picoseconds. - Weight::from_parts(8_362_000, 0) + // Minimum execution time: 7_729_000 picoseconds. + Weight::from_parts(7_954_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -132,8 +132,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 10_084_000 picoseconds. - Weight::from_parts(10_451_000, 0) + // Minimum execution time: 9_871_000 picoseconds. + Weight::from_parts(10_075_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -148,8 +148,8 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 7_948_000 picoseconds. - Weight::from_parts(8_268_000, 0) + // Minimum execution time: 7_869_000 picoseconds. + Weight::from_parts(8_000_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -164,8 +164,24 @@ impl runtime_parachains::configuration::WeightInfo for // Proof Size summary in bytes: // Measured: `151` // Estimated: `1636` - // Minimum execution time: 10_257_000 picoseconds. - Weight::from_parts(10_584_000, 0) + // Minimum execution time: 9_797_000 picoseconds. + Weight::from_parts(10_373_000, 0) + .saturating_add(Weight::from_parts(0, 1636)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `Configuration::PendingConfigs` (r:1 w:1) + /// Proof: `Configuration::PendingConfigs` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `Configuration::BypassConsistencyCheck` (r:1 w:0) + /// Proof: `Configuration::BypassConsistencyCheck` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `ParasShared::CurrentSessionIndex` (r:1 w:0) + /// Proof: `ParasShared::CurrentSessionIndex` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + fn set_config_with_scheduler_params() -> Weight { + // Proof Size summary in bytes: + // Measured: `151` + // Estimated: `1636` + // Minimum execution time: 7_718_000 picoseconds. + Weight::from_parts(7_984_000, 0) .saturating_add(Weight::from_parts(0, 1636)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) diff --git a/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml b/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml index f6bdfeb4877e..2561661de1f8 100644 --- a/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml +++ b/polkadot/zombienet_tests/functional/0002-parachains-disputes.toml @@ -2,9 +2,11 @@ timeout = 1000 [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 5 needed_approvals = 8 +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 5 + [relaychain.genesis.runtimeGenesis.patch.configuration.config.approval_voting_params] max_approval_coalesce_count = 5 diff --git a/polkadot/zombienet_tests/functional/0004-parachains-garbage-candidate.toml b/polkadot/zombienet_tests/functional/0004-parachains-garbage-candidate.toml index 5d6f299d4613..a2a2621f8426 100644 --- a/polkadot/zombienet_tests/functional/0004-parachains-garbage-candidate.toml +++ b/polkadot/zombienet_tests/functional/0004-parachains-garbage-candidate.toml @@ -2,8 +2,10 @@ timeout = 1000 bootnode = true -[relaychain.genesis.runtimeGenesis.patch.configuration.config] +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] max_validators_per_core = 1 + +[relaychain.genesis.runtimeGenesis.patch.configuration.config] needed_approvals = 2 [relaychain] diff --git a/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.toml b/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.toml index e2fbec079b1a..a3bbc82e74ba 100644 --- a/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.toml +++ b/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.toml @@ -3,8 +3,10 @@ timeout = 1000 bootnode = true [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 1 needed_approvals = 2 + +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 1 group_rotation_frequency = 2 [relaychain] diff --git a/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml b/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml index bef54cb8ca41..858f87b9cfe5 100644 --- a/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml +++ b/polkadot/zombienet_tests/functional/0006-parachains-max-tranche0.toml @@ -3,10 +3,12 @@ timeout = 1000 bootnode = true [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 1 needed_approvals = 7 relay_vrf_modulo_samples = 5 +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 1 + [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" diff --git a/polkadot/zombienet_tests/functional/0007-dispute-freshly-finalized.toml b/polkadot/zombienet_tests/functional/0007-dispute-freshly-finalized.toml index 69eb0804d8cb..573ccf961385 100644 --- a/polkadot/zombienet_tests/functional/0007-dispute-freshly-finalized.toml +++ b/polkadot/zombienet_tests/functional/0007-dispute-freshly-finalized.toml @@ -2,9 +2,11 @@ timeout = 1000 [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 1 needed_approvals = 1 +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 1 + [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" diff --git a/polkadot/zombienet_tests/functional/0008-dispute-old-finalized.toml b/polkadot/zombienet_tests/functional/0008-dispute-old-finalized.toml index 1ea385c3a42e..ea1c93a1403f 100644 --- a/polkadot/zombienet_tests/functional/0008-dispute-old-finalized.toml +++ b/polkadot/zombienet_tests/functional/0008-dispute-old-finalized.toml @@ -2,9 +2,11 @@ timeout = 1000 [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 1 needed_approvals = 1 +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 1 + [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" diff --git a/polkadot/zombienet_tests/functional/0010-validator-disabling.toml b/polkadot/zombienet_tests/functional/0010-validator-disabling.toml index 6701d60d74d1..c9d79c5f8f23 100644 --- a/polkadot/zombienet_tests/functional/0010-validator-disabling.toml +++ b/polkadot/zombienet_tests/functional/0010-validator-disabling.toml @@ -3,8 +3,10 @@ timeout = 1000 bootnode = true [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 1 needed_approvals = 2 + +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 1 group_rotation_frequency = 10 [relaychain] diff --git a/polkadot/zombienet_tests/functional/0011-async-backing-6-seconds-rate.toml b/polkadot/zombienet_tests/functional/0011-async-backing-6-seconds-rate.toml index 5a6832b149be..b776622fdce3 100644 --- a/polkadot/zombienet_tests/functional/0011-async-backing-6-seconds-rate.toml +++ b/polkadot/zombienet_tests/functional/0011-async-backing-6-seconds-rate.toml @@ -8,13 +8,16 @@ chain = "rococo-local" [relaychain.genesis.runtimeGenesis.patch.configuration.config] needed_approvals = 4 relay_vrf_modulo_samples = 6 - scheduling_lookahead = 2 - group_rotation_frequency = 4 [relaychain.genesis.runtimeGenesis.patch.configuration.config.async_backing_params] max_candidate_depth = 3 allowed_ancestry_len = 2 +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + lookahead = 2 + group_rotation_frequency = 4 + + [relaychain.default_resources] limits = { memory = "4G", cpu = "2" } requests = { memory = "2G", cpu = "1" } diff --git a/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.toml b/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.toml index 0dfd814e10a5..9b3576eaa3c2 100644 --- a/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.toml +++ b/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.toml @@ -3,9 +3,11 @@ timeout = 1000 bootnode = true [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 2 needed_approvals = 4 - coretime_cores = 2 + +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 2 + num_cores = 2 [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" diff --git a/polkadot/zombienet_tests/misc/0001-paritydb.toml b/polkadot/zombienet_tests/misc/0001-paritydb.toml index 399f848d3ac4..b3ce2081b111 100644 --- a/polkadot/zombienet_tests/misc/0001-paritydb.toml +++ b/polkadot/zombienet_tests/misc/0001-paritydb.toml @@ -3,9 +3,11 @@ timeout = 1000 bootnode = true [relaychain.genesis.runtimeGenesis.patch.configuration.config] - max_validators_per_core = 1 needed_approvals = 3 +[relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] + max_validators_per_core = 1 + [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" chain = "rococo-local" From 761937ecdc8f208361ae2a1154e0f1e6ecf57429 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe <49718502+alexggh@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:21:00 +0200 Subject: [PATCH 10/13] Fix accidental no-shows on node restart (#3277) If approval was in progress we didn't actually restart it, so we end up in a situation where we distribute our assignment, but we don't distribute any approval. --------- Signed-off-by: Alexandru Gheorghe --- polkadot/node/core/approval-voting/src/lib.rs | 85 +- .../approval-voting/src/persisted_entries.rs | 7 + .../node/core/approval-voting/src/tests.rs | 744 +++++++++++++++++- 3 files changed, 780 insertions(+), 56 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 456ae319787b..8cc16a6e1ec1 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -1253,13 +1253,20 @@ async fn handle_actions( Action::BecomeActive => { *mode = Mode::Active; - let messages = distribution_messages_for_activation( + let (messages, next_actions) = distribution_messages_for_activation( + ctx, overlayed_db, state, delayed_approvals_timers, - )?; + session_info_provider, + ) + .await?; ctx.send_messages(messages.into_iter()).await; + let next_actions: Vec = + next_actions.into_iter().map(|v| v.clone()).chain(actions_iter).collect(); + + actions_iter = next_actions.into_iter(); }, Action::Conclude => { conclude = true; @@ -1313,15 +1320,19 @@ fn get_assignment_core_indices( } } -fn distribution_messages_for_activation( +#[overseer::contextbounds(ApprovalVoting, prefix = self::overseer)] +async fn distribution_messages_for_activation( + ctx: &mut Context, db: &OverlayedBackend<'_, impl Backend>, state: &State, delayed_approvals_timers: &mut DelayedApprovalTimer, -) -> SubsystemResult> { + session_info_provider: &mut RuntimeInfo, +) -> SubsystemResult<(Vec, Vec)> { let all_blocks: Vec = db.load_all_blocks()?; let mut approval_meta = Vec::with_capacity(all_blocks.len()); let mut messages = Vec::new(); + let mut actions = Vec::new(); messages.push(ApprovalDistributionMessage::NewBlocks(Vec::new())); // dummy value. @@ -1396,16 +1407,60 @@ fn distribution_messages_for_activation( &claimed_core_indices, &block_entry, ) { - Ok(bitfield) => messages.push( - ApprovalDistributionMessage::DistributeAssignment( - IndirectAssignmentCertV2 { - block_hash, - validator: assignment.validator_index(), - cert: assignment.cert().clone(), - }, - bitfield, - ), - ), + Ok(bitfield) => { + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?candidate_entry.candidate_receipt().hash(), + ?block_hash, + "Discovered, triggered assignment, not approved yet", + ); + + let indirect_cert = IndirectAssignmentCertV2 { + block_hash, + validator: assignment.validator_index(), + cert: assignment.cert().clone(), + }; + messages.push( + ApprovalDistributionMessage::DistributeAssignment( + indirect_cert.clone(), + bitfield.clone(), + ), + ); + + if !block_entry + .candidate_is_pending_signature(*candidate_hash) + { + let ExtendedSessionInfo { ref executor_params, .. } = + match get_extended_session_info( + session_info_provider, + ctx.sender(), + block_entry.block_hash(), + block_entry.session(), + ) + .await + { + Some(i) => i, + None => continue, + }; + + actions.push(Action::LaunchApproval { + claimed_candidate_indices: bitfield, + candidate_hash: candidate_entry + .candidate_receipt() + .hash(), + indirect_cert, + assignment_tranche: assignment.tranche(), + relay_block_hash: block_hash, + session: block_entry.session(), + executor_params: executor_params.clone(), + candidate: candidate_entry + .candidate_receipt() + .clone(), + backing_group: approval_entry.backing_group(), + distribute_assignment: false, + }); + } + }, Err(err) => { // Should never happen. If we fail here it means the // assignment is null (no cores claimed). @@ -1496,7 +1551,7 @@ fn distribution_messages_for_activation( } messages[0] = ApprovalDistributionMessage::NewBlocks(approval_meta); - Ok(messages) + Ok((messages, actions)) } // Handle an incoming signal from the overseer. Returns true if execution should conclude. diff --git a/polkadot/node/core/approval-voting/src/persisted_entries.rs b/polkadot/node/core/approval-voting/src/persisted_entries.rs index ef47bdb2213a..b924a1b52ccf 100644 --- a/polkadot/node/core/approval-voting/src/persisted_entries.rs +++ b/polkadot/node/core/approval-voting/src/persisted_entries.rs @@ -588,6 +588,13 @@ impl BlockEntry { !self.candidates_pending_signature.is_empty() } + /// Returns true if candidate hash is in the queue for a signature. + pub fn candidate_is_pending_signature(&self, candidate_hash: CandidateHash) -> bool { + self.candidates_pending_signature + .values() + .any(|context| context.candidate_hash == candidate_hash) + } + /// Candidate hashes for candidates pending signatures fn candidate_hashes_pending_signature(&self) -> Vec { self.candidates_pending_signature diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 9220e84a2554..c9053232a4c8 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -78,6 +78,7 @@ struct TestSyncOracle { struct TestSyncOracleHandle { done_syncing_receiver: oneshot::Receiver<()>, + is_major_syncing: Arc, } impl TestSyncOracleHandle { @@ -108,8 +109,9 @@ impl SyncOracle for TestSyncOracle { fn make_sync_oracle(val: bool) -> (Box, TestSyncOracleHandle) { let (tx, rx) = oneshot::channel(); let flag = Arc::new(AtomicBool::new(val)); - let oracle = TestSyncOracle { flag, done_syncing_sender: Arc::new(Mutex::new(Some(tx))) }; - let handle = TestSyncOracleHandle { done_syncing_receiver: rx }; + let oracle = + TestSyncOracle { flag: flag.clone(), done_syncing_sender: Arc::new(Mutex::new(Some(tx))) }; + let handle = TestSyncOracleHandle { done_syncing_receiver: rx, is_major_syncing: flag }; (Box::new(oracle), handle) } @@ -465,6 +467,7 @@ struct HarnessConfigBuilder { clock: Option, backend: Option, assignment_criteria: Option>, + major_syncing: bool, } impl HarnessConfigBuilder { @@ -476,9 +479,19 @@ impl HarnessConfigBuilder { self } + pub fn major_syncing(&mut self, value: bool) -> &mut Self { + self.major_syncing = value; + self + } + + pub fn backend(&mut self, store: TestStore) -> &mut Self { + self.backend = Some(store); + self + } + pub fn build(&mut self) -> HarnessConfig { let (sync_oracle, sync_oracle_handle) = - self.sync_oracle.take().unwrap_or_else(|| make_sync_oracle(false)); + self.sync_oracle.take().unwrap_or_else(|| make_sync_oracle(self.major_syncing)); let assignment_criteria = self .assignment_criteria @@ -736,11 +749,13 @@ struct BlockConfig { slot: Slot, candidates: Option>, session_info: Option, + end_syncing: bool, } struct ChainBuilder { blocks_by_hash: HashMap, blocks_at_height: BTreeMap>, + is_major_syncing: Arc, } impl ChainBuilder { @@ -748,16 +763,28 @@ impl ChainBuilder { const GENESIS_PARENT_HASH: Hash = Hash::repeat_byte(0x00); pub fn new() -> Self { - let mut builder = - Self { blocks_by_hash: HashMap::new(), blocks_at_height: BTreeMap::new() }; + let mut builder = Self { + blocks_by_hash: HashMap::new(), + blocks_at_height: BTreeMap::new(), + is_major_syncing: Arc::new(AtomicBool::new(false)), + }; builder.add_block_inner( Self::GENESIS_HASH, Self::GENESIS_PARENT_HASH, 0, - BlockConfig { slot: Slot::from(0), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(0), + candidates: None, + session_info: None, + end_syncing: false, + }, ); builder } + pub fn major_syncing(&mut self, major_syncing: Arc) -> &mut Self { + self.is_major_syncing = major_syncing; + self + } pub fn add_block( &mut self, @@ -808,8 +835,16 @@ impl ChainBuilder { } ancestry.reverse(); - import_block(overseer, ancestry.as_ref(), *number, block_config, false, i > 0) - .await; + import_block( + overseer, + ancestry.as_ref(), + *number, + block_config, + false, + i > 0, + self.is_major_syncing.clone(), + ) + .await; let _: Option<()> = future::pending().timeout(Duration::from_millis(100)).await; } } @@ -863,6 +898,7 @@ async fn import_block( config: &BlockConfig, gap: bool, fork: bool, + major_syncing: Arc, ) { let (new_head, new_header) = &hashes[hashes.len() - 1]; let candidates = config.candidates.clone().unwrap_or(vec![( @@ -891,6 +927,12 @@ async fn import_block( h_tx.send(Ok(Some(new_header.clone()))).unwrap(); } ); + + let is_major_syncing = major_syncing.load(Ordering::SeqCst); + if config.end_syncing { + major_syncing.store(false, Ordering::SeqCst); + } + if !fork { let mut _ancestry_step = 0; if gap { @@ -931,7 +973,7 @@ async fn import_block( } } - if number > 0 { + if number > 0 && !is_major_syncing { assert_matches!( overseer_recv(overseer).await, AllMessages::RuntimeApi( @@ -944,7 +986,6 @@ async fn import_block( c_tx.send(Ok(inclusion_events)).unwrap(); } ); - assert_matches!( overseer_recv(overseer).await, AllMessages::RuntimeApi( @@ -984,14 +1025,14 @@ async fn import_block( ); } - if number == 0 { + if number == 0 && !is_major_syncing { assert_matches!( overseer_recv(overseer).await, AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NewBlocks(v)) => { assert_eq!(v.len(), 0usize); } ); - } else { + } else if number > 0 && !is_major_syncing { if !fork { // SessionInfo won't be called for forks - it's already cached assert_matches!( @@ -1031,20 +1072,23 @@ async fn import_block( ); } - assert_matches!( - overseer_recv(overseer).await, - AllMessages::ApprovalDistribution( - ApprovalDistributionMessage::NewBlocks(mut approval_vec) - ) => { - assert_eq!(approval_vec.len(), 1); - let metadata = approval_vec.pop().unwrap(); - let hash = &hashes[number as usize]; - let parent_hash = &hashes[(number - 1) as usize]; - assert_eq!(metadata.hash, hash.0.clone()); - assert_eq!(metadata.parent_hash, parent_hash.0.clone()); - assert_eq!(metadata.slot, config.slot); - } - ); + if !is_major_syncing { + assert_matches!( + overseer_recv(overseer).await, + + AllMessages::ApprovalDistribution( + ApprovalDistributionMessage::NewBlocks(mut approval_vec) + ) => { + assert_eq!(approval_vec.len(), 1); + let metadata = approval_vec.pop().unwrap(); + let hash = &hashes[number as usize]; + let parent_hash = &hashes[(number - 1) as usize]; + assert_eq!(metadata.hash, hash.0.clone()); + assert_eq!(metadata.parent_hash, parent_hash.0.clone()); + assert_eq!(metadata.slot, config.slot); + } + ); + } } } @@ -1072,7 +1116,7 @@ fn subsystem_rejects_bad_assignment_ok_criteria() { block_hash, head, 1, - BlockConfig { slot, candidates: None, session_info: None }, + BlockConfig { slot, candidates: None, session_info: None, end_syncing: false }, ); builder.build(&mut virtual_overseer).await; @@ -1135,7 +1179,7 @@ fn subsystem_rejects_bad_assignment_err_criteria() { block_hash, head, 1, - BlockConfig { slot, candidates: None, session_info: None }, + BlockConfig { slot, candidates: None, session_info: None, end_syncing: false }, ); builder.build(&mut virtual_overseer).await; @@ -1240,6 +1284,7 @@ fn subsystem_rejects_approval_if_no_candidate_entry() { slot, candidates: Some(vec![(candidate_descriptor, CoreIndex(1), GroupIndex(1))]), session_info: None, + end_syncing: false, }, ); builder.build(&mut virtual_overseer).await; @@ -1345,7 +1390,12 @@ fn subsystem_rejects_approval_before_assignment() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1398,7 +1448,12 @@ fn subsystem_rejects_assignment_in_future() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(0), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(0), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1472,6 +1527,7 @@ fn subsystem_accepts_duplicate_assignment() { (candidate_receipt2, CoreIndex(1), GroupIndex(1)), ]), session_info: None, + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -1537,7 +1593,12 @@ fn subsystem_rejects_assignment_with_unknown_candidate() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1582,7 +1643,12 @@ fn subsystem_rejects_oversized_bitfields() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1650,7 +1716,12 @@ fn subsystem_accepts_and_imports_approval_after_assignment() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1741,6 +1812,7 @@ fn subsystem_second_approval_import_only_schedules_wakeups() { slot: Slot::from(0), candidates: None, session_info: Some(session_info), + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -1824,7 +1896,12 @@ fn subsystem_assignment_import_updates_candidate_entry_and_schedules_wakeup() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1873,7 +1950,12 @@ fn subsystem_process_wakeup_schedules_wakeup() { block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot: Slot::from(1), candidates: None, session_info: None }, + BlockConfig { + slot: Slot::from(1), + candidates: None, + session_info: None, + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -1925,7 +2007,7 @@ fn linear_import_act_on_leaf() { hash, head, i, - BlockConfig { slot, candidates: None, session_info: None }, + BlockConfig { slot, candidates: None, session_info: None, end_syncing: false }, ); head = hash; } @@ -1983,7 +2065,7 @@ fn forkful_import_at_same_height_act_on_leaf() { hash, head, i, - BlockConfig { slot, candidates: None, session_info: None }, + BlockConfig { slot, candidates: None, session_info: None, end_syncing: false }, ); head = hash; } @@ -1997,7 +2079,7 @@ fn forkful_import_at_same_height_act_on_leaf() { hash, head, session, - BlockConfig { slot, candidates: None, session_info: None }, + BlockConfig { slot, candidates: None, session_info: None, end_syncing: false }, ); } builder.build(&mut virtual_overseer).await; @@ -2168,6 +2250,7 @@ fn import_checked_approval_updates_entries_and_schedules() { slot, candidates: Some(vec![(candidate_descriptor, CoreIndex(0), GroupIndex(0))]), session_info: Some(session_info), + end_syncing: false, }, ); builder.build(&mut virtual_overseer).await; @@ -2323,6 +2406,7 @@ fn subsystem_import_checked_approval_sets_one_block_bit_at_a_time() { (candidate_receipt2, CoreIndex(1), GroupIndex(1)), ]), session_info: Some(session_info), + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -2445,6 +2529,7 @@ fn approved_ancestor_test( slot: Slot::from(i as u64), candidates: Some(vec![(candidate_receipt, CoreIndex(0), GroupIndex(0))]), session_info: None, + end_syncing: false, }, ); } @@ -2623,13 +2708,19 @@ fn subsystem_validate_approvals_cache() { slot, candidates: candidates.clone(), session_info: Some(session_info.clone()), + end_syncing: false, }, ) .add_block( fork_block_hash, ChainBuilder::GENESIS_HASH, 1, - BlockConfig { slot, candidates, session_info: Some(session_info) }, + BlockConfig { + slot, + candidates, + session_info: Some(session_info), + end_syncing: false, + }, ) .build(&mut virtual_overseer) .await; @@ -2740,6 +2831,7 @@ fn subsystem_doesnt_distribute_duplicate_compact_assignments() { (candidate_receipt2.clone(), CoreIndex(1), GroupIndex(1)), ]), session_info: None, + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -2997,6 +3089,7 @@ where slot, candidates: Some(vec![(candidate_receipt, CoreIndex(0), GroupIndex(2))]), session_info: Some(session_info), + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -3314,6 +3407,7 @@ fn pre_covers_dont_stall_approval() { slot, candidates: Some(vec![(candidate_descriptor, CoreIndex(0), GroupIndex(0))]), session_info: Some(session_info), + end_syncing: false, }, ); builder.build(&mut virtual_overseer).await; @@ -3491,6 +3585,7 @@ fn waits_until_approving_assignments_are_old_enough() { slot, candidates: Some(vec![(candidate_descriptor, CoreIndex(0), GroupIndex(0))]), session_info: Some(session_info), + end_syncing: false, }, ); builder.build(&mut virtual_overseer).await; @@ -3705,6 +3800,7 @@ fn test_approval_is_sent_on_max_approval_coalesce_count() { slot, candidates: candidates.clone(), session_info: Some(session_info.clone()), + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -3943,8 +4039,7 @@ fn test_approval_is_sent_on_max_approval_coalesce_wait() { let store = config.backend(); test_harness(config, |test_harness| async move { - let TestHarness { mut virtual_overseer, clock, sync_oracle_handle: _sync_oracle_handle } = - test_harness; + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle: _ } = test_harness; assert_matches!( overseer_recv(&mut virtual_overseer).await, @@ -4006,6 +4101,7 @@ fn test_approval_is_sent_on_max_approval_coalesce_wait() { slot, candidates: candidates.clone(), session_info: Some(session_info.clone()), + end_syncing: false, }, ) .build(&mut virtual_overseer) @@ -4040,3 +4136,569 @@ fn test_approval_is_sent_on_max_approval_coalesce_wait() { virtual_overseer }); } + +// Builds a chain with a fork where both relay blocks include the same candidate. +async fn build_chain_with_two_blocks_with_one_candidate_each( + block_hash1: Hash, + block_hash2: Hash, + slot: Slot, + sync_oracle_handle: TestSyncOracleHandle, + candidate_receipt: CandidateReceipt, +) -> (ChainBuilder, SessionInfo) { + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Eve, + ]; + let session_info = SessionInfo { + validator_groups: IndexedVec::>::from(vec![ + vec![ValidatorIndex(0), ValidatorIndex(1)], + vec![ValidatorIndex(2)], + vec![ValidatorIndex(3), ValidatorIndex(4)], + ]), + ..session_info(&validators) + }; + + let candidates = Some(vec![(candidate_receipt.clone(), CoreIndex(0), GroupIndex(0))]); + let mut chain_builder = ChainBuilder::new(); + + chain_builder + .major_syncing(sync_oracle_handle.is_major_syncing.clone()) + .add_block( + block_hash1, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot, + candidates: candidates.clone(), + session_info: Some(session_info.clone()), + end_syncing: false, + }, + ) + .add_block( + block_hash2, + ChainBuilder::GENESIS_HASH, + 1, + BlockConfig { + slot, + candidates, + session_info: Some(session_info.clone()), + end_syncing: true, + }, + ); + (chain_builder, session_info) +} + +async fn setup_overseer_with_two_blocks_each_with_one_assignment_triggered( + virtual_overseer: &mut VirtualOverseer, + store: TestStore, + clock: &Box, + sync_oracle_handle: TestSyncOracleHandle, +) { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + let fork_block_hash = Hash::repeat_byte(0x02); + let candidate_commitments = CandidateCommitments::default(); + let mut candidate_receipt = dummy_candidate_receipt(block_hash); + candidate_receipt.commitments_hash = candidate_commitments.hash(); + let candidate_hash = candidate_receipt.hash(); + let slot = Slot::from(1); + let (chain_builder, _session_info) = build_chain_with_two_blocks_with_one_candidate_each( + block_hash, + fork_block_hash, + slot, + sync_oracle_handle, + candidate_receipt, + ) + .await; + chain_builder.build(virtual_overseer).await; + + assert!(!clock.inner.lock().current_wakeup_is(1)); + clock.inner.lock().wakeup_all(1); + + assert!(clock.inner.lock().current_wakeup_is(slot_to_tick(slot))); + clock.inner.lock().wakeup_all(slot_to_tick(slot)); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + clock.inner.lock().wakeup_all(slot_to_tick(slot + 2)); + + assert_eq!(clock.inner.lock().wakeups.len(), 0); + + futures_timer::Delay::new(Duration::from_millis(200)).await; + + let candidate_entry = store.load_candidate_entry(&candidate_hash).unwrap().unwrap(); + let our_assignment = + candidate_entry.approval_entry(&block_hash).unwrap().our_assignment().unwrap(); + assert!(our_assignment.triggered()); +} + +// Tests that for candidates that we did not approve yet, for which we triggered the assignment and +// the approval work we restart the work to approve it. +#[test] +fn subsystem_relaunches_approval_work_on_restart() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + let store_clone = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle } = test_harness; + + setup_overseer_with_two_blocks_each_with_one_assignment_triggered( + &mut virtual_overseer, + store, + &clock, + sync_oracle_handle, + ) + .await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _, + )) => { + } + ); + + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _ + )) => { + } + ); + + // Bail early after the assignment has been distributed but before we answer with the mocked + // approval from CandidateValidation. + virtual_overseer + }); + + // Restart a new approval voting subsystem with the same database and major syncing true untill + // the last leaf. + let config = HarnessConfigBuilder::default().backend(store_clone).major_syncing(true).build(); + + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle } = test_harness; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + let fork_block_hash = Hash::repeat_byte(0x02); + let candidate_commitments = CandidateCommitments::default(); + let mut candidate_receipt = dummy_candidate_receipt(block_hash); + candidate_receipt.commitments_hash = candidate_commitments.hash(); + let slot = Slot::from(1); + clock.inner.lock().set_tick(slot_to_tick(slot + 2)); + let (chain_builder, session_info) = build_chain_with_two_blocks_with_one_candidate_each( + block_hash, + fork_block_hash, + slot, + sync_oracle_handle, + candidate_receipt, + ) + .await; + + chain_builder.build(&mut virtual_overseer).await; + + futures_timer::Delay::new(Duration::from_millis(2000)).await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request( + _, + RuntimeApiRequest::SessionInfo(_, si_tx), + ) + ) => { + si_tx.send(Ok(Some(session_info.clone()))).unwrap(); + } + ); + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request( + _, + RuntimeApiRequest::SessionExecutorParams(_, si_tx), + ) + ) => { + // Make sure all SessionExecutorParams calls are not made for the leaf (but for its relay parent) + si_tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(_, RuntimeApiRequest::NodeFeatures(_, si_tx), ) + ) => { + si_tx.send(Ok(NodeFeatures::EMPTY)).unwrap(); + } + ); + + // On major syncing ending Approval voting should send all the necessary messages for a + // candidate to be approved. + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NewBlocks( + _, + )) => { + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _, + )) => { + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _, + )) => { + } + ); + + // Guarantees the approval work has been relaunched. + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive { + exec_kind, + response_sender, + .. + }) if exec_kind == PvfExecKind::Approval => { + response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) + .unwrap(); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) + ); + + // Assert that there are no more messages being sent by the subsystem + assert!(overseer_recv(&mut virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); + + virtual_overseer + }); +} + +// Test that cached approvals, which are candidates that we approved but we didn't issue +// the signature yet because we want to coalesce it with more candidate are sent after restart. +#[test] +fn subsystem_sends_pending_approvals_on_approval_restart() { + let assignment_criteria = Box::new(MockAssignmentCriteria( + || { + let mut assignments = HashMap::new(); + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }) + .into(), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + + let _ = assignments.insert( + CoreIndex(0), + approval_db::v2::OurAssignment { + cert: garbage_assignment_cert_v2(AssignmentCertKindV2::RelayVRFModuloCompact { + core_bitfield: vec![CoreIndex(0), CoreIndex(1), CoreIndex(2)] + .try_into() + .unwrap(), + }), + tranche: 0, + validator_index: ValidatorIndex(0), + triggered: false, + } + .into(), + ); + assignments + }, + |_| Ok(0), + )); + let config = HarnessConfigBuilder::default().assignment_criteria(assignment_criteria).build(); + let store = config.backend(); + let store_clone = config.backend(); + + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle } = test_harness; + + setup_overseer_with_two_blocks_each_with_one_assignment_triggered( + &mut virtual_overseer, + store, + &clock, + sync_oracle_handle, + ) + .await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _, + )) => { + } + ); + + recover_available_data(&mut virtual_overseer).await; + fetch_validation_code(&mut virtual_overseer).await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _ + )) => { + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive { + exec_kind, + response_sender, + .. + }) if exec_kind == PvfExecKind::Approval => { + response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) + .unwrap(); + } + ); + + // Configure a big coalesce number, so that the signature is cached instead of being sent to + // approval-distribution. + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 6, + })); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 6, + })); + } + ); + + // Assert that there are no more messages being sent by the subsystem + assert!(overseer_recv(&mut virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); + + virtual_overseer + }); + + let config = HarnessConfigBuilder::default().backend(store_clone).major_syncing(true).build(); + // On restart signatures should be sent to approval-distribution without relaunching the + // approval work. + test_harness(config, |test_harness| async move { + let TestHarness { mut virtual_overseer, clock, sync_oracle_handle } = test_harness; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ChainApi(ChainApiMessage::FinalizedBlockNumber(rx)) => { + rx.send(Ok(0)).unwrap(); + } + ); + + let block_hash = Hash::repeat_byte(0x01); + let fork_block_hash = Hash::repeat_byte(0x02); + let candidate_commitments = CandidateCommitments::default(); + let mut candidate_receipt = dummy_candidate_receipt(block_hash); + candidate_receipt.commitments_hash = candidate_commitments.hash(); + let slot = Slot::from(1); + + clock.inner.lock().set_tick(slot_to_tick(slot + 2)); + let (chain_builder, session_info) = build_chain_with_two_blocks_with_one_candidate_each( + block_hash, + fork_block_hash, + slot, + sync_oracle_handle, + candidate_receipt, + ) + .await; + chain_builder.build(&mut virtual_overseer).await; + + futures_timer::Delay::new(Duration::from_millis(2000)).await; + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NewBlocks( + _, + )) => { + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _, + )) => { + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeAssignment( + _, + _, + )) => { + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request( + _, + RuntimeApiRequest::SessionInfo(_, si_tx), + ) + ) => { + si_tx.send(Ok(Some(session_info.clone()))).unwrap(); + } + ); + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request( + _, + RuntimeApiRequest::SessionExecutorParams(_, si_tx), + ) + ) => { + // Make sure all SessionExecutorParams calls are not made for the leaf (but for its relay parent) + si_tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(_, RuntimeApiRequest::NodeFeatures(_, si_tx), ) + ) => { + si_tx.send(Ok(NodeFeatures::EMPTY)).unwrap(); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request(_, RuntimeApiRequest::ApprovalVotingParams(_, sender))) => { + let _ = sender.send(Ok(ApprovalVotingParams { + max_approval_coalesce_count: 1, + })); + } + ); + + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::ApprovalDistribution(ApprovalDistributionMessage::DistributeApproval(_)) + ); + + // Assert that there are no more messages being sent by the subsystem + assert!(overseer_recv(&mut virtual_overseer).timeout(TIMEOUT / 2).await.is_none()); + + virtual_overseer + }); +} From c244a94e4a77c0798c5a45d3004ec2389378b394 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Thu, 29 Feb 2024 11:41:11 +0000 Subject: [PATCH 11/13] update development setup in sdk-docs (#3506) Co-authored-by: Liam Aharon --- .../development_environment_advice.rs | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/sdk/src/reference_docs/development_environment_advice.rs b/docs/sdk/src/reference_docs/development_environment_advice.rs index 431769597936..21bbe78836c4 100644 --- a/docs/sdk/src/reference_docs/development_environment_advice.rs +++ b/docs/sdk/src/reference_docs/development_environment_advice.rs @@ -111,3 +111,74 @@ //! If you have a powerful remote server available, you may consider using //! [cargo-remote](https://github.com/sgeisler/cargo-remote) to execute cargo commands on it, //! freeing up local resources for other tasks like `rust-analyzer`. +//! +//! When using `cargo-remote`, you can configure your editor to perform the the typical +//! "check-on-save" remotely as well. The configuration for VSCode is as follows: +//! +//! ```json +//! { +//! "rust-analyzer.cargo.buildScripts.overrideCommand": [ +//! "cargo", +//! "remote", +//! "--build-env", +//! "SKIP_WASM_BUILD=1", +//! "--", +//! "check", +//! "--message-format=json", +//! "--all-targets", +//! "--all-features", +//! "--target-dir=target/rust-analyzer" +//! ], +//! "rust-analyzer.check.overrideCommand": [ +//! "cargo", +//! "remote", +//! "--build-env", +//! "SKIP_WASM_BUILD=1", +//! "--", +//! "check", +//! "--workspace", +//! "--message-format=json", +//! "--all-targets", +//! "--all-features", +//! "--target-dir=target/rust-analyzer" +//! ], +//! } +//! ``` +//! +//! //! and the same in Lua for `neovim/nvim-lspconfig`: +//! +//! ```lua +//! ["rust-analyzer"] = { +//! cargo = { +//! buildScripts = { +//! overrideCommand = { +//! "cargo", +//! "remote", +//! "--build-env", +//! "SKIP_WASM_BUILD=1", +//! "--", +//! "check", +//! "--message-format=json", +//! "--all-targets", +//! "--all-features", +//! "--target-dir=target/rust-analyzer" +//! }, +//! }, +//! check = { +//! overrideCommand = { +//! "cargo", +//! "remote", +//! "--build-env", +//! "SKIP_WASM_BUILD=1", +//! "--", +//! "check", +//! "--workspace", +//! "--message-format=json", +//! "--all-targets", +//! "--all-features", +//! "--target-dir=target/rust-analyzer" +//! }, +//! }, +//! }, +//! }, +//! ``` From 7f5d308d2983dcce324a8625edf3f2a838209da8 Mon Sep 17 00:00:00 2001 From: Andrei Sandu <54316454+sandreim@users.noreply.github.com> Date: Thu, 29 Feb 2024 15:32:58 +0200 Subject: [PATCH 12/13] Enable elastic scaling node feature in local testnets genesis (#3509) Signed-off-by: Andrei Sandu --- Cargo.lock | 1 + polkadot/node/service/Cargo.toml | 5 ++- polkadot/node/service/src/chain_spec.rs | 7 +++- polkadot/runtime/parachains/src/builder.rs | 24 ++++++++---- .../functional/0012-elastic-scaling-mvp.zndsl | 11 +----- .../functional/0012-enable-node-feature.js | 37 ------------------- 6 files changed, 27 insertions(+), 58 deletions(-) delete mode 100644 polkadot/zombienet_tests/functional/0012-enable-node-feature.js diff --git a/Cargo.lock b/Cargo.lock index 3838ed0e94a1..991541ee84ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13526,6 +13526,7 @@ version = "7.0.0" dependencies = [ "assert_matches", "async-trait", + "bitvec", "env_logger 0.9.3", "frame-benchmarking", "frame-benchmarking-cli", diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index 8fd9f20b7bcf..734dcbdeb441 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -93,6 +93,7 @@ kvdb-rocksdb = { version = "0.19.0", optional = true } parity-db = { version = "0.4.12", optional = true } codec = { package = "parity-scale-codec", version = "3.6.1" } parking_lot = "0.12.1" +bitvec = { version = "1.0.1", optional = true } # Polkadot polkadot-core-primitives = { path = "../../core-primitives" } @@ -184,8 +185,8 @@ full-node = [ ] # Configure the native runtimes to use. -westend-native = ["westend-runtime", "westend-runtime-constants"] -rococo-native = ["rococo-runtime", "rococo-runtime-constants"] +westend-native = ["bitvec", "westend-runtime", "westend-runtime-constants"] +rococo-native = ["bitvec", "rococo-runtime", "rococo-runtime-constants"] runtime-benchmarks = [ "frame-benchmarking-cli/runtime-benchmarks", diff --git a/polkadot/node/service/src/chain_spec.rs b/polkadot/node/service/src/chain_spec.rs index af05af87a46b..1c44b17b6fd2 100644 --- a/polkadot/node/service/src/chain_spec.rs +++ b/polkadot/node/service/src/chain_spec.rs @@ -122,7 +122,9 @@ pub fn wococo_config() -> Result { fn default_parachains_host_configuration( ) -> polkadot_runtime_parachains::configuration::HostConfiguration { - use polkadot_primitives::{AsyncBackingParams, MAX_CODE_SIZE, MAX_POV_SIZE}; + use polkadot_primitives::{ + vstaging::node_features::FeatureIndex, AsyncBackingParams, MAX_CODE_SIZE, MAX_POV_SIZE, + }; polkadot_runtime_parachains::configuration::HostConfiguration { validation_upgrade_cooldown: 2u32, @@ -155,6 +157,9 @@ fn default_parachains_host_configuration( max_candidate_depth: 3, allowed_ancestry_len: 2, }, + node_features: bitvec::vec::BitVec::from_element( + 1u8 << (FeatureIndex::ElasticScalingMVP as usize), + ), scheduler_params: SchedulerParams { lookahead: 2, group_rotation_frequency: 20, diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 0e4e659fef27..5b218addb1a2 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -25,12 +25,13 @@ use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; use primitives::{ - collator_signature_payload, AvailabilityBitfield, BackedCandidate, CandidateCommitments, - CandidateDescriptor, CandidateHash, CollatorId, CollatorSignature, CommittedCandidateReceipt, - CompactStatement, CoreIndex, DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, - Id as ParaId, IndexedVec, InherentData as ParachainsInherentData, InvalidDisputeStatementKind, - PersistedValidationData, SessionIndex, SigningContext, UncheckedSigned, - ValidDisputeStatementKind, ValidationCode, ValidatorId, ValidatorIndex, ValidityAttestation, + collator_signature_payload, vstaging::node_features::FeatureIndex, AvailabilityBitfield, + BackedCandidate, CandidateCommitments, CandidateDescriptor, CandidateHash, CollatorId, + CollatorSignature, CommittedCandidateReceipt, CompactStatement, CoreIndex, DisputeStatement, + DisputeStatementSet, GroupIndex, HeadData, Id as ParaId, IndexedVec, + InherentData as ParachainsInherentData, InvalidDisputeStatementKind, PersistedValidationData, + SessionIndex, SigningContext, UncheckedSigned, ValidDisputeStatementKind, ValidationCode, + ValidatorId, ValidatorIndex, ValidityAttestation, }; use sp_core::{sr25519, H256}; use sp_runtime::{ @@ -509,7 +510,7 @@ impl BenchBuilder { .iter() .map(|(seed, num_votes)| { assert!(*num_votes <= validators.len() as u32); - let (para_id, _core_idx, group_idx) = self.create_indexes(*seed); + let (para_id, core_idx, group_idx) = self.create_indexes(*seed); // This generates a pair and adds it to the keystore, returning just the public. let collator_public = CollatorId::generate_pair(None); @@ -586,11 +587,18 @@ impl BenchBuilder { }) .collect(); + // Check if the elastic scaling bit is set, if so we need to supply the core index + // in the generated candidate. + let core_idx = configuration::Pallet::::config() + .node_features + .get(FeatureIndex::ElasticScalingMVP as usize) + .map(|_the_bit| core_idx); + BackedCandidate::::new( candidate, validity_votes, bitvec::bitvec![u8, bitvec::order::Lsb0; 1; group_validators.len()], - None, + core_idx, ) }) .collect() diff --git a/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.zndsl b/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.zndsl index a7193c9282b9..edc87c802a09 100644 --- a/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.zndsl +++ b/polkadot/zombienet_tests/functional/0012-elastic-scaling-mvp.zndsl @@ -1,4 +1,4 @@ -Description: Test that a paraid acquiring multiple cores does not brick itself if ElasticScalingMVP feature is enabled +Description: Test that a paraid acquiring multiple cores does not brick itself if ElasticScalingMVP feature is enabled in genesis Network: ./0012-elastic-scaling-mvp.toml Creds: config @@ -15,14 +15,5 @@ alice: js-script ./0012-register-para.js return is 0 within 600 seconds validator: reports substrate_block_height{status="finalized"} is at least 35 within 100 seconds -# Parachain will now be stalled -validator: parachain 2000 block height is lower than 20 within 300 seconds - -# Enable the ElasticScalingMVP node feature. -alice: js-script ./0012-enable-node-feature.js with "1" return is 0 within 600 seconds - -# Wait two sessions for the config to be updated. -sleep 120 seconds - # Ensure parachain is now making progress. validator: parachain 2000 block height is at least 30 within 200 seconds diff --git a/polkadot/zombienet_tests/functional/0012-enable-node-feature.js b/polkadot/zombienet_tests/functional/0012-enable-node-feature.js deleted file mode 100644 index 4822e1f66447..000000000000 --- a/polkadot/zombienet_tests/functional/0012-enable-node-feature.js +++ /dev/null @@ -1,37 +0,0 @@ -async function run(nodeName, networkInfo, index) { - const { wsUri, userDefinedTypes } = networkInfo.nodesByName[nodeName]; - const api = await zombie.connect(wsUri, userDefinedTypes); - - await zombie.util.cryptoWaitReady(); - - // account to submit tx - const keyring = new zombie.Keyring({ type: "sr25519" }); - const alice = keyring.addFromUri("//Alice"); - - await new Promise(async (resolve, reject) => { - const unsub = await api.tx.sudo - .sudo(api.tx.configuration.setNodeFeature(Number(index), true)) - .signAndSend(alice, ({ status, isError }) => { - if (status.isInBlock) { - console.log( - `Transaction included at blockhash ${status.asInBlock}`, - ); - } else if (status.isFinalized) { - console.log( - `Transaction finalized at blockHash ${status.asFinalized}`, - ); - unsub(); - return resolve(); - } else if (isError) { - console.log(`Transaction error`); - reject(`Transaction error`); - } - }); - }); - - - - return 0; -} - -module.exports = { run }; From c0e52a9ed61451152c4e096c58f24fb4e329bd64 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:08:08 +0000 Subject: [PATCH 13/13] Fix call enum's metadata regression (#3513) This fixes an issue introduced in https://github.com/paritytech/substrate/pull/14101, in which I removed the `Call` enum's documentation and replaced it with a link to the `Pallet` struct, but this also removed any docs related to call from the metadata. I tried to add a regression test for this, but it seems to me that this is not possible, given that using `type-info` we only assert in type-ids for `Call`, `Event` and `Error`. I removed some doc comments from a test setup in `frame-support-test` to demonstrate the issue there. @jsdw do you have any comments on this? I also fixed a small issue in the custom html/css of `polkadot-sdk-doc` crate, making sure it does not affect the rust-doc page of all other crates. - [x] Investigate a regression test - [x] prdoc --- docs/sdk/headers/header.html | 111 +++++++++--------- docs/sdk/headers/theme.css | 25 ++-- prdoc/pr_3513.prdoc | 18 +++ substrate/frame/support/Cargo.toml | 21 +++- .../procedural/src/pallet/expand/call.rs | 21 +--- substrate/frame/support/test/tests/pallet.rs | 64 +++++++--- 6 files changed, 157 insertions(+), 103 deletions(-) create mode 100644 prdoc/pr_3513.prdoc diff --git a/docs/sdk/headers/header.html b/docs/sdk/headers/header.html index 68dc5d5509e6..e28458c4ccc7 100644 --- a/docs/sdk/headers/header.html +++ b/docs/sdk/headers/header.html @@ -54,7 +54,7 @@ sidebarElems.style.display = 'none'; // Add click event listener to the button - expandButton.addEventListener('click', function() { + expandButton.addEventListener('click', function () { // Toggle the display of the '.sidebar-elems' if (sidebarElems.style.display === 'none') { sidebarElems.style.display = 'block'; @@ -72,6 +72,9 @@ if (!crate_name.textContent.startsWith("polkadot_sdk_docs")) { console.log("skipping -- not `polkadot_sdk_docs`"); return; + } else { + // insert class 'sdk-docs' to the body, so it enables the custom css rules. + document.body.classList.add("sdk-docs"); } createToC(); @@ -82,58 +85,60 @@ diff --git a/docs/sdk/headers/theme.css b/docs/sdk/headers/theme.css index bb9254ec4a82..a488e15c36b7 100644 --- a/docs/sdk/headers/theme.css +++ b/docs/sdk/headers/theme.css @@ -1,16 +1,17 @@ :root { - --polkadot-pink: #E6007A ; - --polkadot-green: #56F39A ; - --polkadot-lime: #D3FF33 ; - --polkadot-cyan: #00B2FF ; - --polkadot-purple: #552BBF ; - } - -body > nav.sidebar > div.sidebar-crate > a > img { - /* logo width, given that the sidebar width is 200px; */ - width: 190px; + --polkadot-pink: #E6007A; + --polkadot-green: #56F39A; + --polkadot-lime: #D3FF33; + --polkadot-cyan: #00B2FF; + --polkadot-purple: #552BBF; } -body nav.sidebar { - flex: 0 0 250px; +body.sdk-docs { + nav.sidebar>div.sidebar-crate>a>img { + width: 190px; + } + + nav.sidebar { + flex: 0 0 250px; + } } diff --git a/prdoc/pr_3513.prdoc b/prdoc/pr_3513.prdoc new file mode 100644 index 000000000000..e1f2afe280a5 --- /dev/null +++ b/prdoc/pr_3513.prdoc @@ -0,0 +1,18 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Fix call enum's metadata regression + +doc: + - audience: Runtime Dev + - audience: Runtime User + description: | + This PR fixes an issue where in the metadata of all FRAME-based chains, the documentation for + all extrinsic/call/transactions has been replaced by a single line saying "see Pallet::name". + + Many wallets display the metadata of transactions to the user before signing, and this bug + might have affected this process. + +crates: + - name: frame + - name: frame-support diff --git a/substrate/frame/support/Cargo.toml b/substrate/frame/support/Cargo.toml index 72e2d0bfa569..c19f478c803e 100644 --- a/substrate/frame/support/Cargo.toml +++ b/substrate/frame/support/Cargo.toml @@ -18,13 +18,24 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] array-bytes = { version = "6.1", default-features = false } serde = { features = ["alloc", "derive"], workspace = true } -codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive", "max-encoded-len"] } -scale-info = { version = "2.10.0", default-features = false, features = ["derive"] } -frame-metadata = { version = "16.0.0", default-features = false, features = ["current"] } -sp-api = { path = "../../primitives/api", default-features = false, features = ["frame-metadata"] } +codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = [ + "derive", + "max-encoded-len", +] } +scale-info = { version = "2.10.0", default-features = false, features = [ + "derive", +] } +frame-metadata = { version = "16.0.0", default-features = false, features = [ + "current", +] } +sp-api = { path = "../../primitives/api", default-features = false, features = [ + "frame-metadata", +] } sp-std = { path = "../../primitives/std", default-features = false } sp-io = { path = "../../primitives/io", default-features = false } -sp-runtime = { path = "../../primitives/runtime", default-features = false, features = ["serde"] } +sp-runtime = { path = "../../primitives/runtime", default-features = false, features = [ + "serde", +] } sp-tracing = { path = "../../primitives/tracing", default-features = false } sp-core = { path = "../../primitives/core", default-features = false } sp-arithmetic = { path = "../../primitives/arithmetic", default-features = false } diff --git a/substrate/frame/support/procedural/src/pallet/expand/call.rs b/substrate/frame/support/procedural/src/pallet/expand/call.rs index f43faba1ee0c..f395872c8a80 100644 --- a/substrate/frame/support/procedural/src/pallet/expand/call.rs +++ b/substrate/frame/support/procedural/src/pallet/expand/call.rs @@ -18,7 +18,7 @@ use crate::{ pallet::{ expand::warnings::{weight_constant_warning, weight_witness_warning}, - parse::call::{CallVariantDef, CallWeightDef}, + parse::call::CallWeightDef, Def, }, COUNTER, @@ -112,22 +112,7 @@ pub fn expand_call(def: &mut Def) -> proc_macro2::TokenStream { } debug_assert_eq!(fn_weight.len(), methods.len()); - let map_fn_docs = if !def.dev_mode { - // Emit the [`Pallet::method`] documentation only for non-dev modes. - |method: &CallVariantDef| { - let reference = format!("See [`Pallet::{}`].", method.name); - quote!(#reference) - } - } else { - // For the dev-mode do not provide a documenation link as it will break the - // `cargo doc` if the pallet is private inside a test. - |method: &CallVariantDef| { - let reference = format!("See `Pallet::{}`.", method.name); - quote!(#reference) - } - }; - - let fn_doc = methods.iter().map(map_fn_docs).collect::>(); + let fn_doc = methods.iter().map(|method| &method.docs).collect::>(); let args_name = methods .iter() @@ -309,7 +294,7 @@ pub fn expand_call(def: &mut Def) -> proc_macro2::TokenStream { ), #( #cfg_attrs - #[doc = #fn_doc] + #( #[doc = #fn_doc] )* #[codec(index = #call_index)] #fn_name { #( diff --git a/substrate/frame/support/test/tests/pallet.rs b/substrate/frame/support/test/tests/pallet.rs index 7bee4b7c8206..607f54ccc131 100644 --- a/substrate/frame/support/test/tests/pallet.rs +++ b/substrate/frame/support/test/tests/pallet.rs @@ -209,7 +209,7 @@ pub mod pallet { where T::AccountId: From + From + SomeAssociation1, { - /// Doc comment put in metadata + /// call foo doc comment put in metadata #[pallet::call_index(0)] #[pallet::weight(Weight::from_parts(*foo as u64, 0))] pub fn foo( @@ -225,7 +225,7 @@ pub mod pallet { Ok(().into()) } - /// Doc comment put in metadata + /// call foo_storage_layer doc comment put in metadata #[pallet::call_index(1)] #[pallet::weight({1})] pub fn foo_storage_layer( @@ -270,7 +270,7 @@ pub mod pallet { #[pallet::error] #[derive(PartialEq, Eq)] pub enum Error { - /// doc comment put into metadata + /// error doc comment put in metadata InsufficientProposersBalance, NonExistentStorageValue, Code(u8), @@ -287,9 +287,8 @@ pub mod pallet { where T::AccountId: SomeAssociation1 + From, { - /// doc comment put in metadata + /// event doc comment put in metadata Proposed(::AccountId), - /// doc Spending(BalanceOf), Something(u32), SomethingElse(::_1), @@ -750,8 +749,7 @@ pub type UncheckedExtrinsic = sp_runtime::testing::TestXt>; frame_support::construct_runtime!( - pub struct Runtime - { + pub struct Runtime { // Exclude part `Storage` in order not to check its metadata in tests. System: frame_system exclude_parts { Pallet, Storage }, Example: pallet, @@ -772,6 +770,14 @@ fn _ensure_call_is_correctly_excluded_and_included(call: RuntimeCall) { } } +fn maybe_docs(doc: Vec<&'static str>) -> Vec<&'static str> { + if cfg!(feature = "no-metadata-docs") { + vec![] + } else { + doc + } +} + #[test] fn transactional_works() { TestExternalities::default().execute_with(|| { @@ -1362,19 +1368,47 @@ fn migrate_from_pallet_version_to_storage_version() { }); } +#[test] +fn pallet_item_docs_in_metadata() { + // call + let call_variants = match meta_type::>().type_info().type_def { + scale_info::TypeDef::Variant(variants) => variants.variants, + _ => unreachable!(), + }; + + assert_eq!(call_variants[0].docs, maybe_docs(vec!["call foo doc comment put in metadata"])); + assert_eq!( + call_variants[1].docs, + maybe_docs(vec!["call foo_storage_layer doc comment put in metadata"]) + ); + assert!(call_variants[2].docs.is_empty()); + + // event + let event_variants = match meta_type::>().type_info().type_def { + scale_info::TypeDef::Variant(variants) => variants.variants, + _ => unreachable!(), + }; + + assert_eq!(event_variants[0].docs, maybe_docs(vec!["event doc comment put in metadata"])); + assert!(event_variants[1].docs.is_empty()); + + // error + let error_variants = match meta_type::>().type_info().type_def { + scale_info::TypeDef::Variant(variants) => variants.variants, + _ => unreachable!(), + }; + + assert_eq!(error_variants[0].docs, maybe_docs(vec!["error doc comment put in metadata"])); + assert!(error_variants[1].docs.is_empty()); + + // storage is already covered in the main `fn metadata` test. +} + #[test] fn metadata() { use codec::Decode; use frame_metadata::{v15::*, *}; - fn maybe_docs(doc: Vec<&'static str>) -> Vec<&'static str> { - if cfg!(feature = "no-metadata-docs") { - vec![] - } else { - doc - } - } - let readme = "Support code for the runtime.\n\nLicense: Apache-2.0\n"; let expected_pallet_doc = vec![" Pallet documentation", readme, readme];