From d6f6a6ee34d54dab25919719200e9095d362ee30 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Tue, 25 Apr 2023 11:09:33 +0400 Subject: [PATCH 1/5] add batch inserting into remote externalities --- primitives/state-machine/src/testing.rs | 8 +++ utils/frame/remote-externalities/src/lib.rs | 54 ++++++++++----------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 3a0165fe4588d..3846d9dbc56bf 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -132,6 +132,14 @@ where self.offchain_db.clone() } + /// Batch insert key/values into backend + pub fn batch_insert(&mut self, kvs: Vec<(StorageKey, StorageValue)>) { + self.backend.insert( + vec![(None, kvs.iter().map(|(k, v)| (k.clone(), Some(v.clone()))).collect())], + self.state_version, + ); + } + /// Insert key/value into backend pub fn insert(&mut self, k: StorageKey, v: StorageValue) { self.backend.insert(vec![(None, vec![(k, Some(v))])], self.state_version); diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index b60e2bc75d0db..816164383c633 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -638,24 +638,20 @@ where let mut processed = 0usize; loop { match rx.next().await.unwrap() { - Message::Batch(kv) => { - for (k, v) in kv { - processed += 1; - if processed % 50_000 == 0 || processed == keys.len() || processed == 1 { - log::info!( - target: LOG_TARGET, - "inserting keys progress = {:.0}% [{} / {}]", - (processed as f32 / keys.len() as f32) * 100f32, - processed, - keys.len(), - ); - } - // skip writing the child root data. - if is_default_child_storage_key(k.as_ref()) { - continue - } - pending_ext.insert(k, v); - } + Message::Batch(kvs) => { + let kvs = kvs + .into_iter() + .filter(|(k, _)| !is_default_child_storage_key(k)) + .collect::>(); + processed += kvs.len(); + pending_ext.batch_insert(kvs); + log::info!( + target: LOG_TARGET, + "inserting keys progress = {:.0}% [{} / {}]", + (processed as f32 / keys.len() as f32) * 100f32, + processed, + keys.len(), + ); }, Message::BatchFailed(error) => { log::error!(target: LOG_TARGET, "Batch processing failed: {:?}", error); @@ -876,7 +872,7 @@ where ); match self.rpc_get_storage(key.clone(), Some(at)).await? { Some(value) => { - pending_ext.insert(key.clone().0, value.clone().0); + pending_ext.batch_insert(vec![(key.clone().0, value.clone().0)]); keys_and_values.push((key, value)); }, None => { @@ -1010,13 +1006,15 @@ where ); info!(target: LOG_TARGET, "injecting a total of {} top keys", top.len()); - for (k, v) in top { - // skip writing the child root data. - if is_default_child_storage_key(k.as_ref()) { - continue - } - inner_ext.insert(k.0, v.0); - } + let top = top + .into_iter() + .filter(|(k, _)| { + // skip writing the child root data. + !is_default_child_storage_key(k.as_ref()) + }) + .map(|(k, v)| (k.0, v.0)) + .collect::>(); + inner_ext.batch_insert(top); info!( target: LOG_TARGET, @@ -1052,9 +1050,7 @@ where "extending externalities with {} manually injected key-values", self.hashed_key_values.len() ); - for (k, v) in self.hashed_key_values { - ext.insert(k.0, v.0); - } + ext.batch_insert(self.hashed_key_values.into_iter().map(|(k, v)| (k.0, v.0)).collect()); } // exclude manual key values. From 74002fabf50b690fd243f073ad7fd45a48be2baa Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Tue, 25 Apr 2023 11:14:28 +0400 Subject: [PATCH 2/5] use into_iter --- primitives/state-machine/src/testing.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 3846d9dbc56bf..de728f5bca7d5 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -135,7 +135,7 @@ where /// Batch insert key/values into backend pub fn batch_insert(&mut self, kvs: Vec<(StorageKey, StorageValue)>) { self.backend.insert( - vec![(None, kvs.iter().map(|(k, v)| (k.clone(), Some(v.clone()))).collect())], + vec![(None, kvs.into_iter().map(|(k, v)| (k, Some(v))).collect())], self.state_version, ); } From 52216259c29f90487b81930b6974b56897a0694f Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Tue, 25 Apr 2023 11:29:03 +0400 Subject: [PATCH 3/5] remove redundant comment --- utils/frame/remote-externalities/src/lib.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 816164383c633..90a461db84a06 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -1008,10 +1008,7 @@ where info!(target: LOG_TARGET, "injecting a total of {} top keys", top.len()); let top = top .into_iter() - .filter(|(k, _)| { - // skip writing the child root data. - !is_default_child_storage_key(k.as_ref()) - }) + .filter(|(k, _)| !is_default_child_storage_key(k.as_ref())) .map(|(k, v)| (k.0, v.0)) .collect::>(); inner_ext.batch_insert(top); From bd27a7f2fe4f4de40f83188542ee125f5857da45 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Thu, 27 Apr 2023 11:27:31 +0400 Subject: [PATCH 4/5] redundant batch insert --- utils/frame/remote-externalities/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 90a461db84a06..36016cfecb567 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -872,7 +872,7 @@ where ); match self.rpc_get_storage(key.clone(), Some(at)).await? { Some(value) => { - pending_ext.batch_insert(vec![(key.clone().0, value.clone().0)]); + pending_ext.insert(key.clone().0, value.clone().0); keys_and_values.push((key, value)); }, None => { From fd12b18987296349d68c1f4b4d653a5f8ac4d715 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Thu, 27 Apr 2023 11:49:59 +0400 Subject: [PATCH 5/5] avoid extra vec allocations --- primitives/state-machine/src/testing.rs | 7 +++++-- utils/frame/remote-externalities/src/lib.rs | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index de728f5bca7d5..1921287a64745 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -133,9 +133,12 @@ where } /// Batch insert key/values into backend - pub fn batch_insert(&mut self, kvs: Vec<(StorageKey, StorageValue)>) { + pub fn batch_insert(&mut self, kvs: I) + where + I: IntoIterator, + { self.backend.insert( - vec![(None, kvs.into_iter().map(|(k, v)| (k, Some(v))).collect())], + Some((None, kvs.into_iter().map(|(k, v)| (k, Some(v))).collect())), self.state_version, ); } diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index 36016cfecb567..ea9b68ce2f6ae 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -1047,7 +1047,7 @@ where "extending externalities with {} manually injected key-values", self.hashed_key_values.len() ); - ext.batch_insert(self.hashed_key_values.into_iter().map(|(k, v)| (k.0, v.0)).collect()); + ext.batch_insert(self.hashed_key_values.into_iter().map(|(k, v)| (k.0, v.0))); } // exclude manual key values.