From d737acd9ecdeff80fc62712fa19c571ace76097e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Zemanovi=C4=8D?= Date: Mon, 15 Aug 2022 17:51:04 +0200 Subject: [PATCH 1/4] storage_api: build a nicer `iter_prefix` function on top of StorageRead --- shared/src/ledger/storage_api/mod.rs | 66 ++++++++++++++++++++++++++++ tx_prelude/src/lib.rs | 4 +- vp_prelude/src/lib.rs | 10 +++-- 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/shared/src/ledger/storage_api/mod.rs b/shared/src/ledger/storage_api/mod.rs index 0e7e299970..ab2f73784f 100644 --- a/shared/src/ledger/storage_api/mod.rs +++ b/shared/src/ledger/storage_api/mod.rs @@ -96,3 +96,69 @@ pub trait StorageWrite { /// Delete a value at the given key from storage. fn delete(&mut self, key: &storage::Key) -> Result<()>; } + +/// Iterate items matching the given prefix. +pub fn iter_prefix_bytes<'a>( + storage: &'a impl StorageRead<'a>, + prefix: &crate::types::storage::Key, +) -> Result)>> + 'a> { + let iter = storage.iter_prefix(prefix)?; + let iter = itertools::unfold(iter, |iter| { + match storage.iter_next(iter) { + Ok(Some((key, val))) => { + let key = match storage::Key::parse(key).into_storage_result() { + Ok(key) => key, + Err(err) => { + // Propagate key encoding errors into Iterator's Item + return Some(Err(err)); + } + }; + Some(Ok((key, val))) + } + Ok(None) => None, + Err(err) => { + // Propagate `iter_next` errors into Iterator's Item + Some(Err(err)) + } + } + }); + Ok(iter) +} + +/// Iterate Borsh encoded items matching the given prefix. +pub fn iter_prefix<'a, T>( + storage: &'a impl StorageRead<'a>, + prefix: &crate::types::storage::Key, +) -> Result> + 'a> +where + T: BorshDeserialize, +{ + let iter = storage.iter_prefix(prefix)?; + let iter = itertools::unfold(iter, |iter| { + match storage.iter_next(iter) { + Ok(Some((key, val))) => { + let key = match storage::Key::parse(key).into_storage_result() { + Ok(key) => key, + Err(err) => { + // Propagate key encoding errors into Iterator's Item + return Some(Err(err)); + } + }; + let val = match T::try_from_slice(&val).into_storage_result() { + Ok(val) => val, + Err(err) => { + // Propagate val encoding errors into Iterator's Item + return Some(Err(err)); + } + }; + Some(Ok((key, val))) + } + Ok(None) => None, + Err(err) => { + // Propagate `iter_next` errors into Iterator's Item + Some(Err(err)) + } + } + }); + Ok(iter) +} diff --git a/tx_prelude/src/lib.rs b/tx_prelude/src/lib.rs index 02341e70e5..bd1833ec58 100644 --- a/tx_prelude/src/lib.rs +++ b/tx_prelude/src/lib.rs @@ -23,7 +23,9 @@ pub use namada::ledger::governance::storage as gov_storage; pub use namada::ledger::parameters::storage as parameters_storage; pub use namada::ledger::storage::types::encode; use namada::ledger::storage_api; -pub use namada::ledger::storage_api::{StorageRead, StorageWrite}; +pub use namada::ledger::storage_api::{ + iter_prefix, iter_prefix_bytes, StorageRead, StorageWrite, +}; pub use namada::ledger::treasury::storage as treasury_storage; pub use namada::ledger::tx_env::TxEnv; pub use namada::proto::{Signed, SignedTxData}; diff --git a/vp_prelude/src/lib.rs b/vp_prelude/src/lib.rs index 24d702b6b5..3ee761f78f 100644 --- a/vp_prelude/src/lib.rs +++ b/vp_prelude/src/lib.rs @@ -22,7 +22,9 @@ use std::marker::PhantomData; pub use borsh::{BorshDeserialize, BorshSerialize}; pub use error::*; pub use namada::ledger::governance::storage as gov_storage; -pub use namada::ledger::storage_api::{self, StorageRead}; +pub use namada::ledger::storage_api::{ + self, iter_prefix, iter_prefix_bytes, StorageRead, +}; pub use namada::ledger::vp_env::VpEnv; pub use namada::ledger::{parameters, pos as proof_of_stake}; pub use namada::proto::{Signed, SignedTxData}; @@ -334,7 +336,7 @@ impl StorageRead<'_> for CtxPreStorageRead<'_> { prefix: &storage::Key, ) -> Result { // Note that this is the same as `CtxPostStorageRead` - iter_prefix(prefix) + iter_prefix_impl(prefix) } fn iter_next( @@ -411,7 +413,7 @@ impl StorageRead<'_> for CtxPostStorageRead<'_> { prefix: &storage::Key, ) -> Result { // Note that this is the same as `CtxPreStorageRead` - iter_prefix(prefix) + iter_prefix_impl(prefix) } fn iter_next( @@ -442,7 +444,7 @@ impl StorageRead<'_> for CtxPostStorageRead<'_> { } } -fn iter_prefix( +fn iter_prefix_impl( prefix: &storage::Key, ) -> Result)>, storage_api::Error> { let prefix = prefix.to_string(); From 4558dfa42248a9282bb6d4b5a492ac22bb3aa040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Zemanovi=C4=8D?= Date: Mon, 15 Aug 2022 17:59:08 +0200 Subject: [PATCH 2/4] test/vm_host_env: refactor prefix iter tests with new `iter_prefix` fn --- tests/src/vm_host_env/mod.rs | 51 ++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/tests/src/vm_host_env/mod.rs b/tests/src/vm_host_env/mod.rs index 37d1afb790..98f95e7fae 100644 --- a/tests/src/vm_host_env/mod.rs +++ b/tests/src/vm_host_env/mod.rs @@ -148,9 +148,11 @@ mod tests { tx_host_env::init(); let empty_key = storage::Key::parse("empty").unwrap(); - let mut iter = tx::ctx().iter_prefix(&empty_key).unwrap(); + let mut iter = + namada_tx_prelude::iter_prefix_bytes(tx::ctx(), &empty_key) + .unwrap(); assert!( - tx::ctx().iter_next(&mut iter).unwrap().is_none(), + iter.next().is_none(), "Trying to iter a prefix that doesn't have any matching keys \ should yield an empty iterator." ); @@ -167,15 +169,12 @@ mod tests { }); // Then try to iterate over their prefix - let iter = tx::ctx().iter_prefix(&prefix).unwrap(); - let iter = itertools::unfold(iter, |iter| { - if let Ok(Some((key, value))) = tx::ctx().iter_next(iter) { - let decoded_value = i32::try_from_slice(&value[..]).unwrap(); - return Some((key, decoded_value)); - } - None + let iter = namada_tx_prelude::iter_prefix(tx::ctx(), &prefix) + .unwrap() + .map(|item| item.unwrap()); + let expected = (0..10).map(|i| { + (storage::Key::parse(format!("{}/{}", prefix, i)).unwrap(), i) }); - let expected = (0..10).map(|i| (format!("{}/{}", prefix, i), i)); itertools::assert_equal(iter.sorted(), expected.sorted()); } @@ -380,29 +379,25 @@ mod tests { tx::ctx().write(&new_key, 11_i32).unwrap(); }); - let iter_pre = vp::CTX.iter_prefix(&prefix).unwrap(); - let iter_pre = itertools::unfold(iter_pre, |iter| { - if let Ok(Some((key, value))) = vp::CTX.iter_pre_next(iter) { - if let Ok(decoded_value) = i32::try_from_slice(&value[..]) { - return Some((key, decoded_value)); - } - } - None + let ctx_pre = vp::CTX.pre(); + let iter_pre = namada_vp_prelude::iter_prefix(&ctx_pre, &prefix) + .unwrap() + .map(|item| item.unwrap()); + let expected_pre = (0..10).map(|i| { + (storage::Key::parse(format!("{}/{}", prefix, i)).unwrap(), i) }); - let expected_pre = (0..10).map(|i| (format!("{}/{}", prefix, i), i)); itertools::assert_equal(iter_pre.sorted(), expected_pre.sorted()); - let iter_post = vp::CTX.iter_prefix(&prefix).unwrap(); - let iter_post = itertools::unfold(iter_post, |iter| { - if let Ok(Some((key, value))) = vp::CTX.iter_post_next(iter) { - let decoded_value = i32::try_from_slice(&value[..]).unwrap(); - return Some((key, decoded_value)); - } - None - }); + let ctx_post = vp::CTX.post(); + let iter_post = namada_vp_prelude::iter_prefix(&ctx_post, &prefix) + .unwrap() + .map(|item| item.unwrap()); let expected_post = (0..10).map(|i| { let val = if i == 5 { 100 } else { i }; - (format!("{}/{}", prefix, i), val) + ( + storage::Key::parse(format!("{}/{}", prefix, i)).unwrap(), + val, + ) }); itertools::assert_equal(iter_post.sorted(), expected_post.sorted()); } From d32d7af363124d52d41cd7f4a0758fa0fb88d97e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 15 Aug 2022 16:34:57 +0000 Subject: [PATCH 3/4] wasm checksums update --- wasm/checksums.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/wasm/checksums.json b/wasm/checksums.json index 5c658485f0..0c7b7cf504 100644 --- a/wasm/checksums.json +++ b/wasm/checksums.json @@ -1,19 +1,19 @@ { - "tx_bond.wasm": "tx_bond.98a51e759f7b41873faee295fa69ee919d64ee95e4aa5cd923daff59325fda94.wasm", - "tx_from_intent.wasm": "tx_from_intent.c598df3ee8cc5980f83df56b1acd8132c85421d360a4f54c1235c8576f0d3390.wasm", - "tx_ibc.wasm": "tx_ibc.c838d61c5ad5b5f24855f19a85fda1bad159701b90a4672e98db6d8fee966880.wasm", - "tx_init_account.wasm": "tx_init_account.02a6a717930a35962e5d2fddd76bb78194796379d7c584645ce8c30b1e672ae6.wasm", - "tx_init_nft.wasm": "tx_init_nft.09974068959257551e82281839f4506fcc377713d174984c25af37455e08ac84.wasm", - "tx_init_proposal.wasm": "tx_init_proposal.a9a45a5a4ec4fffe2e28d8bec8c013dc571e62d01b730eea2d09fc12a3c700dd.wasm", - "tx_init_validator.wasm": "tx_init_validator.03d30dad39d35d4285ff012af80ad3e94f8d10d802410aac4a7fd1c86e8d3f6c.wasm", - "tx_mint_nft.wasm": "tx_mint_nft.32747cf99afd40b14114812bea65198d454eb9d7c86d6300a8ed9306df9aa9be.wasm", - "tx_transfer.wasm": "tx_transfer.05c24ab9ad8bdd09bc910d9bbfc8686fc5c436594b635cc2105f0ddde5c8ab63.wasm", - "tx_unbond.wasm": "tx_unbond.a3ffc49d7b481e1d43c7f67cbccf9ce31476f98a9822b6b57ddf51c33f427605.wasm", + "tx_bond.wasm": "tx_bond.d80c5ac518c223eea92a51eeb033462e9ea4498530d12de5b6e6ca6b3fa59ea8.wasm", + "tx_from_intent.wasm": "tx_from_intent.49565974c6e2c3d64a05729bd57217b244d804fc9f4e5369d1f3515aeaa8397f.wasm", + "tx_ibc.wasm": "tx_ibc.0d9d639037a8dc54c53ecbedc8396ea103ee7c8f485790ddf7223f4e7e6a9779.wasm", + "tx_init_account.wasm": "tx_init_account.97bfee0b78c87abc217c58351f1d6242990a06c7379b27f948950f04f36b49a2.wasm", + "tx_init_nft.wasm": "tx_init_nft.6207eabda37cd356b6093d069f388bd84463b5e1b8860811a36a9b63da84951f.wasm", + "tx_init_proposal.wasm": "tx_init_proposal.1073d69f69172276c61c104f7b4678019723df19ddb30aedf6293a00de973846.wasm", + "tx_init_validator.wasm": "tx_init_validator.40f0152c1bd59f46ec26123d98d0b49a0a458335b6012818cf8504b7708bf625.wasm", + "tx_mint_nft.wasm": "tx_mint_nft.f1c8039a6fb5e01e7441cfa2e3fb2f84a1cb60ed660745ddc172d0d01f1b0ab1.wasm", + "tx_transfer.wasm": "tx_transfer.191d28c340900e6dc297e66b054cd83b690ae0357a8e13b37962b265b2e17da8.wasm", + "tx_unbond.wasm": "tx_unbond.ea73369f68abef405c4f7a3a09c3da6aa68493108d48b1e4e243d26766f00283.wasm", "tx_update_vp.wasm": "tx_update_vp.a304b3c70361cde5fda458ba5637d6825eb002198e73990a1c74351113b83d43.wasm", "tx_vote_proposal.wasm": "tx_vote_proposal.5e51a66746df8c7e786cc6837a74f87d239442ace67d1c4ef4e6751aa725514f.wasm", - "tx_withdraw.wasm": "tx_withdraw.efab1590d68edee54058ddf03313697be4aaa8adf8b012f98338be514e9f6e87.wasm", + "tx_withdraw.wasm": "tx_withdraw.f776265133f972e6705797561b6bb37f9e21de07f3611b23cfdd6e39cb252c0f.wasm", "vp_nft.wasm": "vp_nft.1a32d37b32c616119d156128560a0eeb43206e91069e2c3875e74211ed3ad01f.wasm", - "vp_testnet_faucet.wasm": "vp_testnet_faucet.4c91dfdf6ac6cdccc931ee391167033a9cdbd1b8551a428e90295bd0d6f68d85.wasm", + "vp_testnet_faucet.wasm": "vp_testnet_faucet.c30baa6d2dab2c336a6b2485e3ccf41cd548b135ca5245b80fab15858c70365c.wasm", "vp_token.wasm": "vp_token.9f27c2e823691487bb6631fbfecb39837c1f404ef9b263e810a5a2a76f4c5957.wasm", - "vp_user.wasm": "vp_user.fb06cb724aa92d615b8da971b0ca1c89bf0caf8c1b9ea3536c252fd1c3b98881.wasm" + "vp_user.wasm": "vp_user.59b7a9262c951ff451d3aec0604ae3dd0fc4729f8d994c114be6320ce5d38712.wasm" } \ No newline at end of file From fec72205d4417afa358121674d2110ffb81c2a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Zemanovi=C4=8D?= Date: Mon, 22 Aug 2022 13:27:11 +0200 Subject: [PATCH 4/4] changelog: add #335 --- .../improvements/335-refactor-storage-prefix-iter.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/unreleased/improvements/335-refactor-storage-prefix-iter.md diff --git a/.changelog/unreleased/improvements/335-refactor-storage-prefix-iter.md b/.changelog/unreleased/improvements/335-refactor-storage-prefix-iter.md new file mode 100644 index 0000000000..d51f6c72f0 --- /dev/null +++ b/.changelog/unreleased/improvements/335-refactor-storage-prefix-iter.md @@ -0,0 +1,3 @@ +- Added a simpler prefix iterator API that returns `std::iter::Iterator` with + the storage keys parsed and a variant that also decodes stored values with + Borsh ([#335](https://github.com/anoma/namada/pull/335)) \ No newline at end of file