Add CLI option to control feerate argument from memppol

- add in-top-x-mb argument to control how much to prioritize the
  transaction feerate so the transaction can get in the top x MB of the
mempool
- add functional test

node-gui/src/backend/backend_impl.rs

@@ -31,8 +31,8 @@ use wallet::{
     DefaultWallet,
 };
 use wallet_controller::{
-    read::ReadOnlyController, synced_controller::SyncedController, HandlesController, UtxoState,
-    WalletHandlesClient,
+    read::ReadOnlyController, synced_controller::SyncedController, ControllerConfig,
+    HandlesController, UtxoState, WalletHandlesClient,
 };
 use wallet_types::{seed_phrase::StoreSeedPhrase, with_locked::WithLocked};
 
@@ -49,6 +49,10 @@ use super::{
 };
 
 const TRANSACTION_LIST_PAGE_COUNT: usize = 10;
+/// In which top N MB should we aim for our transactions to be in the mempool
+/// e.g. for 5, we aim to be in the top 5 MB of transactions based on paid fees
+/// This is to avoid getting trimmed off the lower end if the mempool runs out of memory
+const IN_TOP_X_MB: usize = 5;
 
 pub type GuiController = HandlesController<GuiWalletEvents>;
 
@@ -376,7 +380,13 @@ impl Backend {
             .get_mut(&wallet_id)
             .ok_or(BackendError::UnknownWalletIndex(wallet_id))?
             .controller
-            .synced_controller(account_index)
+            // TODO: add option to select from GUI
+            .synced_controller(
+                account_index,
+                ControllerConfig {
+                    in_top_x_mb: IN_TOP_X_MB,
+                },
+            )
             .await
             .map_err(|e| BackendError::WalletError(e.to_string()))
     }

test-rpc-functions/src/rpc.rs

@@ -21,13 +21,15 @@ use common::{
     chain::{
         block::timestamp::BlockTimestamp,
         config::{regtest::GenesisStakingSettings, EpochIndex},
+        output_value::OutputValue,
+        signature::inputsig::InputWitness,
         stakelock::StakePoolData,
-        PoolId, TxOutput,
+        Destination, OutPointSourceId, PoolId, SignedTransaction, Transaction, TxInput, TxOutput,
     },
-    primitives::H256,
+    primitives::{Amount, Id, Idable, H256},
 };
 use crypto::key::Signature;
-use serialization::{hex::HexDecode, hex::HexEncode};
+use serialization::{hex::HexDecode, hex::HexEncode, hex_encoded::HexEncoded};
 
 use crate::{RpcTestFunctionsError, RpcTestFunctionsHandle};
 
@@ -94,6 +96,15 @@ trait RpcTestFunctionsRpc {
         vrf_public_key: String,
         block_timestamp: BlockTimestamp,
     ) -> rpc::Result<String>;
+
+    #[method(name = "generate_transactions")]
+    async fn generate_transactions(
+        &self,
+        input_tx_id: Id<Transaction>,
+        num_transactions: u32,
+        amount_to_spend: u64,
+        fee_per_tx: u64,
+    ) -> rpc::Result<Vec<HexEncoded<SignedTransaction>>>;
 }
 
 #[async_trait::async_trait]
@@ -269,6 +280,51 @@ impl RpcTestFunctionsRpcServer for super::RpcTestFunctionsHandle {
 
         Ok(vrf_output.hex_encode())
     }
+
+    async fn generate_transactions(
+        &self,
+        mut input_tx_id: Id<Transaction>,
+        num_transactions: u32,
+        amount_to_spend: u64,
+        fee_per_tx: u64,
+    ) -> rpc::Result<Vec<HexEncoded<SignedTransaction>>> {
+        let coin_decimals = self
+            .call(|this| this.get_chain_config().map(|chain| chain.coin_decimals()))
+            .await
+            .expect("Subsystem call ok")
+            .expect("chain config is present");
+        let coin_decimal_factor = 10u128.pow(coin_decimals as u32);
+        let mut amount_to_spend = (amount_to_spend as u128) * coin_decimal_factor;
+        let fee_per_tx = (fee_per_tx as u128) * coin_decimal_factor;
+        let mut transactions = vec![];
+        for _ in 0..num_transactions {
+            let inputs = vec![TxInput::from_utxo(OutPointSourceId::Transaction(input_tx_id), 0)];
+            let mut outputs = vec![TxOutput::Transfer(
+                OutputValue::Coin(Amount::from_atoms(amount_to_spend)),
+                Destination::AnyoneCanSpend,
+            )];
+            outputs.extend((0..9999).map(|_| {
+                TxOutput::Transfer(
+                    OutputValue::Coin(Amount::from_atoms(1)),
+                    Destination::AnyoneCanSpend,
+                )
+            }));
+
+            let transaction = SignedTransaction::new(
+                Transaction::new(0, inputs, outputs).expect("should not fail"),
+                vec![InputWitness::NoSignature(None)],
+            )
+            .expect("num signatures ok");
+
+            input_tx_id = transaction.transaction().get_id();
+            amount_to_spend -= 10000;
+            amount_to_spend -= fee_per_tx;
+
+            transactions.push(HexEncoded::new(transaction));
+        }
+
+        Ok(transactions)
+    }
 }
 
 async fn assert_genesis_values(

test/functional/test_framework/wallet_cli_controller.py

@@ -48,15 +48,16 @@ class DelegationData:
 
 class WalletCliController:
 
-    def __init__(self, node, config, log):
+    def __init__(self, node, config, log, wallet_args: List[str] = []):
         self.log = log
         self.node = node
         self.config = config
+        self.wallet_args = wallet_args
 
     async def __aenter__(self):
         wallet_cli = os.path.join(self.config["environment"]["BUILDDIR"], "test_wallet"+self.config["environment"]["EXEEXT"] )
         cookie_file = os.path.join(self.node.datadir, ".cookie")
-        wallet_args = ["--network", "regtest", "--rpc-address", self.node.url.split("@")[1], "--rpc-cookie-file", cookie_file]
+        wallet_args = ["--network", "regtest", "--rpc-address", self.node.url.split("@")[1], "--rpc-cookie-file", cookie_file] + self.wallet_args
         self.wallet_log_file = NamedTemporaryFile(prefix="wallet_stderr_", dir=os.path.dirname(self.node.datadir), delete=False)
         self.wallet_commands_file = NamedTemporaryFile(prefix="wallet_commands_responses_", dir=os.path.dirname(self.node.datadir), delete=False)
 

test/functional/test_runner.py

@@ -133,6 +133,7 @@ class UnicodeOnWindowsError(ValueError):
     'wallet_tokens.py',
     'wallet_nfts.py',
     'wallet_delegations.py',
+    'wallet_high_fee.py',
     'mempool_basic_reorg.py',
     'mempool_eviction.py',
     'mempool_ibd.py',

test/functional/wallet_delegations.py

@@ -376,6 +376,7 @@ async def async_test(self):
                 assert "The transaction was submitted successfully" in await wallet.send_to_address(acc1_address, 1)
                 transactions = node.mempool_transactions()
                 self.wait_until(lambda: node.chainstate_best_block_id() != tip_id, timeout = 5)
+                assert "Success" in await wallet.sync()
 
                 delegations = await wallet.list_delegation_ids()
                 assert len(delegations) == 1
@@ -386,6 +387,7 @@ async def async_test(self):
             assert "Success" in await wallet.select_account(DEFAULT_ACCOUNT_INDEX)
             assert "Success" in await wallet.stake_delegation(10, delegation_id)
             self.wait_until(lambda: node.chainstate_best_block_id() != tip_id, timeout = 5)
+            assert "Success" in await wallet.sync()
 
             # check that we still don't have any delagations for this account
             delegations = await wallet.list_delegation_ids()
@@ -395,6 +397,7 @@ async def async_test(self):
             delegation_id = await wallet.create_delegation(acc1_address, pools[0].pool_id)
             tip_id = node.chainstate_best_block_id()
             self.wait_until(lambda: node.chainstate_best_block_id() != tip_id, timeout = 5)
+            assert "Success" in await wallet.sync()
 
             # check that we still don't have any delagations for this account
             delegations = await wallet.list_delegation_ids()

test/functional/wallet_high_fee.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+#  Copyright (c) 2023 RBB S.r.l
+#  Copyright (c) 2017-2021 The Bitcoin Core developers
+#  opensource@mintlayer.org
+#  SPDX-License-Identifier: MIT
+#  Licensed under the MIT License;
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  https://github.com/mintlayer/mintlayer-core/blob/master/LICENSE
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""Wallet high fee submission test
+
+Check that:
+* We can create a new wallet,
+* get an address
+* send coins to the wallet's address
+* sync the wallet with the node
+* check balance
+* submit many txs with high fee
+* try to spend coins from the wallet should fail
+"""
+
+from time import time
+import scalecodec
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.mintlayer import (calc_tx_id, make_tx_dict, reward_input, tx_input, MLT_COIN, tx_output)
+from test_framework.util import assert_raises_rpc_error
+from test_framework.mintlayer import mintlayer_hash, block_input_data_obj
+from test_framework.wallet_cli_controller import WalletCliController
+
+import asyncio
+import sys
+
+class WalletSubmitTransaction(BitcoinTestFramework):
+
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 1
+        self.extra_args = [[
+            "--blockprod-min-peers-to-produce-blocks=0",
+        ]]
+
+    def setup_network(self):
+        self.setup_nodes()
+        self.sync_all(self.nodes[0:1])
+
+    def make_tx(self, inputs, outputs, flags = 0, calc_id = True):
+        self.log.info(f"making tx")
+        signed_tx = make_tx_dict(inputs, outputs, flags)
+        self.log.info(f"calc tx id")
+        tx_id = calc_tx_id(signed_tx) if calc_id else None
+        self.log.info(f"obj")
+        signed_tx_obj = scalecodec.base.RuntimeConfiguration().create_scale_object('SignedTransaction')
+        self.log.info(f"encode")
+        encoded_tx = signed_tx_obj.encode(signed_tx).to_hex()[2:]
+        return (encoded_tx, tx_id)
+
+
+    def generate_block(self):
+        node = self.nodes[0]
+
+        block_input_data = { "PoW": { "reward_destination": "AnyoneCanSpend" } }
+        block_input_data = block_input_data_obj.encode(block_input_data).to_hex()[2:]
+
+        # create a new block, taking transactions from mempool
+        block = node.blockprod_generate_block(block_input_data, None)
+        node.chainstate_submit_block(block)
+        block_id = node.chainstate_best_block_id()
+
+        # Wait for mempool to sync
+        self.wait_until(lambda: node.mempool_local_best_block_id() == block_id, timeout = 5)
+
+        return block_id
+
+    def run_test(self):
+        if 'win32' in sys.platform:
+            asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+        asyncio.run(self.async_test())
+
+    async def async_test(self):
+        node = self.nodes[0]
+        async with WalletCliController(node, self.config, self.log, ["--in-top-x-mb", "1"]) as wallet:
+            # new wallet
+            await wallet.create_wallet()
+
+            # check it is on genesis
+            best_block_height = await wallet.get_best_block_height()
+            self.log.info(f"best block height = {best_block_height}")
+            assert best_block_height == '0'
+
+            # new address
+            pub_key_bytes = await wallet.new_public_key()
+            assert len(pub_key_bytes) == 33
+
+            # Get chain tip
+            tip_id = node.chainstate_best_block_id()
+            self.log.debug(f'Tip: {tip_id}')
+
+            # Submit a valid transaction
+            output = {
+                    'Transfer': [ { 'Coin': 10 * MLT_COIN }, { 'PublicKey': {'key': {'Secp256k1Schnorr' : {'pubkey_data': pub_key_bytes}}} } ],
+            }
+            total = 300000
+            output2 = {
+                'Transfer': [ { 'Coin': total * MLT_COIN }, { 'AnyoneCanSpend': None } ],
+            }
+            encoded_tx, tx_id = self.make_tx([reward_input(tip_id)], [output2, output], 0)
+
+            self.log.debug(f"Encoded transaction {tx_id}: {encoded_tx}")
+
+            node.mempool_submit_transaction(encoded_tx)
+            assert node.mempool_contains_tx(tx_id)
+
+            block_id = self.generate_block() # Block 1
+            assert not node.mempool_contains_tx(tx_id)
+
+            # sync the wallet
+            output = await wallet.sync()
+            assert "Success" in output
+
+            # check wallet best block if it is synced
+            best_block_height = await wallet.get_best_block_height()
+            assert best_block_height == '1'
+
+            best_block_id = await wallet.get_best_block()
+            assert best_block_id == block_id
+
+            balance = await wallet.get_balance()
+            assert "Coins amount: 10" in balance
+
+            transactions = node.test_functions_generate_transactions(tx_id, 25, total - 300, 300)
+            for idx, encoded_tx in enumerate(transactions):
+                self.log.info(f"submitting tx {idx}")
+                node.mempool_submit_transaction(encoded_tx)
+
+            # try to send 9 out of 10 to itself, 1 coin should not be enough to pay the high fee
+            address = await wallet.new_address()
+            output = await wallet.send_to_address(address, 9)
+            self.log.info(output)
+            assert "successfully" not in output
+
+
+if __name__ == '__main__':
+    WalletSubmitTransaction().main()
+

wallet/wallet-cli-lib/src/cli_event_loop.rs

@@ -17,7 +17,7 @@ use std::sync::Arc;
 
 use common::chain::ChainConfig;
 use tokio::sync::{mpsc, oneshot};
-use wallet_controller::NodeRpcClient;
+use wallet_controller::{ControllerConfig, NodeRpcClient};
 
 use crate::{
     commands::{CommandHandler, ConsoleCommand, WalletCommand},
@@ -36,8 +36,9 @@ pub async fn run(
     chain_config: &Arc<ChainConfig>,
     rpc_client: &NodeRpcClient,
     mut event_rx: mpsc::UnboundedReceiver<Event>,
+    in_top_x_mb: usize,
 ) {
-    let mut command_handler = CommandHandler::new();
+    let mut command_handler = CommandHandler::new(ControllerConfig { in_top_x_mb });
 
     loop {
         let mut controller_opt = command_handler.controller_opt();

wallet/wallet-cli-lib/src/commands/mod.rs

@@ -35,8 +35,8 @@ use wallet::{
     account::Currency, version::get_version, wallet_events::WalletEventsNoOp, WalletError,
 };
 use wallet_controller::{
-    read::ReadOnlyController, synced_controller::SyncedController, ControllerError, NodeInterface,
-    NodeRpcClient, PeerId, DEFAULT_ACCOUNT_INDEX,
+    read::ReadOnlyController, synced_controller::SyncedController, ControllerConfig,
+    ControllerError, NodeInterface, NodeRpcClient, PeerId, DEFAULT_ACCOUNT_INDEX,
 };
 
 use crate::{errors::WalletCliError, CliController};
@@ -413,11 +413,15 @@ struct CliWalletState {
 pub struct CommandHandler {
     // the CliController if there is a loaded wallet
     state: Option<(CliController, CliWalletState)>,
+    config: ControllerConfig,
 }
 
 impl CommandHandler {
-    pub fn new() -> Self {
-        CommandHandler { state: None }
+    pub fn new(config: ControllerConfig) -> Self {
+        CommandHandler {
+            state: None,
+            config,
+        }
     }
 
     fn set_selected_account(&mut self, account_index: U31) -> Result<(), WalletCliError> {
@@ -474,7 +478,7 @@ impl CommandHandler {
     ) -> Result<SyncedController<'_, NodeRpcClient, WalletEventsNoOp>, WalletCliError> {
         let (controller, state) = self.state.as_mut().ok_or(WalletCliError::NoWallet)?;
         controller
-            .synced_controller(state.selected_account)
+            .synced_controller(state.selected_account, self.config)
             .await
             .map_err(WalletCliError::Controller)
     }

wallet/wallet-cli-lib/src/config.rs

@@ -76,6 +76,12 @@ pub struct WalletCliArgs {
     /// vi input mode
     #[clap(long)]
     pub vi_mode: bool,
+
+    /// In which top N MB should we aim for our transactions to be in the mempool
+    /// e.g. for 5, we aim to be in the top 5 MB of transactions based on paid fees
+    /// This is to avoid getting trimmed off the lower end if the mempool runs out of memory
+    #[arg(long, default_value_t = 5)]
+    pub in_top_x_mb: usize,
 }
 
 impl From<Network> for ChainType {