From cbf69c6a601edee2e69d31c92733ec1f3bb31e86 Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Thu, 28 Nov 2019 22:15:55 +0000 Subject: [PATCH 1/2] Implement no-history synchronization No-history is a method for synchronizing a wallet by scanning the UTXO set. It can be useful for checking whether seed phrase backups have money on them before committing the time and effort required to rescanning the blockchain. No-history sync is compatible with pruning. The sync method cannot tell which empty addresses have been used, so cannot guarentee avoidance of address reuse. For this reason no-history sync disables wallet address generation and can only be used with wallet-tool and for sending transactions without change addresses. --- .../release-notes-nohistory-sync.md | 12 ++ jmclient/jmclient/blockchaininterface.py | 119 ++++++++++++++---- jmclient/jmclient/configure.py | 14 ++- jmclient/jmclient/wallet.py | 9 +- jmclient/jmclient/wallet_service.py | 84 ++++++++----- jmclient/jmclient/wallet_utils.py | 36 +++--- 6 files changed, 200 insertions(+), 74 deletions(-) create mode 100644 docs/release-notes/release-notes-nohistory-sync.md diff --git a/docs/release-notes/release-notes-nohistory-sync.md b/docs/release-notes/release-notes-nohistory-sync.md new file mode 100644 index 000000000..101dd3736 --- /dev/null +++ b/docs/release-notes/release-notes-nohistory-sync.md @@ -0,0 +1,12 @@ +Notable changes +=============== + +### No-history wallet synchronization + +The no-history synchronization method is enabled by setting `blockchain_source = bitcoin-rpc-no-history` in the `joinmarket.cfg` file. + +The method can be used to import a seed phrase to see whether it has any money on it within just 5-10 minutes. No-history sync doesn't require a long blockchain rescan, although it needs a full node which can be pruned. + +No-history sync works by scanning the full node's UTXO set. The downside is that it cannot find the history but only the current unspent balance, so it cannot avoid address reuse. Therefore when using no-history synchronization the wallet cannot generate new addresses. Any found money can only be spent by fully-sweeping the funds but not partially spending them which requires a change address. When using the method make sure to increase the gap limit to a large amount to cover all the possible bitcoin addresses where coins might be. + +The mode does not work with the Joinmarket-Qt GUI application but might do in future. diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index 62a55e1db..b4aa34633 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -5,6 +5,7 @@ import abc import random import sys +import time from decimal import Decimal from twisted.internet import reactor, task @@ -12,7 +13,7 @@ from jmclient.jsonrpc import JsonRpcConnectionError, JsonRpcError from jmclient.configure import jm_single -from jmbase.support import get_log, jmprint, EXIT_SUCCESS, EXIT_FAILURE +from jmbase.support import get_log, jmprint, EXIT_FAILURE # an inaccessible blockheight; consider rewriting in 1900 years @@ -57,6 +58,11 @@ def estimate_fee_per_kb(self, N): required for inclusion in the next N blocks. ''' + @abc.abstractmethod + def import_addresses_if_needed(self, addresses, wallet_name): + """import addresses to the underlying blockchain interface if needed + returns True if the sync call needs to do a system exit""" + def fee_per_kb_has_been_manually_set(self, N): '''if the 'block' target is higher than 1000, interpret it as manually set fee/Kb. @@ -66,7 +72,6 @@ def fee_per_kb_has_been_manually_set(self, N): else: return False - class ElectrumWalletInterface(BlockchainInterface): #pragma: no cover """A pseudo-blockchain interface using the existing Electrum server connection in an Electrum wallet. @@ -167,7 +172,9 @@ def __init__(self, jsonRpc, network): actualNet = blockchainInfo['chain'] netmap = {'main': 'mainnet', 'test': 'testnet', 'regtest': 'regtest'} - if netmap[actualNet] != network: + if netmap[actualNet] != network and \ + (not (actualNet == "regtest" and network == "testnet")): + #special case of regtest and testnet having the same addr format raise Exception('wrong network configured') def get_block(self, blockheight): @@ -182,7 +189,8 @@ def get_block(self, blockheight): def rpc(self, method, args): if method not in ['importaddress', 'walletpassphrase', 'getaccount', 'gettransaction', 'getrawtransaction', 'gettxout', - 'importmulti', 'listtransactions', 'getblockcount']: + 'importmulti', 'listtransactions', 'getblockcount', + 'scantxoutset']: log.debug('rpc: ' + method + " " + str(args)) res = self.jsonRpc.call(method, args) return res @@ -230,25 +238,20 @@ def import_addresses(self, addr_list, wallet_name, restart_cb=None): jmprint(fatal_msg, "important") sys.exit(EXIT_FAILURE) - def add_watchonly_addresses(self, addr_list, wallet_name, restart_cb=None): - """For backwards compatibility, this fn name is preserved - as the case where we quit the program if a rescan is required; - but in some cases a rescan is not required (if the address is known - to be new/unused). For that case use import_addresses instead. - """ - self.import_addresses(addr_list, wallet_name) - if jm_single().config.get("BLOCKCHAIN", - "blockchain_source") != 'regtest': #pragma: no cover - #Exit conditions cannot be included in tests - restart_msg = ("restart Bitcoin Core with -rescan or use " - "`bitcoin-cli rescanblockchain` if you're " - "recovering an existing wallet from backup seed\n" - "Otherwise just restart this joinmarket application.") - if restart_cb: - restart_cb(restart_msg) + def import_addresses_if_needed(self, addresses, wallet_name): + try: + imported_addresses = set(self.rpc('getaddressesbyaccount', + [wallet_name])) + except JsonRpcError: + if wallet_name in self.rpc('listlabels', []): + imported_addresses = set(self.rpc('getaddressesbylabel', + [wallet_name]).keys()) else: - jmprint(restart_msg, "important") - sys.exit(EXIT_SUCCESS) + imported_addresses = set() + import_needed = not addresses.issubset(imported_addresses) + if import_needed: + self.import_addresses(addresses - imported_addresses, wallet_name) + return import_needed def _yield_transactions(self, wallet_name): batch_size = 1000 @@ -387,6 +390,78 @@ def estimate_fee_per_kb(self, N): return 10000 return int(Decimal(1e8) * Decimal(estimate)) +class BitcoinCoreNoHistoryInterface(BitcoinCoreInterface): + + def __init__(self, jsonRpc, network): + super(BitcoinCoreNoHistoryInterface, self).__init__(jsonRpc, network) + self.import_addresses_call_count = 0 + self.wallet_name = None + self.scan_result = None + + def import_addresses_if_needed(self, addresses, wallet_name): + self.import_addresses_call_count += 1 + if self.import_addresses_call_count == 1: + self.wallet_name = wallet_name + addr_list = ["addr(" + a + ")" for a in addresses] + log.debug("Starting scan of UTXO set") + st = time.time() + try: + self.rpc("scantxoutset", ["abort", []]) + self.scan_result = self.rpc("scantxoutset", ["start", + addr_list]) + except JsonRpcError as e: + raise RuntimeError("Bitcoin Core 0.17.0 or higher required " + + "for no-history sync (" + repr(e) + ")") + et = time.time() + log.debug("UTXO set scan took " + str(et - st) + "sec") + elif self.import_addresses_call_count > 4: + #called twice for the first call of sync_addresses(), then two + # more times for the second call. the second call happens because + # sync_addresses() re-runs in order to have gap_limit new addresses + assert False + return False + + def _get_addr_from_desc(self, desc_str): + #example + #'desc': 'addr(2MvAfRVvRAeBS18NT7mKVc1gFim169GkFC5)#h5yn9eq4', + assert desc_str.startswith("addr(") + return desc_str[5:desc_str.find(")")] + + def _yield_transactions(self, wallet_name): + for u in self.scan_result["unspents"]: + tx = {"category": "receive", "address": + self._get_addr_from_desc(u["desc"])} + yield tx + + def list_transactions(self, num): + return [] + + def rpc(self, method, args): + if method == "listaddressgroupings": + raise RuntimeError("default sync not supported by bitcoin-rpc-nohistory, use --recoversync") + elif method == "listunspent": + minconf = 0 if len(args) < 1 else args[0] + maxconf = 9999999 if len(args) < 2 else args[1] + return [{ + "address": self._get_addr_from_desc(u["desc"]), + "label": self.wallet_name, + "height": u["height"], + "txid": u["txid"], + "vout": u["vout"], + "scriptPubKey": u["scriptPubKey"], + "amount": u["amount"] + } for u in self.scan_result["unspents"]] + else: + return super(BitcoinCoreNoHistoryInterface, self).rpc(method, args) + + def set_wallet_no_history(self, wallet): + #make wallet-tool not display any new addresses + #because no-history cant tell if an address is used and empty + #so this is necessary to avoid address reuse + wallet.gap_limit = 0 + #disable generating change addresses, also because cant guarantee + # avoidance of address reuse + wallet.disable_new_scripts = True # class for regtest chain access # running on local daemon. Only diff --git a/jmclient/jmclient/configure.py b/jmclient/jmclient/configure.py index 9f1f9f8f6..eb345ac4f 100644 --- a/jmclient/jmclient/configure.py +++ b/jmclient/jmclient/configure.py @@ -107,7 +107,8 @@ def jm_single(): use_ssl = false [BLOCKCHAIN] -#options: bitcoin-rpc, regtest +#options: bitcoin-rpc, regtest, bitcoin-rpc-no-history +# when using bitcoin-rpc-no-history remember to increase the gap limit to scan for more addresses, try -g 5000 blockchain_source = bitcoin-rpc network = mainnet rpc_host = localhost @@ -516,12 +517,13 @@ def get_blockchain_interface_instance(_config): # todo: refactor joinmarket module to get rid of loops # importing here is necessary to avoid import loops from jmclient.blockchaininterface import BitcoinCoreInterface, \ - RegtestBitcoinCoreInterface, ElectrumWalletInterface + RegtestBitcoinCoreInterface, ElectrumWalletInterface, \ + BitcoinCoreNoHistoryInterface from jmclient.electruminterface import ElectrumInterface source = _config.get("BLOCKCHAIN", "blockchain_source") network = get_network() testnet = network == 'testnet' - if source in ('bitcoin-rpc', 'regtest'): + if source in ('bitcoin-rpc', 'regtest', 'bitcoin-rpc-no-history'): rpc_host = _config.get("BLOCKCHAIN", "rpc_host") rpc_port = _config.get("BLOCKCHAIN", "rpc_port") rpc_user, rpc_password = get_bitcoin_rpc_credentials(_config) @@ -530,8 +532,12 @@ def get_blockchain_interface_instance(_config): rpc_wallet_file) if source == 'bitcoin-rpc': #pragma: no cover bc_interface = BitcoinCoreInterface(rpc, network) - else: + elif source == 'regtest': bc_interface = RegtestBitcoinCoreInterface(rpc) + elif source == "bitcoin-rpc-no-history": + bc_interface = BitcoinCoreNoHistoryInterface(rpc, network) + else: + assert 0 elif source == 'electrum': bc_interface = ElectrumWalletInterface(testnet) elif source == 'electrum-server': diff --git a/jmclient/jmclient/wallet.py b/jmclient/jmclient/wallet.py index 9dd201a3b..6dbfcae87 100644 --- a/jmclient/jmclient/wallet.py +++ b/jmclient/jmclient/wallet.py @@ -1283,6 +1283,7 @@ def __init__(self, storage, **kwargs): self.get_bip32_priv_export(0, 0).encode('ascii')).digest())\ .digest()[:3] self._populate_script_map() + self.disable_new_scripts = False @classmethod def initialize(cls, storage, network, max_mixdepth=2, timestamp=None, @@ -1372,7 +1373,7 @@ def get_script_path(self, path): current_index = self._index_cache[md][int_type] if index == current_index: - return self.get_new_script(md, int_type) + return self.get_new_script_override_disable(md, int_type) priv, engine = self._get_priv_from_path(path) script = engine.privkey_to_script(priv) @@ -1454,6 +1455,12 @@ def _is_my_bip32_path(self, path): return path[0] == self._key_ident def get_new_script(self, mixdepth, internal): + if self.disable_new_scripts: + raise RuntimeError("Obtaining new wallet addresses " + + "disabled, due to nohistory mode") + return self.get_new_script_override_disable(mixdepth, internal) + + def get_new_script_override_disable(self, mixdepth, internal): # This is called by get_script_path and calls back there. We need to # ensure all conditions match to avoid endless recursion. int_type = self._get_internal_type(internal) diff --git a/jmclient/jmclient/wallet_service.py b/jmclient/jmclient/wallet_service.py index 3667a1b74..6be395b83 100644 --- a/jmclient/jmclient/wallet_service.py +++ b/jmclient/jmclient/wallet_service.py @@ -6,6 +6,7 @@ import time import ast import binascii +import sys from decimal import Decimal from copy import deepcopy from twisted.internet import reactor @@ -14,8 +15,9 @@ from numbers import Integral from jmclient.configure import jm_single, get_log from jmclient.output import fmt_tx_data -from jmclient.jsonrpc import JsonRpcError -from jmclient.blockchaininterface import INF_HEIGHT +from jmclient.blockchaininterface import (INF_HEIGHT, BitcoinCoreInterface, + BitcoinCoreNoHistoryInterface) +from jmbase.support import jmprint, EXIT_SUCCESS """Wallet service The purpose of this independent service is to allow @@ -333,6 +335,8 @@ def sync_wallet(self, fast=True): # before startup self.old_txs = [x['txid'] for x in self.bci.list_transactions(100) if "txid" in x] + if isinstance(self.bci, BitcoinCoreNoHistoryInterface): + self.bci.set_wallet_no_history(self.wallet) return self.synced def resync_wallet(self, fast=True): @@ -447,6 +451,24 @@ def get_address_usages(self): self.restart_callback) self.synced = True + def display_rescan_message_and_system_exit(self, restart_cb): + #TODO using system exit here should be avoided as it makes the code + # harder to understand and reason about + #theres also a sys.exit() in BitcoinCoreInterface.import_addresses() + #perhaps have sys.exit() placed inside the restart_cb that only + # CLI scripts will use + if self.bci.__class__ == BitcoinCoreInterface: + #Exit conditions cannot be included in tests + restart_msg = ("restart Bitcoin Core with -rescan or use " + "`bitcoin-cli rescanblockchain` if you're " + "recovering an existing wallet from backup seed\n" + "Otherwise just restart this joinmarket application.") + if restart_cb: + restart_cb(restart_msg) + else: + jmprint(restart_msg, "important") + sys.exit(EXIT_SUCCESS) + def sync_addresses(self): """ Triggered by use of --recoversync option in scripts, attempts a full scan of the blockchain without assuming @@ -456,19 +478,11 @@ def sync_addresses(self): jlog.debug("requesting detailed wallet history") wallet_name = self.get_wallet_name() addresses, saved_indices = self.collect_addresses_init() - try: - imported_addresses = set(self.bci.rpc('getaddressesbyaccount', - [wallet_name])) - except JsonRpcError: - if wallet_name in self.bci.rpc('listlabels', []): - imported_addresses = set(self.bci.rpc('getaddressesbylabel', - [wallet_name]).keys()) - else: - imported_addresses = set() - if not addresses.issubset(imported_addresses): - self.bci.add_watchonly_addresses(addresses - imported_addresses, - wallet_name, self.restart_callback) + import_needed = self.bci.import_addresses_if_needed(addresses, + wallet_name) + if import_needed: + self.display_rescan_message_and_system_exit(self.restart_callback) return used_addresses_gen = (tx['address'] @@ -480,13 +494,12 @@ def sync_addresses(self): self.rewind_wallet_indices(used_indices, saved_indices) new_addresses = self.collect_addresses_gap() - if not new_addresses.issubset(imported_addresses): - jlog.debug("Syncing iteration finished, additional step required") - self.bci.add_watchonly_addresses(new_addresses - imported_addresses, - wallet_name, self.restart_callback) + if self.bci.import_addresses_if_needed(new_addresses, wallet_name): + jlog.debug("Syncing iteration finished, additional step required (more address import required)") self.synced = False + self.display_rescan_message_and_system_exit(self.restart_callback) elif gap_limit_used: - jlog.debug("Syncing iteration finished, additional step required") + jlog.debug("Syncing iteration finished, additional step required (gap limit used)") self.synced = False else: jlog.debug("Wallet successfully synced") @@ -513,14 +526,30 @@ def sync_unspent(self): our_unspent_list = [x for x in unspent_list if ( self.bci.is_address_labeled(x, wallet_name) or self.bci.is_address_labeled(x, self.EXTERNAL_WALLET_LABEL))] - for u in our_unspent_list: - if not self.is_known_addr(u['address']): + for utxo in our_unspent_list: + if not self.is_known_addr(utxo['address']): continue - self._add_unspent_utxo(u, current_blockheight) + # The result of bitcoin core's listunspent RPC call does not have + # a "height" field, only "confirmations". + # But the result of scantxoutset used in no-history sync does + # have "height". + if "height" in utxo: + height = utxo["height"] + else: + height = None + # wallet's utxo database needs to store an absolute rather + # than relative height measure: + confs = int(utxo['confirmations']) + if confs < 0: + jlog.warning("Utxo not added, has a conflict: " + str(utxo)) + continue + if confs >= 1: + height = current_blockheight - confs + 1 + self._add_unspent_txo(utxo, height) et = time.time() jlog.debug('bitcoind sync_unspent took ' + str((et - st)) + 'sec') - def _add_unspent_utxo(self, utxo, current_blockheight): + def _add_unspent_txo(self, utxo, height): """ Add a UTXO as returned by rpc's listunspent call to the wallet. @@ -532,15 +561,6 @@ def _add_unspent_utxo(self, utxo, current_blockheight): txid = binascii.unhexlify(utxo['txid']) script = binascii.unhexlify(utxo['scriptPubKey']) value = int(Decimal(str(utxo['amount'])) * Decimal('1e8')) - confs = int(utxo['confirmations']) - # wallet's utxo database needs to store an absolute rather - # than relative height measure: - height = None - if confs < 0: - jlog.warning("Utxo not added, has a conflict: " + str(utxo)) - return - if confs >=1 : - height = current_blockheight - confs + 1 self.add_utxo(txid, int(utxo['vout']), script, value, height) diff --git a/jmclient/jmclient/wallet_utils.py b/jmclient/jmclient/wallet_utils.py index 9d5536b2d..6b7b0e23c 100644 --- a/jmclient/jmclient/wallet_utils.py +++ b/jmclient/jmclient/wallet_utils.py @@ -386,7 +386,7 @@ def wallet_showutxos(wallet, showprivkey): return json.dumps(unsp, indent=4) -def wallet_display(wallet_service, gaplimit, showprivkey, displayall=False, +def wallet_display(wallet_service, showprivkey, displayall=False, serialized=True, summarized=False): """build the walletview object, then return its serialization directly if serialized, @@ -399,16 +399,22 @@ def get_addr_status(addr_path, utxos, is_new, is_internal): if addr_path != utxodata['path']: continue addr_balance += utxodata['value'] - is_coinjoin, cj_amount, cj_n = \ - get_tx_info(binascii.hexlify(utxo[0]).decode('ascii'))[:3] - if is_coinjoin and utxodata['value'] == cj_amount: - status.append('cj-out') - elif is_coinjoin: - status.append('change-out') - elif is_internal: - status.append('non-cj-change') - else: - status.append('deposit') + #TODO it is a failure of abstraction here that + # the bitcoin core interface is used directly + #the function should either be removed or added to bci + #or possibly add some kind of `gettransaction` function + # to bci + if jm_single().bc_interface.__class__ == BitcoinCoreInterface: + is_coinjoin, cj_amount, cj_n = \ + get_tx_info(binascii.hexlify(utxo[0]).decode('ascii'))[:3] + if is_coinjoin and utxodata['value'] == cj_amount: + status.append('cj-out') + elif is_coinjoin: + status.append('change-out') + elif is_internal: + status.append('non-cj-change') + else: + status.append('deposit') out_status = 'new' if is_new else 'used' if len(status) > 1: @@ -433,7 +439,7 @@ def get_addr_status(addr_path, utxos, is_new, is_internal): xpub_key = "" unused_index = wallet_service.get_next_unused_index(m, forchange) - for k in range(unused_index + gaplimit): + for k in range(unused_index + wallet_service.gap_limit): path = wallet_service.get_path(m, forchange, k) addr = wallet_service.get_addr_path(path) balance, used = get_addr_status( @@ -1254,12 +1260,12 @@ def wallet_tool_main(wallet_root_path): #Now the wallet/data is prepared, execute the script according to the method if method == "display": - return wallet_display(wallet_service, options.gaplimit, options.showprivkey) + return wallet_display(wallet_service, options.showprivkey) elif method == "displayall": - return wallet_display(wallet_service, options.gaplimit, options.showprivkey, + return wallet_display(wallet_service, options.showprivkey, displayall=True) elif method == "summary": - return wallet_display(wallet_service, options.gaplimit, options.showprivkey, summarized=True) + return wallet_display(wallet_service, options.showprivkey, summarized=True) elif method == "history": if not isinstance(jm_single().bc_interface, BitcoinCoreInterface): jmprint('showing history only available when using the Bitcoin Core ' + From f5e27c39dc90d739a0a079141fe8078b32fa23cb Mon Sep 17 00:00:00 2001 From: chris-belcher Date: Mon, 11 Nov 2019 08:18:51 +0000 Subject: [PATCH 2/2] Add test code for no-history sync --- jmclient/jmclient/blockchaininterface.py | 11 +++++ jmclient/test/test_core_nohistory_sync.py | 51 +++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 jmclient/test/test_core_nohistory_sync.py diff --git a/jmclient/jmclient/blockchaininterface.py b/jmclient/jmclient/blockchaininterface.py index b4aa34633..483d92476 100644 --- a/jmclient/jmclient/blockchaininterface.py +++ b/jmclient/jmclient/blockchaininterface.py @@ -463,6 +463,17 @@ def set_wallet_no_history(self, wallet): # avoidance of address reuse wallet.disable_new_scripts = True + ##these two functions are hacks to make the test code be able to use the + ##same helper functions, perhaps it would be nicer to create mixin classes + ##and use multiple inheritance to make the code more OOP, but its not + ##worth it now + def grab_coins(self, receiving_addr, amt=50): + RegtestBitcoinCoreInterface.grab_coins(self, receiving_addr, amt) + + def tick_forward_chain(self, n): + self.destn_addr = self.rpc("getnewaddress", []) + RegtestBitcoinCoreInterface.tick_forward_chain(self, n) + # class for regtest chain access # running on local daemon. Only # to be instantiated after network is up diff --git a/jmclient/test/test_core_nohistory_sync.py b/jmclient/test/test_core_nohistory_sync.py new file mode 100644 index 000000000..8498b9dc5 --- /dev/null +++ b/jmclient/test/test_core_nohistory_sync.py @@ -0,0 +1,51 @@ +#! /usr/bin/env python +from __future__ import (absolute_import, division, + print_function, unicode_literals) +from builtins import * # noqa: F401 +'''Wallet functionality tests.''' + +"""BitcoinCoreNoHistoryInterface functionality tests.""" + +from commontest import create_wallet_for_sync + +import pytest +from jmbase import get_log +from jmclient import load_program_config + +log = get_log() + +def test_fast_sync_unavailable(setup_sync): + load_program_config(bs="bitcoin-rpc-no-history") + wallet_service = create_wallet_for_sync([0, 0, 0, 0, 0], + ['test_fast_sync_unavailable']) + with pytest.raises(RuntimeError) as e_info: + wallet_service.sync_wallet(fast=True) + +@pytest.mark.parametrize('internal', (False, True)) +def test_sync(setup_sync, internal): + load_program_config(bs="bitcoin-rpc-no-history") + used_count = [1, 3, 6, 2, 23] + wallet_service = create_wallet_for_sync(used_count, ['test_sync'], + populate_internal=internal) + ##the gap limit should be not zero before sync + assert wallet_service.gap_limit > 0 + for md in range(len(used_count)): + ##obtaining an address should be possible without error before sync + wallet_service.get_new_script(md, internal) + + wallet_service.sync_wallet(fast=False) + + for md in range(len(used_count)): + ##plus one to take into account the one new script obtained above + assert used_count[md] + 1 == wallet_service.get_next_unused_index(md, + internal) + #gap limit is zero after sync + assert wallet_service.gap_limit == 0 + #obtaining an address leads to an error after sync + with pytest.raises(RuntimeError) as e_info: + wallet_service.get_new_script(0, internal) + + +@pytest.fixture(scope='module') +def setup_sync(): + pass