Wallet no history sync

docs/release-notes/release-notes-nohistory-sync.md

@@ -0,0 +1,12 @@
+Notable changes
+===============
+
+### No-history wallet synchronization
+
+The no-history synchronization method is enabled by setting `blockchain_source = bitcoin-rpc-no-history` in the `joinmarket.cfg` file.
+
+The method can be used to import a seed phrase to see whether it has any money on it within just 5-10 minutes. No-history sync doesn't require a long blockchain rescan, although it needs a full node which can be pruned.
+
+No-history sync works by scanning the full node's UTXO set. The downside is that it cannot find the history but only the current unspent balance, so it cannot avoid address reuse. Therefore when using no-history synchronization the wallet cannot generate new addresses. Any found money can only be spent by fully-sweeping the funds but not partially spending them which requires a change address. When using the method make sure to increase the gap limit to a large amount to cover all the possible bitcoin addresses where coins might be.
+
+The mode does not work with the Joinmarket-Qt GUI application but might do in future.

jmclient/jmclient/blockchaininterface.py

@@ -5,14 +5,15 @@
 import abc
 import random
 import sys
+import time
 from decimal import Decimal
 from twisted.internet import reactor, task
 
 import jmbitcoin as btc
 
 from jmclient.jsonrpc import JsonRpcConnectionError, JsonRpcError
 from jmclient.configure import jm_single
-from jmbase.support import get_log, jmprint, EXIT_SUCCESS, EXIT_FAILURE
+from jmbase.support import get_log, jmprint, EXIT_FAILURE
 
 
 # an inaccessible blockheight; consider rewriting in 1900 years
@@ -57,6 +58,11 @@ def estimate_fee_per_kb(self, N):
         required for inclusion in the next N blocks.
 	'''
 
+    @abc.abstractmethod
+    def import_addresses_if_needed(self, addresses, wallet_name):
+        """import addresses to the underlying blockchain interface if needed
+        returns True if the sync call needs to do a system exit"""
+
     def fee_per_kb_has_been_manually_set(self, N):
         '''if the 'block' target is higher than 1000, interpret it
         as manually set fee/Kb.
@@ -66,7 +72,6 @@ def fee_per_kb_has_been_manually_set(self, N):
         else:
             return False
 
-
 class ElectrumWalletInterface(BlockchainInterface): #pragma: no cover
     """A pseudo-blockchain interface using the existing 
     Electrum server connection in an Electrum wallet.
@@ -167,7 +172,9 @@ def __init__(self, jsonRpc, network):
         actualNet = blockchainInfo['chain']
 
         netmap = {'main': 'mainnet', 'test': 'testnet', 'regtest': 'regtest'}
-        if netmap[actualNet] != network:
+        if netmap[actualNet] != network and \
+                (not (actualNet == "regtest" and network == "testnet")):
+            #special case of regtest and testnet having the same addr format
             raise Exception('wrong network configured')
 
     def get_block(self, blockheight):
@@ -182,7 +189,8 @@ def get_block(self, blockheight):
     def rpc(self, method, args):
         if method not in ['importaddress', 'walletpassphrase', 'getaccount',
                           'gettransaction', 'getrawtransaction', 'gettxout',
-                          'importmulti', 'listtransactions', 'getblockcount']:
+                          'importmulti', 'listtransactions', 'getblockcount',
+                          'scantxoutset']:
             log.debug('rpc: ' + method + " " + str(args))
         res = self.jsonRpc.call(method, args)
         return res
@@ -230,25 +238,20 @@ def import_addresses(self, addr_list, wallet_name, restart_cb=None):
                 jmprint(fatal_msg, "important")
             sys.exit(EXIT_FAILURE)
 
-    def add_watchonly_addresses(self, addr_list, wallet_name, restart_cb=None):
-        """For backwards compatibility, this fn name is preserved
-        as the case where we quit the program if a rescan is required;
-        but in some cases a rescan is not required (if the address is known
-        to be new/unused). For that case use import_addresses instead.
-        """
-        self.import_addresses(addr_list, wallet_name)
-        if jm_single().config.get("BLOCKCHAIN",
-                                  "blockchain_source") != 'regtest': #pragma: no cover
-            #Exit conditions cannot be included in tests
-            restart_msg = ("restart Bitcoin Core with -rescan or use "
-                           "`bitcoin-cli rescanblockchain` if you're "
-                           "recovering an existing wallet from backup seed\n"
-                           "Otherwise just restart this joinmarket application.")
-            if restart_cb:
-                restart_cb(restart_msg)
+    def import_addresses_if_needed(self, addresses, wallet_name):
+        try:
+            imported_addresses = set(self.rpc('getaddressesbyaccount',
+                                                  [wallet_name]))
+        except JsonRpcError:
+            if wallet_name in self.rpc('listlabels', []):
+                imported_addresses = set(self.rpc('getaddressesbylabel',
+                                                      [wallet_name]).keys())
             else:
-                jmprint(restart_msg, "important")
-                sys.exit(EXIT_SUCCESS)
+                imported_addresses = set()
+        import_needed = not addresses.issubset(imported_addresses)
+        if import_needed:
+            self.import_addresses(addresses - imported_addresses, wallet_name)
+        return import_needed
 
     def _yield_transactions(self, wallet_name):
         batch_size = 1000
@@ -387,6 +390,89 @@ def estimate_fee_per_kb(self, N):
             return 10000
         return int(Decimal(1e8) * Decimal(estimate))
 
+class BitcoinCoreNoHistoryInterface(BitcoinCoreInterface):
+
+    def __init__(self, jsonRpc, network):
+        super(BitcoinCoreNoHistoryInterface, self).__init__(jsonRpc, network)
+        self.import_addresses_call_count = 0
+        self.wallet_name = None
+        self.scan_result = None
+
+    def import_addresses_if_needed(self, addresses, wallet_name):
+        self.import_addresses_call_count += 1
+        if self.import_addresses_call_count == 1:
+            self.wallet_name = wallet_name
+            addr_list = ["addr(" + a + ")" for a in addresses]
+            log.debug("Starting scan of UTXO set")
+            st = time.time()
+            try:
+                self.rpc("scantxoutset", ["abort", []])
+                self.scan_result = self.rpc("scantxoutset", ["start",
+                    addr_list])
+            except JsonRpcError as e:
+                raise RuntimeError("Bitcoin Core 0.17.0 or higher required "
+                    + "for no-history sync (" + repr(e) + ")")
+            et = time.time()
+            log.debug("UTXO set scan took " + str(et - st) + "sec")
+        elif self.import_addresses_call_count > 4:
+            #called twice for the first call of sync_addresses(), then two
+            # more times for the second call. the second call happens because
+            # sync_addresses() re-runs in order to have gap_limit new addresses
+            assert False
+        return False
+
+    def _get_addr_from_desc(self, desc_str):
+        #example
+        #'desc': 'addr(2MvAfRVvRAeBS18NT7mKVc1gFim169GkFC5)#h5yn9eq4',
+        assert desc_str.startswith("addr(")
+        return desc_str[5:desc_str.find(")")]
+
+    def _yield_transactions(self, wallet_name):
+        for u in self.scan_result["unspents"]:
+            tx = {"category": "receive", "address":
+                self._get_addr_from_desc(u["desc"])}
+            yield tx
+
+    def list_transactions(self, num):
+        return []
+
+    def rpc(self, method, args):
+        if method == "listaddressgroupings":
+            raise RuntimeError("default sync not supported by bitcoin-rpc-nohistory, use --recoversync")
+        elif method == "listunspent":
+            minconf = 0 if len(args) < 1 else args[0]
+            maxconf = 9999999 if len(args) < 2 else args[1]
+            return [{
+                "address": self._get_addr_from_desc(u["desc"]),
+                "label": self.wallet_name,
+                "height": u["height"],
+                "txid": u["txid"],
+                "vout": u["vout"],
+                "scriptPubKey": u["scriptPubKey"],
+                "amount": u["amount"]
+                } for u in self.scan_result["unspents"]]
+        else:
+            return super(BitcoinCoreNoHistoryInterface, self).rpc(method, args)
+
+    def set_wallet_no_history(self, wallet):
+        #make wallet-tool not display any new addresses
+        #because no-history cant tell if an address is used and empty
+        #so this is necessary to avoid address reuse
+        wallet.gap_limit = 0
+        #disable generating change addresses, also because cant guarantee
+        # avoidance of address reuse
+        wallet.disable_new_scripts = True
+
+    ##these two functions are hacks to make the test code be able to use the
+    ##same helper functions, perhaps it would be nicer to create mixin classes
+    ##and use multiple inheritance to make the code more OOP, but its not
+    ##worth it now
+    def grab_coins(self, receiving_addr, amt=50):
+        RegtestBitcoinCoreInterface.grab_coins(self, receiving_addr, amt)
+
+    def tick_forward_chain(self, n):
+        self.destn_addr = self.rpc("getnewaddress", [])
+        RegtestBitcoinCoreInterface.tick_forward_chain(self, n)
 
 # class for regtest chain access
 # running on local daemon. Only

jmclient/jmclient/configure.py

@@ -107,7 +107,8 @@ def jm_single():
 use_ssl = false
 
 [BLOCKCHAIN]
-#options: bitcoin-rpc, regtest
+#options: bitcoin-rpc, regtest, bitcoin-rpc-no-history
+# when using bitcoin-rpc-no-history remember to increase the gap limit to scan for more addresses, try -g 5000
 blockchain_source = bitcoin-rpc
 network = mainnet
 rpc_host = localhost
@@ -516,12 +517,13 @@ def get_blockchain_interface_instance(_config):
     # todo: refactor joinmarket module to get rid of loops
     # importing here is necessary to avoid import loops
     from jmclient.blockchaininterface import BitcoinCoreInterface, \
-        RegtestBitcoinCoreInterface, ElectrumWalletInterface
+        RegtestBitcoinCoreInterface, ElectrumWalletInterface, \
+        BitcoinCoreNoHistoryInterface
     from jmclient.electruminterface import ElectrumInterface
     source = _config.get("BLOCKCHAIN", "blockchain_source")
     network = get_network()
     testnet = network == 'testnet'
-    if source in ('bitcoin-rpc', 'regtest'):
+    if source in ('bitcoin-rpc', 'regtest', 'bitcoin-rpc-no-history'):
         rpc_host = _config.get("BLOCKCHAIN", "rpc_host")
         rpc_port = _config.get("BLOCKCHAIN", "rpc_port")
         rpc_user, rpc_password = get_bitcoin_rpc_credentials(_config)
@@ -530,8 +532,12 @@ def get_blockchain_interface_instance(_config):
             rpc_wallet_file)
         if source == 'bitcoin-rpc': #pragma: no cover
             bc_interface = BitcoinCoreInterface(rpc, network)
-        else:
+        elif source == 'regtest':
             bc_interface = RegtestBitcoinCoreInterface(rpc)
+        elif source == "bitcoin-rpc-no-history":
+            bc_interface = BitcoinCoreNoHistoryInterface(rpc, network)
+        else:
+            assert 0
     elif source == 'electrum':
         bc_interface = ElectrumWalletInterface(testnet)
     elif source == 'electrum-server':

jmclient/jmclient/wallet.py

@@ -1283,6 +1283,7 @@ def __init__(self, storage, **kwargs):
             self.get_bip32_priv_export(0, 0).encode('ascii')).digest())\
             .digest()[:3]
         self._populate_script_map()
+        self.disable_new_scripts = False
 
     @classmethod
     def initialize(cls, storage, network, max_mixdepth=2, timestamp=None,
@@ -1372,7 +1373,7 @@ def get_script_path(self, path):
         current_index = self._index_cache[md][int_type]
 
         if index == current_index:
-            return self.get_new_script(md, int_type)
+            return self.get_new_script_override_disable(md, int_type)
 
         priv, engine = self._get_priv_from_path(path)
         script = engine.privkey_to_script(priv)
@@ -1454,6 +1455,12 @@ def _is_my_bip32_path(self, path):
         return path[0] == self._key_ident
 
     def get_new_script(self, mixdepth, internal):
+        if self.disable_new_scripts:
+            raise RuntimeError("Obtaining new wallet addresses "
+                + "disabled, due to nohistory mode")
+        return self.get_new_script_override_disable(mixdepth, internal)
+
+    def get_new_script_override_disable(self, mixdepth, internal):
         # This is called by get_script_path and calls back there. We need to
         # ensure all conditions match to avoid endless recursion.
         int_type = self._get_internal_type(internal)