Use RPC call importmulti to import addresses

Imports of wallets are now much faster because the EC calculations
are done in optimized C code in Bitcoin Core. This does not apply
to the old-style-seed wallets, so they will still be slow.

electrumpersonalserver/server/__init__.py

@@ -21,12 +21,13 @@
 )
 from electrumpersonalserver.server.transactionmonitor import (
     TransactionMonitor,
-    import_addresses,
-    ADDRESSES_LABEL,
 )
 from electrumpersonalserver.server.deterministicwallet import (
     parse_electrum_master_public_key,
     DeterministicWallet,
+    DescriptorDeterministicWallet,
+    import_addresses,
+    ADDRESSES_LABEL,
 )
 from electrumpersonalserver.server.socks import (
     socksocket,

electrumpersonalserver/server/common.py

@@ -161,13 +161,13 @@ def get_scriptpubkeys_to_monitor(rpc, config):
     st = time.time()
     try:
         imported_addresses = set(rpc.call("getaddressesbyaccount",
-            [transactionmonitor.ADDRESSES_LABEL]))
+            [deterministicwallet.ADDRESSES_LABEL]))
         logger.debug("using deprecated accounts interface")
     except JsonRpcError:
         #bitcoin core 0.17 deprecates accounts, replaced with labels
-        if transactionmonitor.ADDRESSES_LABEL in rpc.call("listlabels", []):
+        if deterministicwallet.ADDRESSES_LABEL in rpc.call("listlabels", []):
             imported_addresses = set(rpc.call("getaddressesbylabel",
-                [transactionmonitor.ADDRESSES_LABEL]).keys())
+                [deterministicwallet.ADDRESSES_LABEL]).keys())
         else:
             #no label, no addresses imported at all
             imported_addresses = set()
@@ -188,8 +188,7 @@ def get_scriptpubkeys_to_monitor(rpc, config):
 
     #check whether these deterministic wallets have already been imported
     import_needed = False
-    wallets_imported = 0
-    addresses_to_import = []
+    wallets_to_import = []
     TEST_ADDR_COUNT = 3
     logger.info("Displaying first " + str(TEST_ADDR_COUNT) + " addresses of " +
         "each master public key:")
@@ -203,11 +202,7 @@ def get_scriptpubkeys_to_monitor(rpc, config):
             config.get("bitcoin-rpc", "initial_import_count")) - 1, count=1)
         if not set(first_addrs + last_addr).issubset(imported_addresses):
             import_needed = True
-            wallets_imported += 1
-            for change in [0, 1]:
-                addrs, spks = wal.get_addresses(change, 0,
-                    int(config.get("bitcoin-rpc", "initial_import_count")))
-                addresses_to_import.extend(addrs)
+            wallets_to_import.append(wal)
     logger.info("Obtaining bitcoin addresses to monitor . . .")
     #check whether watch-only addresses have been imported
     watch_only_addresses = []
@@ -224,18 +219,17 @@ def get_scriptpubkeys_to_monitor(rpc, config):
     if len(deterministic_wallets) == 0 and len(watch_only_addresses) == 0:
         logger.error("No master public keys or watch-only addresses have " +
             "been configured at all. Exiting..")
-        #import = true and no addresses means exit
-        return (True, [], None)
+        #import = true and none other params means exit
+        return (True, None, None)
 
     #if addresses need to be imported then return them
     if import_needed:
         #TODO minus imported_addresses
-        logger.info("Importing " + str(wallets_imported) + " wallets and " +
-            str(len(watch_only_addresses_to_import)) + " watch-only " +
-            "addresses into the Bitcoin node")
+        logger.info("Importing " + str(len(wallets_to_import))
+            + " wallets and " + str(len(watch_only_addresses_to_import))
+            + " watch-only addresses into the Bitcoin node")
         time.sleep(5)
-        return (True, addresses_to_import + list(
-            watch_only_addresses_to_import), None)
+        return True, watch_only_addresses_to_import, wallets_to_import
 
     #test
     # importing one det wallet and no addrs, two det wallets and no addrs
@@ -423,10 +417,12 @@ def main():
     import_needed, relevant_spks_addrs, deterministic_wallets = \
         get_scriptpubkeys_to_monitor(rpc, config)
     if import_needed:
-        if len(relevant_spks_addrs) == 0:
+        if not relevant_spks_addrs and not deterministic_wallets:
             #import = true and no addresses means exit
             return
-        transactionmonitor.import_addresses(rpc, relevant_spks_addrs)
+        deterministicwallet.import_addresses(rpc, relevant_spks_addrs,
+            deterministic_wallets, change_param=-1,
+            count=int(config.get("bitcoin-rpc", "initial_import_count")))
         logger.info("Done.\nIf recovering a wallet which already has existing" +
             " transactions, then\nrun the rescan script. If you're confident" +
             " that the wallets are new\nand empty then there's no need to" +

electrumpersonalserver/server/deterministicwallet.py

@@ -1,4 +1,6 @@
 
+import logging
+
 import electrumpersonalserver.bitcoin as btc
 from electrumpersonalserver.server.hashes import bh2u, hash_160, bfh, sha256,\
     address_to_script, script_to_address
@@ -9,6 +11,48 @@
 #and
 #https://github.com/spesmilo/electrum-docs/blob/master/xpub_version_bytes.rst
 
+ADDRESSES_LABEL = "electrum-watchonly-addresses"
+
+def import_addresses(rpc, watchonly_addrs, wallets, change_param, count,
+        logger=None):
+    """
+    change_param = 0 for receive, 1 for change, -1 for both
+    """
+    logger = logger if logger else logging.getLogger('ELECTRUMPERSONALSERVER')
+    logger.debug("Importing " + str(len(watchonly_addrs)) + " watch-only "
+        + "address[es] and " + str(len(wallets)) + " wallet[s] into label \""
+        + ADDRESSES_LABEL + "\"")
+
+    watchonly_addr_param = [{"scriptPubKey": {"address": addr}, "label":
+        ADDRESSES_LABEL, "watchonly": True, "timestamp": "now"}
+        for addr in watchonly_addrs]
+    rpc.call("importmulti", [watchonly_addr_param, {"rescan": False}])
+
+    for i, wal in enumerate(wallets):
+        logger.info("Importing wallet " + str(i+1) + "/" + str(len(wallets)))
+        if isinstance(wal, DescriptorDeterministicWallet):
+            if change_param in (0, -1):
+                #import receive addrs
+                rpc.call("importmulti", [[{"desc": wal.descriptors[0], "range":
+                    [0, count-1], "label": ADDRESSES_LABEL, "watchonly": True,
+                    "timestamp": "now"}], {"rescan": False}])
+            if change_param in (1, -1):
+                #import change addrs
+                rpc.call("importmulti", [[{"desc": wal.descriptors[1], "range":
+                    [0, count-1], "label": ADDRESSES_LABEL, "watchonly": True,
+                    "timestamp": "now"}], {"rescan": False}])
+        else:
+            #old-style-seed wallets
+            logger.info("importing an old-style-seed wallet, will be slow...")
+            for change in [0, 1]:
+                addrs, spks = wal.get_addresses(change, 0, count)
+                addr_param = [{"scriptPubKey": {"address": a}, "label":
+                    ADDRESSES_LABEL, "watchonly": True, "timestamp": "now"}
+                    for a in addrs]
+                rpc.call("importmulti", [addr_param, {"rescan": False}])
+    logger.debug("Importing done")
+
+
 def is_string_parsable_as_hex_int(s):
     try:
         int(s, 16)

electrumpersonalserver/server/transactionmonitor.py

@@ -13,47 +13,20 @@
     script_to_scripthash,
     script_to_address
 )
+from electrumpersonalserver.server.deterministicwallet import import_addresses
 
 #internally this code uses scriptPubKeys, it only converts to bitcoin addresses
 # when importing to bitcoind or checking whether enough addresses have been
 # imported
 #the electrum protocol uses sha256(scriptpubkey) as a key for lookups
 # this code calls them scripthashes
 
-#code will generate the first address from each deterministic wallet
-# and check whether they have been imported into the bitcoin node
-# if no then initial_import_count addresses will be imported, then exit
-# if yes then initial_import_count addresses will be generated and extra
-# addresses will be generated one-by-one, each time checking whether they have
-# been imported into the bitcoin node
-# when an address has been reached that has not been imported, that means
-# we've reached the end, then rewind the deterministic wallet index by one
-
 #when a transaction happens paying to an address from a deterministic wallet
 # lookup the position of that address, if its less than gap_limit then
 # import more addresses
 
-ADDRESSES_LABEL = "electrum-watchonly-addresses"
 CONFIRMATIONS_SAFE_FROM_REORG = 100
 
-def import_addresses(rpc, addrs, logger=None):
-    logger = logger if logger else logging.getLogger('ELECTRUMPERSONALSERVER')
-    logger.debug("importing addrs = " + str(addrs))
-    logger.debug("into label \"" + ADDRESSES_LABEL + "\"")
-    logger.info("Importing " + str(len(addrs)) + " addresses in total")
-    addr_i = iter(addrs)
-    notifications = 20
-    for i in range(notifications):
-        pc = int(100.0 * i / notifications)
-        sys.stdout.write("[" + str(pc) + "%]... ")
-        sys.stdout.flush()
-        for j in range(int(len(addrs) / notifications)):
-            rpc.call("importaddress", [next(addr_i), ADDRESSES_LABEL, False])
-    for a in addr_i: #import the reminder of addresses
-        rpc.call("importaddress", [a, ADDRESSES_LABEL, False])
-    print("[100%]")
-    logger.info("Importing done")
-
 class TransactionMonitor(object):
     """
     Class which monitors the bitcoind wallet for new transactions
@@ -507,7 +480,7 @@ def check_for_new_txes(self):
                                 spk)] =  {'history': [], 'subscribed': False}
                         logger.debug("importing " + str(len(spks)) +
                             " into change=" + str(change))
-                        import_addresses(self.rpc, new_addrs, logger)
+                        import_addresses(self.rpc, new_addrs, [], -1, 0, logger)
 
             updated_scripthashes.extend(matching_scripthashes)
             new_history_element = self.generate_new_history_element(tx, txd)