Handle losing connection to bitcoin node

Previously if the json-rpc connection to the bitcoin node was lost then
the server would crash. Now it will close the Electrum connection and
refuse all new connections until it reestablishes a link to the node.
Electrum will then display a red dot as an indication that something is
wrong, and so the server operator can be reminded to restart the node.

Also, the json-rpc functions will no longer cache the username and
password values obtained from the cookie file. Then if the node is
restarted and generates a new cookie then the server will correctly
use the new authentication information.

electrumpersonalserver/server/common.py

@@ -8,6 +8,7 @@
 import tempfile
 import platform
 import json
+import traceback
 from json.decoder import JSONDecodeError
 from configparser import RawConfigParser, NoSectionError, NoOptionError
 from ipaddress import ip_network, ip_address
@@ -30,14 +31,20 @@
 
 def on_heartbeat_listening(txmonitor):
     logger = logging.getLogger('ELECTRUMPERSONALSERVER')
-    txmonitor.check_for_updated_txes()
+    try:
+        txmonitor.check_for_updated_txes()
+        is_node_reachable = True
+    except JsonRpcError:
+        is_node_reachable = False
+    return is_node_reachable
 
 def on_heartbeat_connected(rpc, txmonitor, protocol):
     logger = logging.getLogger('ELECTRUMPERSONALSERVER')
     is_tip_updated, header = check_for_new_blockchain_tip(rpc,
         protocol.are_headers_raw)
     if is_tip_updated:
-        logger.debug("Blockchain tip updated")
+        logger.debug("Blockchain tip updated " + (str(header["height"]) if
+            "height" in header else ""))
         protocol.on_blockchain_tip_updated(header)
     updated_scripthashes = txmonitor.check_for_updated_txes()
     protocol.on_updated_scripthashes(updated_scripthashes)
@@ -90,35 +97,44 @@ def run_electrum_server(rpc, txmonitor, config):
 
     server_sock = create_server_socket(hostport)
     server_sock.settimeout(poll_interval_listening)
+    accepting_clients = True
     while True:
-        try:
-            sock = None
-            while sock == None:
-                try:
-                    sock, addr = server_sock.accept()
-                    if not any([ip_address(addr[0]) in ipnet
-                            for ipnet in ip_whitelist]):
-                        logger.debug(addr[0] + " not in whitelist, closing")
-                        raise ConnectionRefusedError()
-                    sock = ssl.wrap_socket(sock, server_side=True,
-                        certfile=certfile, keyfile=keyfile,
-                        ssl_version=ssl.PROTOCOL_SSLv23)
-                except socket.timeout:
-                    on_heartbeat_listening(txmonitor)
-                except (ConnectionRefusedError, ssl.SSLError):
-                    sock.close()
-                    sock = None
-            logger.debug('Electrum connected from ' + str(addr[0]))
-
-            def send_reply_fun(reply):
-                line = json.dumps(reply)
-                sock.sendall(line.encode('utf-8') + b'\n')
-                logger.debug('<= ' + line)
-            protocol.set_send_reply_fun(send_reply_fun)
+        # main server loop, runs forever
+        sock = None
+        while sock == None:
+            # loop waiting for a successful connection from client
+            try:
+                sock, addr = server_sock.accept()
+                if not accepting_clients:
+                    logger.debug("Refusing connection from client because"
+                        + " Bitcoin node isnt reachable")
+                    raise ConnectionRefusedError()
+                if not any([ip_address(addr[0]) in ipnet
+                        for ipnet in ip_whitelist]):
+                    logger.debug(addr[0] + " not in whitelist, closing")
+                    raise ConnectionRefusedError()
+                sock = ssl.wrap_socket(sock, server_side=True,
+                    certfile=certfile, keyfile=keyfile,
+                    ssl_version=ssl.PROTOCOL_SSLv23)
+            except socket.timeout:
+                is_node_reachable = on_heartbeat_listening(txmonitor)
+                accepting_clients = is_node_reachable
+            except (ConnectionRefusedError, ssl.SSLError):
+                sock.close()
+                sock = None
+        logger.debug('Electrum connected from ' + str(addr[0]))
+
+        def send_reply_fun(reply):
+            line = json.dumps(reply)
+            sock.sendall(line.encode('utf-8') + b'\n')
+            logger.debug('<= ' + line)
+        protocol.set_send_reply_fun(send_reply_fun)
 
+        try:
             sock.settimeout(poll_interval_connected)
             recv_buffer = bytearray()
             while True:
+                # loop for replying to client queries
                 try:
                     recv_data = sock.recv(4096)
                     if not recv_data or len(recv_data) == 0:
@@ -140,6 +156,10 @@ def send_reply_fun(reply):
                         protocol.handle_query(query)
                 except socket.timeout:
                     on_heartbeat_connected(rpc, txmonitor, protocol)
+        except JsonRpcError as e:
+            logger.debug("Error with node connection, e = " + repr(e)
+                + "\ntraceback = " + str(traceback.format_exc()))
+            accepting_clients = False
         except (IOError, EOFError) as e:
             if isinstance(e, (EOFError, ConnectionRefusedError)):
                 logger.debug("Electrum wallet disconnected")
@@ -150,9 +170,8 @@ def send_reply_fun(reply):
                     sock.close()
             except IOError:
                 pass
-            sock = None
-            protocol.on_disconnect()
-            time.sleep(0.2)
+        protocol.on_disconnect()
+        time.sleep(0.2)
 
 def get_scriptpubkeys_to_monitor(rpc, config):
     logger = logging.getLogger('ELECTRUMPERSONALSERVER')
@@ -274,7 +293,7 @@ def get_certs(config):
             raise ValueError('invalid cert: {}, key: {}'.format(
                 certfile, keyfile))
 
-def obtain_rpc_username_password(datadir):
+def obtain_cookie_file_path(datadir):
     logger = logging.getLogger('ELECTRUMPERSONALSERVER')
     if len(datadir.strip()) == 0:
         logger.debug("no datadir configuration, checking in default location")
@@ -291,11 +310,8 @@ def obtain_rpc_username_password(datadir):
     if not os.path.exists(cookie_path):
         logger.warning("Unable to find .cookie file, try setting `datadir`" +
             " config")
-        return None, None
-    fd = open(cookie_path)
-    username, password = fd.read().strip().split(":")
-    fd.close()
-    return username, password
+        return None
+    return cookie_path
 
 def parse_args():
     from argparse import ArgumentParser
@@ -349,19 +365,22 @@ def main():
         SERVER_VERSION_NUMBER))
     logger.info('Logging to ' + logfilename)
     logger.debug("Process ID (PID) = " + str(os.getpid()))
+    rpc_u = None
+    rpc_p = None
+    cookie_path = None
     try:
         rpc_u = config.get("bitcoin-rpc", "rpc_user")
         rpc_p = config.get("bitcoin-rpc", "rpc_password")
         logger.debug("obtaining auth from rpc_user/pass")
     except NoOptionError:
-        rpc_u, rpc_p = obtain_rpc_username_password(config.get(
+        cookie_path = obtain_cookie_file_path(config.get(
             "bitcoin-rpc", "datadir"))
         logger.debug("obtaining auth from .cookie")
-    if rpc_u == None:
+    if rpc_u == None and cookie_path == None:
         return
     rpc = JsonRpc(host = config.get("bitcoin-rpc", "host"),
         port = int(config.get("bitcoin-rpc", "port")),
-        user = rpc_u, password = rpc_p,
+        user = rpc_u, password = rpc_p, cookie_path = cookie_path,
         wallet_filename=config.get("bitcoin-rpc", "wallet_filename").strip(),
         logger=logger)
 

electrumpersonalserver/server/jsonrpc.py

@@ -15,19 +15,34 @@ class JsonRpc(object):
     Simple implementation of a JSON-RPC client that is used
     to connect to Bitcoin.
     """
-    def __init__(self, host, port, user, password, wallet_filename="",
-            logger=None):
+    def __init__(self, host, port, user, password, cookie_path=None,
+            wallet_filename="", logger=None):
         self.host = host
         self.port = port
+
+        self.cookie_path = cookie_path
+        if cookie_path:
+            self.load_from_cookie()
+        else:
+            self.create_authstr(user, password)
+
         self.conn = http.client.HTTPConnection(self.host, self.port)
-        self.authstr = "%s:%s" % (user, password)
         if len(wallet_filename) > 0:
             self.url = "/wallet/" + wallet_filename
         else:
             self.url = ""
         self.logger = logger
         self.queryId = 1
 
+    def create_authstr(self, username, password):
+        self.authstr = "%s:%s" % (username, password)
+
+    def load_from_cookie(self):
+        fd = open(self.cookie_path)
+        username, password = fd.read().strip().split(":")
+        fd.close()
+        self.create_authstr(username, password)
+
     def queryHTTP(self, obj):
         """
         Send an appropriate HTTP query to the server.  The JSON-RPC
@@ -41,14 +56,22 @@ def queryHTTP(self, obj):
         headers["Authorization"] = (b"Basic " +
             base64.b64encode(self.authstr.encode('utf-8')))
         body = json.dumps(obj)
+        auth_failed_once = False
         for i in range(20):
             try:
                 self.conn.request("POST", self.url, body, headers)
                 response = self.conn.getresponse()
                 if response.status == 401:
-                    self.conn.close()
-                    raise JsonRpcConnectionError(
-                            "authentication for JSON-RPC failed")
+                    if self.cookie_path == None or auth_failed_once:
+                        self.conn.close()
+                        raise JsonRpcConnectionError(
+                                "authentication for JSON-RPC failed")
+                    else:
+                        auth_failed_once = True
+                        #try reloading u/p from the cookie file once
+                        self.load_from_cookie()
+                        raise OSError() #jump to error handler below
+                auth_failed_once = False
                 #All the codes below are 'fine' from a JSON-RPC point of view.
                 if response.status not in [200, 404, 500]:
                     self.conn.close()
@@ -59,11 +82,14 @@ def queryHTTP(self, obj):
                 raise exc
             except http.client.BadStatusLine:
                 return "CONNFAILURE"
-            except OSError as e:
-                    self.logger.debug('Reconnecting RPC after dropped ' +
-                        'connection: ' + repr(e))
-                    self.conn.close()
-                    self.conn.connect()
+            except OSError:
+                    # connection dropped, reconnect
+                    try:
+                        self.conn.close()
+                        self.conn.connect()
+                    except ConnectionError as e:
+                        #node probably offline, notify with jsonrpc error
+                        raise JsonRpcConnectionError(repr(e))
                     continue
             except Exception as exc:
                 raise JsonRpcConnectionError("JSON-RPC connection failed. Err:"