Auto merge of zcash#2176 - str4d:1593-libevent, r=bitcartel

libevent-based http server Cherry-picked from the following upstream PRs: - bitcoin/bitcoin#5677 - bitcoin/bitcoin#6695 - bitcoin/bitcoin#6899 - bitcoin/bitcoin#7016 - bitcoin/bitcoin#7964 - bitcoin/bitcoin#8722 - bitcoin/bitcoin#8730 - bitcoin/bitcoin#9073 - bitcoin/bitcoin#9265 - bitcoin/bitcoin#9387 - bitcoin/bitcoin#9471 - bitcoin/bitcoin#9647 - bitcoin/bitcoin#9903 - bitcoin/bitcoin#6640 - bitcoin/bitcoin#8139 - bitcoin/bitcoin#8839 Closes zcash#1593 and zcash#1856.
wip-alex · Mar 25, 2017 · f9f4866 · f9f4866
2 parents 2bdb372 + 3da13e8
commit f9f4866
Show file tree

Hide file tree

Showing 34 changed files with 1,656 additions and 1,167 deletions.
diff --git a/configure.ac b/configure.ac
@@ -687,6 +687,12 @@ if test x$use_pkgconfig = xyes; then
     [
       PKG_CHECK_MODULES([SSL], [libssl],, [AC_MSG_ERROR(openssl  not found.)])
       PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto  not found.)])
+      if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then
+        PKG_CHECK_MODULES([EVENT], [libevent],, [AC_MSG_ERROR(libevent not found.)])
+        if test x$TARGET_OS != xwindows; then
+          PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads],, [AC_MSG_ERROR(libevent_pthreads not found.)])
+        fi
+      fi
 
       if test "x$use_zmq" = "xyes"; then
         PKG_CHECK_MODULES([ZMQ],[libzmq >= 4],
@@ -709,6 +715,14 @@ else
   AC_CHECK_HEADER([openssl/ssl.h],, AC_MSG_ERROR(libssl headers missing),)
   AC_CHECK_LIB([ssl],         [main],SSL_LIBS=-lssl, AC_MSG_ERROR(libssl missing))
 
+  if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then
+    AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),)
+    AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing))
+    if test x$TARGET_OS != xwindows; then
+      AC_CHECK_LIB([event_pthreads],[main],EVENT_PTHREADS_LIBS=-levent_pthreads,AC_MSG_ERROR(libevent_pthreads missing))
+    fi
+  fi
+
   if test "x$use_zmq" = "xyes"; then
      AC_CHECK_HEADER([zmq.h],
        [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])],

diff --git a/contrib/bitcoind.bash-completion b/contrib/bitcoind.bash-completion
@@ -16,7 +16,7 @@ _zcashd() {
     _get_comp_words_by_ref -n = cur prev words cword
 
     case "$cur" in
-        -conf=*|-pid=*|-loadblock=*|-rpccookiefile=*|-wallet=*|-rpcsslcertificatechainfile=*|-rpcsslprivatekeyfile=*)
+        -conf=*|-pid=*|-loadblock=*|-rpccookiefile=*|-wallet=*)
             cur="${cur#*=}"
             _filedir
             return 0

diff --git a/contrib/debian/examples/zcash.conf b/contrib/debian/examples/zcash.conf
@@ -73,7 +73,7 @@
 
 # How many seconds zcash will wait for a complete RPC HTTP request.
 # after the HTTP connection is established. 
-#rpctimeout=30
+#rpcclienttimeout=30
 
 # By default, only RPC connections from localhost are allowed.
 # Specify as many rpcallowip= settings as you like to allow connections from other hosts,
@@ -95,15 +95,6 @@
 # running on another host using this option:
 #rpcconnect=127.0.0.1
 
-# Use Secure Sockets Layer (also known as TLS or HTTPS) to communicate
-# with Zcash -server or zcashd
-#rpcssl=1
-
-# OpenSSL settings used when rpcssl=1
-#rpcsslciphers=TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!AH:!3DES:@STRENGTH
-#rpcsslcertificatechainfile=server.cert
-#rpcsslprivatekeyfile=server.pem
-
 # Transaction Fee
 
 # Send transactions as zero-fee transactions if possible (default: 0)

diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk
@@ -0,0 +1,31 @@
+package=libevent
+$(package)_version=2.1.8
+$(package)_download_path=https://github.com/libevent/libevent/archive/
+$(package)_file_name=$(package)-$($(package)_version).tar.gz
+$(package)_download_file=release-$($(package)_version)-stable.tar.gz
+$(package)_sha256_hash=316ddb401745ac5d222d7c529ef1eada12f58f6376a66c1118eee803cb70f83d
+
+define $(package)_preprocess_cmds
+  ./autogen.sh
+endef
+
+define $(package)_set_vars
+  $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress
+  $(package)_config_opts_release=--disable-debug-mode
+  $(package)_config_opts_linux=--with-pic
+endef
+
+define $(package)_config_cmds
+  $($(package)_autoconf)
+endef
+
+define $(package)_build_cmds
+  $(MAKE)
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install
+endef
+
+define $(package)_postprocess_cmds
+endef
diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk
@@ -1,6 +1,6 @@
 rust_packages := rust librustzcash
 zcash_packages := libsnark libgmp libsodium
-packages := boost openssl zeromq $(zcash_packages) googletest googlemock
+packages := boost openssl libevent zeromq $(zcash_packages) googletest googlemock
 native_packages := native_ccache
 
 wallet_packages=bdb

diff --git a/doc/man/zcash-cli.1 b/doc/man/zcash-cli.1
@@ -57,12 +57,6 @@ Username for JSON\-RPC connections
 \fB\-rpcpassword=\fR<pw>
 .IP
 Password for JSON\-RPC connections
-.PP
-SSL options: (see the Bitcoin Wiki for SSL setup instructions)
-.HP
-\fB\-rpcssl\fR
-.IP
-Use OpenSSL (https) for JSON\-RPC connections
 .SH COPYRIGHT
 Copyright (C) 2009-2017 The Bitcoin Core Developers
 Copyright (C) 2015-2017 The Zcash Developers

diff --git a/doc/man/zcashd.1 b/doc/man/zcashd.1
@@ -422,25 +422,6 @@ multiple times
 .IP
 Set the number of threads to service RPC calls (default: 4)
 .PP
-RPC SSL options: (see the Bitcoin Wiki for SSL setup instructions)
-.HP
-\fB\-rpcssl\fR
-.IP
-Use OpenSSL (https) for JSON\-RPC connections
-.HP
-\fB\-rpcsslcertificatechainfile=\fR<file.cert>
-.IP
-Server certificate file (default: server.cert)
-.HP
-\fB\-rpcsslprivatekeyfile=\fR<file.pem>
-.IP
-Server private key (default: server.pem)
-.HP
-\fB\-rpcsslciphers=\fR<ciphers>
-.IP
-Acceptable ciphers (default:
-TLSv1.2+HIGH:TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!3DES:@STRENGTH)
-.PP
 Metrics Options (only if \fB\-daemon\fR and \fB\-printtoconsole\fR are not set):
 .HP
 \fB\-showmetrics\fR

diff --git a/qa/rpc-tests/httpbasics.py b/qa/rpc-tests/httpbasics.py
@@ -22,7 +22,7 @@
 
 class HTTPBasicsTest (BitcoinTestFramework):
     def setup_nodes(self):
-        return start_nodes(4, self.options.tmpdir, extra_args=[['-rpckeepalive=1'], ['-rpckeepalive=0'], [], []])
+        return start_nodes(4, self.options.tmpdir)
 
     def run_test(self):
 
@@ -38,9 +38,13 @@ def run_test(self):
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read();
         assert_equal('"error":null' in out1, True)
+        assert_equal(conn.sock!=None, True) #according to http/1.1 connection must still be open!
 
-        # TODO #1856: Re-enable support for persistent connections.
-        assert_equal(conn.sock!=None, False)
+        #send 2nd request without closing connection
+        conn.request('POST', '/', '{"method": "getchaintips"}', headers)
+        out2 = conn.getresponse().read();
+        assert_equal('"error":null' in out1, True) #must also response with a correct json-rpc message
+        assert_equal(conn.sock!=None, True) #according to http/1.1 connection must still be open!
         conn.close()
 
         #same should be if we add keep-alive because this should be the std. behaviour
@@ -51,9 +55,13 @@ def run_test(self):
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read();
         assert_equal('"error":null' in out1, True)
+        assert_equal(conn.sock!=None, True) #according to http/1.1 connection must still be open!
 
-        # TODO #1856: Re-enable support for persistent connections.
-        assert_equal(conn.sock!=None, False)
+        #send 2nd request without closing connection
+        conn.request('POST', '/', '{"method": "getchaintips"}', headers)
+        out2 = conn.getresponse().read();
+        assert_equal('"error":null' in out1, True) #must also response with a correct json-rpc message
+        assert_equal(conn.sock!=None, True) #according to http/1.1 connection must still be open!
         conn.close()
 
         #now do the same with "Connection: close"
@@ -76,9 +84,8 @@ def run_test(self):
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read();
         assert_equal('"error":null' in out1, True)
-        assert_equal(conn.sock!=None, False) #connection must be closed because keep-alive was set to false
 
-        #node2 (third node) is running with standard keep-alive parameters which means keep-alive is off
+        #node2 (third node) is running with standard keep-alive parameters which means keep-alive is on
         urlNode2 = urlparse.urlparse(self.nodes[2].url)
         authpair = urlNode2.username + ':' + urlNode2.password
         headers = {"Authorization": "Basic " + base64.b64encode(authpair)}
@@ -88,10 +95,7 @@ def run_test(self):
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read();
         assert_equal('"error":null' in out1, True)
-
-        # TODO #1856: Re-enable support for persistent connections.
-        assert_equal(conn.sock!=None, False)
-        conn.close()
+        assert_equal(conn.sock!=None, True) #connection must be closed because bitcoind should use keep-alive by default
 
 if __name__ == '__main__':
     HTTPBasicsTest ().main ()
diff --git a/qa/rpc-tests/rest.py b/qa/rpc-tests/rest.py
@@ -32,10 +32,20 @@ def deser_uint256(f):
         r += t << (i * 32)
     return r
 
-#allows simple http get calls with a request body
-def http_get_call(host, port, path, requestdata = '', response_object = 0):
+#allows simple http get calls
+def http_get_call(host, port, path, response_object = 0):
     conn = httplib.HTTPConnection(host, port)
-    conn.request('GET', path, requestdata)
+    conn.request('GET', path)
+
+    if response_object:
+        return conn.getresponse()
+
+    return conn.getresponse().read()
+
+#allows simple http post calls with a request body
+def http_post_call(host, port, path, requestdata = '', response_object = 0):
+    conn = httplib.HTTPConnection(host, port)
+    conn.request('POST', path, requestdata)
 
     if response_object:
         return conn.getresponse()
@@ -137,7 +147,7 @@ def run_test(self):
         binaryRequest += binascii.unhexlify(vintx);
         binaryRequest += pack("i", 0);
 
-        bin_response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', binaryRequest)
+        bin_response = http_post_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', binaryRequest)
         output = StringIO.StringIO()
         output.write(bin_response)
         output.seek(0)
@@ -175,29 +185,29 @@ def run_test(self):
 
         #do some invalid requests
         json_request = '{"checkmempool'
-        response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True)
+        response = http_post_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True)
         assert_equal(response.status, 500) #must be a 500 because we send a invalid json request
 
         json_request = '{"checkmempool'
-        response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', json_request, True)
+        response = http_post_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', json_request, True)
         assert_equal(response.status, 500) #must be a 500 because we send a invalid bin request
 
-        response = http_get_call(url.hostname, url.port, '/rest/getutxos/checkmempool'+self.FORMAT_SEPARATOR+'bin', '', True)
+        response = http_post_call(url.hostname, url.port, '/rest/getutxos/checkmempool'+self.FORMAT_SEPARATOR+'bin', '', True)
         assert_equal(response.status, 500) #must be a 500 because we send a invalid bin request
 
         #test limits
         json_request = '/checkmempool/'
         for x in range(0, 20):
             json_request += txid+'-'+str(n)+'/'
         json_request = json_request.rstrip("/")
-        response = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True)
+        response = http_post_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True)
         assert_equal(response.status, 500) #must be a 500 because we exceeding the limits
 
         json_request = '/checkmempool/'
         for x in range(0, 15):
             json_request += txid+'-'+str(n)+'/'
         json_request = json_request.rstrip("/");
-        response = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True)
+        response = http_post_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True)
         assert_equal(response.status, 200) #must be a 500 because we exceeding the limits
 
         self.nodes[0].generate(1) #generate block to not affect upcoming tests
@@ -217,27 +227,27 @@ def run_test(self):
         #                this is 32 9-bit indices
 
         # check binary format
-        response = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"bin", "", True)
+        response = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"bin", True)
         assert_equal(response.status, 200)
         assert_greater_than(int(response.getheader('content-length')), 177)
         response_str = response.read()
 
         # compare with block header
-        response_header = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"bin", "", True)
+        response_header = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"bin", True)
         assert_equal(response_header.status, 200)
         assert_equal(int(response_header.getheader('content-length')), 177)
         response_header_str = response_header.read()
         assert_equal(response_str[0:177], response_header_str)
 
         # check block hex format
-        response_hex = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"hex", "", True)
+        response_hex = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"hex", True)
         assert_equal(response_hex.status, 200)
         assert_greater_than(int(response_hex.getheader('content-length')), 354)
         response_hex_str = response_hex.read()
         assert_equal(response_str.encode("hex")[0:354], response_hex_str[0:354])
 
         # compare with hex block header
-        response_header_hex = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"hex", "", True)
+        response_header_hex = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"hex", True)
         assert_equal(response_header_hex.status, 200)
         assert_greater_than(int(response_header_hex.getheader('content-length')), 354)
         response_header_hex_str = response_header_hex.read()
@@ -250,7 +260,7 @@ def run_test(self):
         assert_equal(block_json_obj['hash'], bb_hash)
 
         # compare with json block header
-        response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"json", "", True)
+        response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"json", True)
         assert_equal(response_header_json.status, 200)
         response_header_json_str = response_header_json.read()
         json_obj = json.loads(response_header_json_str, parse_float=decimal.Decimal)
@@ -274,7 +284,7 @@ def run_test(self):
         #see if we can get 5 headers in one response
         self.nodes[1].generate(5)
         self.sync_all()
-        response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/5/'+bb_hash+self.FORMAT_SEPARATOR+"json", "", True)
+        response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/5/'+bb_hash+self.FORMAT_SEPARATOR+"json", True)
         assert_equal(response_header_json.status, 200)
         response_header_json_str = response_header_json.read()
         json_obj = json.loads(response_header_json_str)
@@ -287,7 +297,7 @@ def run_test(self):
         assert_equal(json_obj['txid'], tx_hash)
 
         # check hex format response
-        hex_string = http_get_call(url.hostname, url.port, '/rest/tx/'+tx_hash+self.FORMAT_SEPARATOR+"hex", "", True)
+        hex_string = http_get_call(url.hostname, url.port, '/rest/tx/'+tx_hash+self.FORMAT_SEPARATOR+"hex", True)
         assert_equal(hex_string.status, 200)
         assert_greater_than(int(response.getheader('content-length')), 10)
 

diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py
@@ -106,6 +106,32 @@ def __getattr__(self, name):
             name = "%s.%s" % (self.__service_name, name)
         return AuthServiceProxy(self.__service_url, name, connection=self.__conn)
 
+    def _request(self, method, path, postdata):
+        '''
+        Do a HTTP request, with retry if we get disconnected (e.g. due to a timeout).
+        This is a workaround for https://bugs.python.org/issue3566 which is fixed in Python 3.5.
+        '''
+        headers = {'Host': self.__url.hostname,
+                   'User-Agent': USER_AGENT,
+                   'Authorization': self.__auth_header,
+                   'Content-type': 'application/json'}
+        try:
+            self.__conn.request(method, path, postdata, headers)
+            return self._get_response()
+        except httplib.BadStatusLine as e:
+            if e.line == "''": # if connection was closed, try again
+                self.__conn.close()
+                self.__conn.request(method, path, postdata, headers)
+                return self._get_response()
+            else:
+                raise
+        except (BrokenPipeError,ConnectionResetError):
+            # Python 3.5+ raises BrokenPipeError instead of BadStatusLine when the connection was reset
+            # ConnectionResetError happens on FreeBSD with Python 3.4
+            self.__conn.close()
+            self.__conn.request(method, path, postdata, headers)
+            return self._get_response()
+
     def __call__(self, *args):
         AuthServiceProxy.__id_count += 1
 
@@ -115,13 +141,7 @@ def __call__(self, *args):
                                'method': self.__service_name,
                                'params': args,
                                'id': AuthServiceProxy.__id_count}, default=EncodeDecimal)
-        self.__conn.request('POST', self.__url.path, postdata,
-                            {'Host': self.__url.hostname,
-                             'User-Agent': USER_AGENT,
-                             'Authorization': self.__auth_header,
-                             'Content-type': 'application/json'})
-
-        response = self._get_response()
+        response = self._request('POST', self.__url.path, postdata)
         if response['error'] is not None:
             raise JSONRPCException(response['error'])
         elif 'result' not in response:
@@ -133,13 +153,7 @@ def __call__(self, *args):
     def _batch(self, rpc_call_list):
         postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal)
         log.debug("--> "+postdata)
-        self.__conn.request('POST', self.__url.path, postdata,
-                            {'Host': self.__url.hostname,
-                             'User-Agent': USER_AGENT,
-                             'Authorization': self.__auth_header,
-                             'Content-type': 'application/json'})
-
-        return self._get_response()
+        return self._request('POST', self.__url.path, postdata)
 
     def _get_response(self):
         http_response = self.__conn.getresponse()