Add travis build that tests bitcoin functional tests

.travis.yml

@@ -63,6 +63,7 @@ script:
     - if [ "$RUN_BENCH" = "true" ]; then BEGIN_FOLD bench; DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib $OUTDIR/bin/bench_bitcoin -scaling=0.001 ; END_FOLD; fi
     - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then extended="--extended --exclude feature_pruning,feature_dbcrash"; fi
     - if [ "$RUN_TESTS" = "true" ]; then BEGIN_FOLD functional-tests; DOCKER_EXEC test/functional/test_runner.py --combinedlogslen=4000 --coverage --quiet --failfast ${extended}; END_FOLD; fi
+    - if [ "$RUN_BITCOIN_TESTS" = "true" ]; then BEGIN_FOLD bitcoin-functional-tests; DOCKER_EXEC test/bitcoin_functional/functional/test_runner.py --combinedlogslen=4000 --coverage --quiet --failfast ${extended}; END_FOLD; fi
 after_script:
     - echo $TRAVIS_COMMIT_RANGE
     - echo $TRAVIS_COMMIT_LOG
@@ -124,6 +125,15 @@ jobs:
         RUN_TESTS=true
         GOAL="install"
         BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --enable-glibc-back-compat --enable-reduce-exports --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER"
+# x86_64 Linux w/ Bitcoin functional tests (Qt5 & system libs)
+    - stage: test
+      env: >-
+        HOST=x86_64-unknown-linux-gnu
+        PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev"
+        NO_DEPENDS=1
+        RUN_BITCOIN_TESTS=true
+        GOAL="install"
+        BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --enable-glibc-back-compat --enable-reduce-exports --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER"
 # x86_64 Linux, No wallet
     - stage: test
       env: >-

configure.ac

@@ -1401,6 +1401,7 @@ AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/spl
 AM_COND_IF([HAVE_DOXYGEN], [AC_CONFIG_FILES([doc/Doxyfile])])
 AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py])
 AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py])
+AC_CONFIG_LINKS([test/bitcoin_functional/functional/test_runner.py:test/bitcoin_functional/functional/test_runner.py])
 AC_CONFIG_LINKS([test/util/bitcoin-util-test.py:test/util/bitcoin-util-test.py])
 AC_CONFIG_LINKS([test/util/rpcauth-test.py:test/util/rpcauth-test.py])
 

test/bitcoin_functional/functional/.gitignore

@@ -0,0 +1,2 @@
+*.pyc
+cache

test/bitcoin_functional/functional/README.md

@@ -0,0 +1,115 @@
+# Functional tests
+
+### Writing Functional Tests
+
+#### Example test
+
+The [example_test.py](example_test.py) is a heavily commented example of a test case that uses both
+the RPC and P2P interfaces. If you are writing your first test, copy that file
+and modify to fit your needs.
+
+#### Coverage
+
+Running `test_runner.py` with the `--coverage` argument tracks which RPCs are
+called by the tests and prints a report of uncovered RPCs in the summary. This
+can be used (along with the `--extended` argument) to find out which RPCs we
+don't have test cases for.
+
+#### Style guidelines
+
+- Where possible, try to adhere to [PEP-8 guidelines](https://www.python.org/dev/peps/pep-0008/)
+- Use a python linter like flake8 before submitting PRs to catch common style
+  nits (eg trailing whitespace, unused imports, etc)
+- See [the python lint script](/test/lint/lint-python.sh) that checks for violations that
+  could lead to bugs and issues in the test code.
+- Avoid wildcard imports where possible
+- Use a module-level docstring to describe what the test is testing, and how it
+  is testing it.
+- When subclassing the BitcoinTestFramwork, place overrides for the
+  `set_test_params()`, `add_options()` and `setup_xxxx()` methods at the top of
+  the subclass, then locally-defined helper methods, then the `run_test()` method.
+- Use `'{}'.format(x)` for string formatting, not `'%s' % x`.
+
+#### Naming guidelines
+
+- Name the test `<area>_test.py`, where area can be one of the following:
+    - `feature` for tests for full features that aren't wallet/mining/mempool, eg `feature_rbf.py`
+    - `interface` for tests for other interfaces (REST, ZMQ, etc), eg `interface_rest.py`
+    - `mempool` for tests for mempool behaviour, eg `mempool_reorg.py`
+    - `mining` for tests for mining features, eg `mining_prioritisetransaction.py`
+    - `p2p` for tests that explicitly test the p2p interface, eg `p2p_disconnect_ban.py`
+    - `rpc` for tests for individual RPC methods or features, eg `rpc_listtransactions.py`
+    - `wallet` for tests for wallet features, eg `wallet_keypool.py`
+- use an underscore to separate words
+    - exception: for tests for specific RPCs or command line options which don't include underscores, name the test after the exact RPC or argument name, eg `rpc_decodescript.py`, not `rpc_decode_script.py`
+- Don't use the redundant word `test` in the name, eg `interface_zmq.py`, not `interface_zmq_test.py`
+
+#### General test-writing advice
+
+- Set `self.num_nodes` to the minimum number of nodes necessary for the test.
+  Having additional unrequired nodes adds to the execution time of the test as
+  well as memory/CPU/disk requirements (which is important when running tests in
+  parallel or on Travis).
+- Avoid stop-starting the nodes multiple times during the test if possible. A
+  stop-start takes several seconds, so doing it several times blows up the
+  runtime of the test.
+- Set the `self.setup_clean_chain` variable in `set_test_params()` to control whether
+  or not to use the cached data directories. The cached data directories
+  contain a 200-block pre-mined blockchain and wallets for four nodes. Each node
+  has 25 mature blocks (25x50=1250 BTC) in its wallet.
+- When calling RPCs with lots of arguments, consider using named keyword
+  arguments instead of positional arguments to make the intent of the call
+  clear to readers.
+
+#### RPC and P2P definitions
+
+Test writers may find it helpful to refer to the definitions for the RPC and
+P2P messages. These can be found in the following source files:
+
+- `/src/rpc/*` for RPCs
+- `/src/wallet/rpc*` for wallet RPCs
+- `ProcessMessage()` in `/src/net_processing.cpp` for parsing P2P messages
+
+#### Using the P2P interface
+
+- `mininode.py` contains all the definitions for objects that pass
+over the network (`CBlock`, `CTransaction`, etc, along with the network-level
+wrappers for them, `msg_block`, `msg_tx`, etc).
+
+- P2P tests have two threads. One thread handles all network communication
+with the bitcoind(s) being tested in a callback-based event loop; the other
+implements the test logic.
+
+- `P2PConnection` is the class used to connect to a bitcoind.  `P2PInterface`
+contains the higher level logic for processing P2P payloads and connecting to
+the Bitcoin Core node application logic. For custom behaviour, subclass the
+P2PInterface object and override the callback methods.
+
+- Can be used to write tests where specific P2P protocol behavior is tested.
+Examples tests are `p2p_unrequested_blocks.py`, `p2p_compactblocks.py`.
+
+### test-framework modules
+
+#### [test_framework/authproxy.py](test_framework/authproxy.py)
+Taken from the [python-bitcoinrpc repository](https://github.com/jgarzik/python-bitcoinrpc).
+
+#### [test_framework/test_framework.py](test_framework/test_framework.py)
+Base class for functional tests.
+
+#### [test_framework/util.py](test_framework/util.py)
+Generally useful functions.
+
+#### [test_framework/mininode.py](test_framework/mininode.py)
+Basic code to support P2P connectivity to a bitcoind.
+
+#### [test_framework/script.py](test_framework/script.py)
+Utilities for manipulating transaction scripts (originally from python-bitcoinlib)
+
+#### [test_framework/key.py](test_framework/key.py)
+Wrapper around OpenSSL EC_Key (originally from python-bitcoinlib)
+
+#### [test_framework/bignum.py](test_framework/bignum.py)
+Helpers for script.py
+
+#### [test_framework/blocktools.py](test_framework/blocktools.py)
+Helper functions for creating blocks and transactions.

test/bitcoin_functional/functional/combine_logs.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""Combine logs from multiple bitcoin nodes as well as the test_framework log.
+
+This streams the combined log output to stdout. Use combine_logs.py > outputfile
+to write to an outputfile."""
+
+import argparse
+from collections import defaultdict, namedtuple
+import heapq
+import itertools
+import os
+import re
+import sys
+
+# Matches on the date format at the start of the log event
+TIMESTAMP_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z")
+
+LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event'])
+
+def main():
+    """Main function. Parses args, reads the log files and renders them as text or html."""
+
+    parser = argparse.ArgumentParser(usage='%(prog)s [options] <test temporary directory>', description=__doc__)
+    parser.add_argument('-c', '--color', dest='color', action='store_true', help='outputs the combined log with events colored by source (requires posix terminal colors. Use less -r for viewing)')
+    parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2')
+    parser.add_argument('--chain', dest='chain', help='selected chain in the tests (default: regtest2)', default='regtest2')
+    args, unknown_args = parser.parse_known_args()
+
+    if args.color and os.name != 'posix':
+        print("Color output requires posix terminal colors.")
+        sys.exit(1)
+
+    if args.html and args.color:
+        print("Only one out of --color or --html should be specified")
+        sys.exit(1)
+
+    # There should only be one unknown argument - the path of the temporary test directory
+    if len(unknown_args) != 1:
+        print("Unexpected arguments" + str(unknown_args))
+        sys.exit(1)
+
+    log_events = read_logs(unknown_args[0], args.chain)
+
+    print_logs(log_events, color=args.color, html=args.html)
+
+def read_logs(tmp_dir, chain):
+    """Reads log files.
+
+    Delegates to generator function get_log_events() to provide individual log events
+    for each of the input log files."""
+
+    files = [("test", "%s/test_framework.log" % tmp_dir)]
+    for i in itertools.count():
+        logfile = "{}/node{}/{}/debug.log".format(tmp_dir, i, chain)
+        if not os.path.isfile(logfile):
+            break
+        files.append(("node%d" % i, logfile))
+
+    return heapq.merge(*[get_log_events(source, f) for source, f in files])
+
+def get_log_events(source, logfile):
+    """Generator function that returns individual log events.
+
+    Log events may be split over multiple lines. We use the timestamp
+    regex match as the marker for a new log event."""
+    try:
+        with open(logfile, 'r', encoding='utf-8') as infile:
+            event = ''
+            timestamp = ''
+            for line in infile:
+                # skip blank lines
+                if line == '\n':
+                    continue
+                # if this line has a timestamp, it's the start of a new log event.
+                time_match = TIMESTAMP_PATTERN.match(line)
+                if time_match:
+                    if event:
+                        yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip())
+                    event = line
+                    timestamp = time_match.group()
+                # if it doesn't have a timestamp, it's a continuation line of the previous log.
+                else:
+                    event += "\n" + line
+            # Flush the final event
+            yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip())
+    except FileNotFoundError:
+        print("File %s could not be opened. Continuing without it." % logfile, file=sys.stderr)
+
+def print_logs(log_events, color=False, html=False):
+    """Renders the iterator of log events into text or html."""
+    if not html:
+        colors = defaultdict(lambda: '')
+        if color:
+            colors["test"] = "\033[0;36m"   # CYAN
+            colors["node0"] = "\033[0;34m"  # BLUE
+            colors["node1"] = "\033[0;32m"  # GREEN
+            colors["node2"] = "\033[0;31m"  # RED
+            colors["node3"] = "\033[0;33m"  # YELLOW
+            colors["reset"] = "\033[0m"     # Reset font color
+
+        for event in log_events:
+            print("{0} {1: <5} {2} {3}".format(colors[event.source.rstrip()], event.source, event.event, colors["reset"]))
+
+    else:
+        try:
+            import jinja2
+        except ImportError:
+            print("jinja2 not found. Try `pip install jinja2`")
+            sys.exit(1)
+        print(jinja2.Environment(loader=jinja2.FileSystemLoader('./'))
+                    .get_template('combined_log_template.html')
+                    .render(title="Combined Logs from testcase", log_events=[event._asdict() for event in log_events]))
+
+if __name__ == '__main__':
+    main()

test/bitcoin_functional/functional/combined_log_template.html

@@ -0,0 +1,40 @@
+<html lang="en">
+<head>
+    <title> {{ title }} </title>
+    <style>
+        ul {
+            list-style-type: none;
+            font-family: monospace;
+        }
+        li {
+            border: 1px solid slategray;
+            margin-bottom: 1px;
+        }
+        li:hover {
+            filter: brightness(85%);
+        }
+        li.log-test {
+            background-color: cyan;
+        }
+        li.log-node0 {
+            background-color: lightblue;
+        }
+        li.log-node1 {
+            background-color: lightgreen;
+        }
+        li.log-node2 {
+            background-color: lightsalmon;
+        }
+        li.log-node3 {
+            background-color: lightyellow;
+        }
+    </style>
+</head>
+<body>
+<ul>
+{% for event in log_events %}
+<li class="log-{{ event.source }}"> {{ event.source }} {{ event.timestamp }} {{event.event}}</li>
+{% endfor %}
+</ul>
+</body>
+</html>

test/bitcoin_functional/functional/create_cache.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016-2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Create a blockchain cache.
+
+Creating a cache of the blockchain speeds up test execution when running
+multiple functional tests. This helper script is executed by test_runner when multiple
+tests are being run in parallel.
+"""
+
+from test_framework.test_framework import BitcoinTestFramework
+
+class CreateCache(BitcoinTestFramework):
+    # Test network and test nodes are not required:
+
+    def set_test_params(self):
+        self.num_nodes = 0
+        self.supports_cli = True
+
+    def setup_network(self):
+        pass
+
+    def run_test(self):
+        pass
+
+if __name__ == '__main__':
+    CreateCache().main()