Auto merge of zcash#2601 - str4d:2156-large-wallet-utxos, r=str4d

Benchmark time to call sendtoaddress with many UTXOs

Part of zcash#2156.

qa/zcash/create_wallet_200k_utxos.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python2
+# Copyright (c) 2017 The Zcash developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# Create a large wallet
+#
+# To use:
+# - Copy to qa/rpc-tests/wallet_large.py
+# - Add wallet_large.py to RPC tests list
+# - ./qa/pull-tester/rpc-tests.sh wallet_large --nocleanup
+# - Archive the resulting /tmp/test###### directory
+#
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    connect_nodes_bi,
+    initialize_chain_clean,
+    start_nodes,
+)
+
+from decimal import Decimal
+
+
+class LargeWalletTest(BitcoinTestFramework):
+
+    def setup_chain(self):
+        print("Initializing test directory "+self.options.tmpdir)
+        initialize_chain_clean(self.options.tmpdir, 2)
+
+    def setup_network(self):
+        self.nodes = start_nodes(2, self.options.tmpdir)
+        connect_nodes_bi(self.nodes, 0, 1)
+        self.is_network_split = False
+        self.sync_all()
+
+    def run_test(self):
+        self.nodes[1].generate(103)
+        self.sync_all()
+
+        inputs = []
+        for i in range(200000):
+            taddr = self.nodes[0].getnewaddress()
+            inputs.append(self.nodes[1].sendtoaddress(taddr, Decimal("0.001")))
+            if i % 1000 == 0:
+                self.nodes[1].generate(1)
+                self.sync_all()
+
+        self.nodes[1].generate(1)
+        self.sync_all()
+        print('Node 0: %d transactions, %d UTXOs' %
+              (len(self.nodes[0].listtransactions()), len(self.nodes[0].listunspent())))
+        print('Node 1: %d transactions, %d UTXOs' %
+              (len(self.nodes[1].listtransactions()), len(self.nodes[1].listunspent())))
+        assert_equal(len(self.nodes[0].listunspent()), len(inputs))
+
+if __name__ == '__main__':
+    LargeWalletTest().main()

qa/zcash/performance-measurements.sh

@@ -28,10 +28,52 @@ function zcashd_generate {
     zcash_rpc generate 101 > /dev/null
 }
 
+function extract_benchmark_datadir {
+    if [ -f "$1.tar.xz" ]; then
+        # Check the hash of the archive:
+        "$SHA256CMD" $SHA256ARGS -c <<EOF
+$2  $1.tar.xz
+EOF
+        ARCHIVE_RESULT=$?
+    else
+        echo "$1.tar.xz not found."
+        ARCHIVE_RESULT=1
+    fi
+    if [ $ARCHIVE_RESULT -ne 0 ]; then
+        zcashd_stop
+        echo
+        echo "Please download it and place it in the base directory of the repository."
+        exit 1
+    fi
+    xzcat "$1.tar.xz" | tar x
+}
+
+function use_200k_benchmark {
+    rm -rf benchmark-200k-UTXOs
+    extract_benchmark_datadir benchmark-200k-UTXOs dc8ab89eaa13730da57d9ac373c1f4e818a37181c1443f61fd11327e49fbcc5e
+    DATADIR="./benchmark-200k-UTXOs/node$1"
+}
+
 function zcashd_start {
-    rm -rf "$DATADIR"
-    mkdir -p "$DATADIR/regtest"
-    touch "$DATADIR/zcash.conf"
+    case "$1" in
+        sendtoaddress)
+            case "$2" in
+                200k-recv)
+                    use_200k_benchmark 0
+                    ;;
+                200k-send)
+                    use_200k_benchmark 1
+                    ;;
+                *)
+                    echo "Bad arguments."
+                    exit 1
+            esac
+            ;;
+        *)
+            rm -rf "$DATADIR"
+            mkdir -p "$DATADIR/regtest"
+            touch "$DATADIR/zcash.conf"
+    esac
     ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 &
     ZCASHD_PID=$!
     zcash_rpc_wait_for_start
@@ -43,9 +85,25 @@ function zcashd_stop {
 }
 
 function zcashd_massif_start {
-    rm -rf "$DATADIR"
-    mkdir -p "$DATADIR/regtest"
-    touch "$DATADIR/zcash.conf"
+    case "$1" in
+        sendtoaddress)
+            case "$2" in
+                200k-recv)
+                    use_200k_benchmark 0
+                    ;;
+                200k-send)
+                    use_200k_benchmark 1
+                    ;;
+                *)
+                    echo "Bad arguments."
+                    exit 1
+            esac
+            ;;
+        *)
+            rm -rf "$DATADIR"
+            mkdir -p "$DATADIR/regtest"
+            touch "$DATADIR/zcash.conf"
+    esac
     rm -f massif.out
     valgrind --tool=massif --time-unit=ms --massif-out-file=massif.out ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 &
     ZCASHD_PID=$!
@@ -101,15 +159,15 @@ case "$1" in
     *)
         case "$2" in
             verifyjoinsplit)
-                zcashd_start
+                zcashd_start "${@:2}"
                 RAWJOINSPLIT=$(zcash_rpc zcsamplejoinsplit)
                 zcashd_stop
         esac
 esac
 
 case "$1" in
     time)
-        zcashd_start
+        zcashd_start "${@:2}"
         case "$2" in
             sleep)
                 zcash_rpc zcbenchmark sleep 10
@@ -142,6 +200,9 @@ case "$1" in
                 extract_benchmark_data
                 zcash_rpc zcbenchmark connectblockslow 10
                 ;;
+            sendtoaddress)
+                zcash_rpc zcbenchmark sendtoaddress 10 "${@:4}"
+                ;;
             *)
                 zcashd_stop
                 echo "Bad arguments."
@@ -150,7 +211,7 @@ case "$1" in
         zcashd_stop
         ;;
     memory)
-        zcashd_massif_start
+        zcashd_massif_start "${@:2}"
         case "$2" in
             sleep)
                 zcash_rpc zcbenchmark sleep 1
@@ -180,6 +241,9 @@ case "$1" in
                 extract_benchmark_data
                 zcash_rpc zcbenchmark connectblockslow 1
                 ;;
+            sendtoaddress)
+                zcash_rpc zcbenchmark sendtoaddress 1 "${@:4}"
+                ;;
             *)
                 zcashd_massif_stop
                 echo "Bad arguments."

src/wallet/rpcwallet.cpp

@@ -2596,6 +2596,12 @@ UniValue zc_benchmark(const UniValue& params, bool fHelp)
                 throw JSONRPCError(RPC_TYPE_ERROR, "Benchmark must be run in regtest mode");
             }
             sample_times.push_back(benchmark_connectblock_slow());
+        } else if (benchmarktype == "sendtoaddress") {
+            if (Params().NetworkIDString() != "regtest") {
+                throw JSONRPCError(RPC_TYPE_ERROR, "Benchmark must be run in regtest mode");
+            }
+            auto amount = AmountFromValue(params[2]);
+            sample_times.push_back(benchmark_sendtoaddress(amount));
         } else {
             throw JSONRPCError(RPC_TYPE_ERROR, "Invalid benchmarktype");
         }

src/zcbenchmarks.cpp

@@ -17,6 +17,7 @@
 #include "main.h"
 #include "miner.h"
 #include "pow.h"
+#include "rpcserver.h"
 #include "script/sign.h"
 #include "sodium.h"
 #include "streams.h"
@@ -405,3 +406,17 @@ double benchmark_connectblock_slow()
     return duration;
 }
 
+double benchmark_sendtoaddress(CAmount amount)
+{
+    UniValue params(UniValue::VARR);
+    auto addr = getnewaddress(params, false);
+
+    params.push_back(addr);
+    params.push_back(ValueFromAmount(amount));
+
+    struct timeval tv_start;
+    timer_start(tv_start);
+    auto txid = sendtoaddress(params, false);
+    return timer_stop(tv_start);
+}
+

src/zcbenchmarks.h

@@ -16,5 +16,6 @@ extern double benchmark_large_tx();
 extern double benchmark_try_decrypt_notes(size_t nAddrs);
 extern double benchmark_increment_note_witnesses(size_t nTxs);
 extern double benchmark_connectblock_slow();
+extern double benchmark_sendtoaddress(CAmount amount);
 
 #endif