From 8a97563a932a3bff17cf4c04a58608044a3f486a Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 17 Mar 2019 15:39:15 +0000 Subject: [PATCH 01/10] Use threadpool instead of new threads for DNS queries --- src/common/dns_utils.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 417b5b4ac3c..d1c59fab71b 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -32,9 +32,9 @@ #include #include "include_base_utils.h" +#include "common/threadpool.h" #include #include -#include #include #include using namespace epee; @@ -496,16 +496,16 @@ bool load_txt_records_from_dns(std::vector &good_records, const std size_t first_index = dis(gen); // send all requests in parallel - std::vector threads(dns_urls.size()); std::deque avail(dns_urls.size(), false), valid(dns_urls.size(), false); + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; for (size_t n = 0; n < dns_urls.size(); ++n) { - threads[n] = boost::thread([n, dns_urls, &records, &avail, &valid](){ + tpool.submit(&waiter,[n, dns_urls, &records, &avail, &valid](){ records[n] = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); }); } - for (size_t n = 0; n < dns_urls.size(); ++n) - threads[n].join(); + waiter.wait(&tpool); size_t cur_index = first_index; do From e99b2b12fd4a7da8687c5da8bf0dd356641ec91f Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 17 Mar 2019 18:37:29 +0000 Subject: [PATCH 02/10] simplewallet: make the long payment ids warnigns more warney We want people to really realize it's bad for *their* privacy. --- src/simplewallet/simplewallet.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 8c0b300ad02..52d880d2c92 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -116,7 +116,9 @@ typedef cryptonote::simple_wallet sw; #define LONG_PAYMENT_ID_SUPPORT_CHECK() \ do { \ if (!m_long_payment_id_support) { \ - fail_msg_writer() << tr("Long payment IDs are obsolete. Use --long-payment-id-support if you really must use one, and warn the recipient they are using an obsolete feature that will disappear in the future."); \ + fail_msg_writer() << tr("Warning: Long payment IDs are obsolete."); \ + fail_msg_writer() << tr("Long payment IDs are not encrypted on the blockchain, and will harm your privacy."); \ + fail_msg_writer() << tr("Use --long-payment-id-support if you really must use one, and warn the recipient they are using an obsolete feature that will disappear in the future."); \ return true; \ } \ } while(0) @@ -149,7 +151,7 @@ namespace const command_line::arg_descriptor arg_create_address_file = {"create-address-file", sw::tr("Create an address file for new wallets"), false}; const command_line::arg_descriptor arg_subaddress_lookahead = {"subaddress-lookahead", tools::wallet2::tr("Set subaddress lookahead sizes to :"), ""}; const command_line::arg_descriptor arg_use_english_language_names = {"use-english-language-names", sw::tr("Display English language names"), false}; - const command_line::arg_descriptor arg_long_payment_id_support = {"long-payment-id-support", sw::tr("Support obsolete long (unencrypted) payment ids"), false}; + const command_line::arg_descriptor arg_long_payment_id_support = {"long-payment-id-support-bad-for-privacy", sw::tr("Support obsolete long (unencrypted) payment ids (using them harms your privacy)"), false}; const command_line::arg_descriptor< std::vector > arg_command = {"command", ""}; @@ -5335,7 +5337,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector Date: Thu, 21 Feb 2019 12:55:55 +0000 Subject: [PATCH 03/10] rpc: quantize db size up to 5 GB in restricted mode --- src/rpc/core_rpc_server.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index ff55167fef5..8c6ea7cd126 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1736,7 +1736,9 @@ namespace cryptonote boost::shared_lock lock(m_bootstrap_daemon_mutex); res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; } - res.database_size = restricted ? 0 : m_core.get_blockchain_storage().get_db().get_database_size(); + res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); + if (restricted) + res.database_size = round_up(res.database_size, 5ull * 1024 * 1024 * 1024); res.update_available = restricted ? false : m_core.is_update_available(); res.version = restricted ? "" : MONERO_VERSION; return true; From 43042a28ec47044b39024152e6bb706616e0d7fa Mon Sep 17 00:00:00 2001 From: Guido Vranken Date: Sun, 17 Mar 2019 17:39:37 +0100 Subject: [PATCH 04/10] Implement array_entry_t copy constructor Manually initialize the array_entry_t iterator to ensure it points to the correct m_array, thereby preventing a potential use-after-free situation. Signed-off-by: Guido Vranken --- contrib/epee/include/storages/portable_storage_base.h | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/epee/include/storages/portable_storage_base.h b/contrib/epee/include/storages/portable_storage_base.h index da84fd8eaf7..ca7c81ddc13 100644 --- a/contrib/epee/include/storages/portable_storage_base.h +++ b/contrib/epee/include/storages/portable_storage_base.h @@ -82,6 +82,7 @@ namespace epee struct array_entry_t { array_entry_t():m_it(m_array.end()){} + array_entry_t(const array_entry_t& other):m_array(other.m_array), m_it(m_array.end()){} const t_entry_type* get_first_val() const { From d7dd857579ce11dc997008bdbf795f1af812ebaa Mon Sep 17 00:00:00 2001 From: Norman Moeschter Date: Tue, 19 Mar 2019 13:55:00 +0100 Subject: [PATCH 05/10] Upgraded static dependency versions and hashes in Dockerfile. --- Dockerfile | 57 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9fe7cfb8f63..0dd57c27f0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,10 +25,13 @@ RUN set -ex && \ WORKDIR /usr/local +ENV CFLAGS='-fPIC' +ENV CXXFLAGS='-fPIC' + #Cmake -ARG CMAKE_VERSION=3.13.0 -ARG CMAKE_VERSION_DOT=v3.13 -ARG CMAKE_HASH=4058b2f1a53c026564e8936698d56c3b352d90df067b195cb749a97a3d273c90 +ARG CMAKE_VERSION=3.14.0 +ARG CMAKE_VERSION_DOT=v3.14 +ARG CMAKE_HASH=aa76ba67b3c2af1946701f847073f4652af5cbd9f141f221c97af99127e75502 RUN set -ex \ && curl -s -O https://cmake.org/files/${CMAKE_VERSION_DOT}/cmake-${CMAKE_VERSION}.tar.gz \ && echo "${CMAKE_HASH} cmake-${CMAKE_VERSION}.tar.gz" | sha256sum -c \ @@ -39,41 +42,41 @@ RUN set -ex \ && make install ## Boost -ARG BOOST_VERSION=1_68_0 -ARG BOOST_VERSION_DOT=1.68.0 -ARG BOOST_HASH=7f6130bc3cf65f56a618888ce9d5ea704fa10b462be126ad053e80e553d6d8b7 +ARG BOOST_VERSION=1_69_0 +ARG BOOST_VERSION_DOT=1.69.0 +ARG BOOST_HASH=8f32d4617390d1c2d16f26a27ab60d97807b35440d45891fa340fc2648b04406 RUN set -ex \ && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://dl.bintray.com/boostorg/release/${BOOST_VERSION_DOT}/source/boost_${BOOST_VERSION}.tar.bz2 \ && echo "${BOOST_HASH} boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \ && tar -xvf boost_${BOOST_VERSION}.tar.bz2 \ && cd boost_${BOOST_VERSION} \ && ./bootstrap.sh \ - && ./b2 --build-type=minimal link=static runtime-link=static --with-chrono --with-date_time --with-filesystem --with-program_options --with-regex --with-serialization --with-system --with-thread --with-locale threading=multi threadapi=pthread cflags="-fPIC" cxxflags="-fPIC" stage + && ./b2 --build-type=minimal link=static runtime-link=static --with-chrono --with-date_time --with-filesystem --with-program_options --with-regex --with-serialization --with-system --with-thread --with-locale threading=multi threadapi=pthread cflags="$CFLAGS" cxxflags="$CXXFLAGS" stage ENV BOOST_ROOT /usr/local/boost_${BOOST_VERSION} # OpenSSL -ARG OPENSSL_VERSION=1.1.0j -ARG OPENSSL_HASH=31bec6c203ce1a8e93d5994f4ed304c63ccf07676118b6634edded12ad1b3246 +ARG OPENSSL_VERSION=1.1.1b +ARG OPENSSL_HASH=5c557b023230413dfb0756f3137a13e6d726838ccd1430888ad15bfb2b43ea4b RUN set -ex \ && curl -s -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ && echo "${OPENSSL_HASH} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c \ && tar -xzf openssl-${OPENSSL_VERSION}.tar.gz \ && cd openssl-${OPENSSL_VERSION} \ - && ./Configure linux-x86_64 no-shared --static -fPIC \ + && ./Configure linux-x86_64 no-shared --static "$CFLAGS" \ && make build_generated \ && make libcrypto.a \ && make install ENV OPENSSL_ROOT_DIR=/usr/local/openssl-${OPENSSL_VERSION} # ZMQ -ARG ZMQ_VERSION=v4.2.5 -ARG ZMQ_HASH=d062edd8c142384792955796329baf1e5a3377cd +ARG ZMQ_VERSION=v4.3.1 +ARG ZMQ_HASH=2cb1240db64ce1ea299e00474c646a2453a8435b RUN set -ex \ && git clone https://github.com/zeromq/libzmq.git -b ${ZMQ_VERSION} \ && cd libzmq \ && test `git rev-parse HEAD` = ${ZMQ_HASH} || exit 1 \ && ./autogen.sh \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \ + && ./configure --enable-static --disable-shared \ && make \ && make install \ && ldconfig @@ -88,39 +91,39 @@ RUN set -ex \ && mv *.hpp /usr/local/include # Readline -ARG READLINE_VERSION=7.0 -ARG READLINE_HASH=750d437185286f40a369e1e4f4764eda932b9459b5ec9a731628393dd3d32334 +ARG READLINE_VERSION=8.0 +ARG READLINE_HASH=e339f51971478d369f8a053a330a190781acb9864cf4c541060f12078948e461 RUN set -ex \ && curl -s -O https://ftp.gnu.org/gnu/readline/readline-${READLINE_VERSION}.tar.gz \ && echo "${READLINE_HASH} readline-${READLINE_VERSION}.tar.gz" | sha256sum -c \ && tar -xzf readline-${READLINE_VERSION}.tar.gz \ && cd readline-${READLINE_VERSION} \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure \ + && ./configure \ && make \ && make install # Sodium -ARG SODIUM_VERSION=1.0.16 -ARG SODIUM_HASH=675149b9b8b66ff44152553fb3ebf9858128363d +ARG SODIUM_VERSION=1.0.17 +ARG SODIUM_HASH=b732443c442239c2e0184820e9b23cca0de0828c RUN set -ex \ && git clone https://github.com/jedisct1/libsodium.git -b ${SODIUM_VERSION} \ && cd libsodium \ && test `git rev-parse HEAD` = ${SODIUM_HASH} || exit 1 \ && ./autogen.sh \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure \ + && ./configure \ && make \ && make check \ && make install # Udev -ARG UDEV_VERSION=v3.2.6 -ARG UDEV_HASH=0c35b136c08d64064efa55087c54364608e65ed6 +ARG UDEV_VERSION=v3.2.7 +ARG UDEV_HASH=4758e346a14126fc3a964de5831e411c27ebe487 RUN set -ex \ && git clone https://github.com/gentoo/eudev -b ${UDEV_VERSION} \ && cd eudev \ && test `git rev-parse HEAD` = ${UDEV_HASH} || exit 1 \ && ./autogen.sh \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared \ + && ./configure --disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared \ && make \ && make install @@ -132,7 +135,7 @@ RUN set -ex \ && cd libusb \ && test `git rev-parse HEAD` = ${USB_HASH} || exit 1 \ && ./autogen.sh \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-shared \ + && ./configure --disable-shared \ && make \ && make install @@ -144,20 +147,20 @@ RUN set -ex \ && cd hidapi \ && test `git rev-parse HEAD` = ${HIDAPI_HASH} || exit 1 \ && ./bootstrap \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \ + && ./configure --enable-static --disable-shared \ && make \ && make install # Protobuf -ARG PROTOBUF_VERSION=v3.6.1 -ARG PROTOBUF_HASH=48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 +ARG PROTOBUF_VERSION=v3.7.0 +ARG PROTOBUF_HASH=582743bf40c5d3639a70f98f183914a2c0cd0680 RUN set -ex \ && git clone https://github.com/protocolbuffers/protobuf -b ${PROTOBUF_VERSION} \ && cd protobuf \ && test `git rev-parse HEAD` = ${PROTOBUF_HASH} || exit 1 \ && git submodule update --init --recursive \ && ./autogen.sh \ - && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \ + && ./configure --enable-static --disable-shared \ && make \ && make install \ && ldconfig From 85f2f8c933c7167e04b7b0d7042fbcace3847cc1 Mon Sep 17 00:00:00 2001 From: rbrunner7 Date: Wed, 20 Mar 2019 16:40:59 +0100 Subject: [PATCH 06/10] Fix daemon startup parameter '--limit-rate' processing after parameter defaults --- src/p2p/net_node.inl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index ba6e79d3f98..565fcf0eaa7 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -2236,11 +2236,10 @@ namespace nodetool template bool node_server::set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit) { - this->islimitup=true; + this->islimitup=(limit != -1) && (limit != default_limit_up); if (limit==-1) { limit=default_limit_up; - this->islimitup=false; } epee::net_utils::connection >::set_rate_up_limit( limit ); @@ -2251,10 +2250,9 @@ namespace nodetool template bool node_server::set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit) { - this->islimitdown=true; + this->islimitdown=(limit != -1) && (limit != default_limit_down); if(limit==-1) { limit=default_limit_down; - this->islimitdown=false; } epee::net_utils::connection >::set_rate_down_limit( limit ); MINFO("Set limit-down to " << limit << " kB/s"); From 760d3a2a0c3e6778e3b1f2949b6b4eddebf963e5 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 20 Mar 2019 23:45:00 +0000 Subject: [PATCH 07/10] daemon: init public_port in all ctors Coverity 196596 --- src/daemon/daemon.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 3d1d893ea82..531c080de0f 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -115,6 +115,7 @@ t_daemon::t_daemon(t_daemon && other) { mp_internals = std::move(other.mp_internals); other.mp_internals.reset(nullptr); + public_rpc_port = other.public_rpc_port; } } @@ -125,6 +126,7 @@ t_daemon & t_daemon::operator=(t_daemon && other) { mp_internals = std::move(other.mp_internals); other.mp_internals.reset(nullptr); + public_rpc_port = other.public_rpc_port; } return *this; } From dc20d774595f95deaed30217711ff3d0ae427219 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Thu, 21 Mar 2019 00:23:23 +0000 Subject: [PATCH 08/10] rpc: add miner tx hash to block header response --- src/rpc/core_rpc_server.cpp | 1 + src/rpc/core_rpc_server_commands_defs.h | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index ff55167fef5..2475eda7b28 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1382,6 +1382,7 @@ namespace cryptonote response.num_txes = blk.tx_hashes.size(); response.pow_hash = fill_pow_hash ? string_tools::pod_to_hex(get_block_longhash(blk, height)) : ""; response.long_term_weight = m_core.get_blockchain_storage().get_db().get_block_long_term_weight(height); + response.miner_tx_hash = string_tools::pod_to_hex(cryptonote::get_transaction_hash(blk.miner_tx)); return true; } //------------------------------------------------------------------------------------------------------------------------------ diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index f65c7c8dd90..e50a5938a82 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -84,7 +84,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 4 +#define CORE_RPC_VERSION_MINOR 5 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -1234,6 +1234,7 @@ namespace cryptonote uint64_t num_txes; std::string pow_hash; uint64_t long_term_weight; + std::string miner_tx_hash; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(major_version) @@ -1253,6 +1254,7 @@ namespace cryptonote KV_SERIALIZE(num_txes) KV_SERIALIZE(pow_hash) KV_SERIALIZE_OPT(long_term_weight, (uint64_t)0) + KV_SERIALIZE(miner_tx_hash) END_KV_SERIALIZE_MAP() }; From c23ea7962debe4e66fd3c0e7719117dcb7966d1f Mon Sep 17 00:00:00 2001 From: rbrunner7 Date: Thu, 21 Mar 2019 11:03:24 +0100 Subject: [PATCH 09/10] New interactive daemon command 'print_net_stats': Global traffic stats --- .../include/net/network_throttle-detail.hpp | 3 + contrib/epee/include/net/network_throttle.hpp | 3 +- contrib/epee/src/network_throttle-detail.cpp | 10 ++++ src/common/util.cpp | 36 +++++++++++ src/common/util.h | 2 + src/daemon/command_parser_executor.cpp | 7 +++ src/daemon/command_parser_executor.h | 2 + src/daemon/command_server.cpp | 5 ++ src/daemon/rpc_command_executor.cpp | 60 +++++++++++++++++++ src/daemon/rpc_command_executor.h | 2 + src/rpc/core_rpc_server.cpp | 17 ++++++ src/rpc/core_rpc_server.h | 2 + src/rpc/core_rpc_server_commands_defs.h | 35 ++++++++++- 13 files changed, 182 insertions(+), 2 deletions(-) diff --git a/contrib/epee/include/net/network_throttle-detail.hpp b/contrib/epee/include/net/network_throttle-detail.hpp index d7f2cc37aae..353ae0c0c38 100644 --- a/contrib/epee/include/net/network_throttle-detail.hpp +++ b/contrib/epee/include/net/network_throttle-detail.hpp @@ -66,6 +66,8 @@ class network_throttle : public i_network_throttle { network_time_seconds m_last_sample_time; // time of last history[0] - so we know when to rotate the buffer network_time_seconds m_start_time; // when we were created bool m_any_packet_yet; // did we yet got any packet to count + uint64_t m_total_packets; + uint64_t m_total_bytes; std::string m_name; // my name for debug and logs std::string m_nameshort; // my name for debug and logs (used in log file name) @@ -95,6 +97,7 @@ class network_throttle : public i_network_throttle { virtual size_t get_recommended_size_of_planned_transport() const; ///< what should be the size (bytes) of next data block to be transported virtual size_t get_recommended_size_of_planned_transport_window(double force_window) const; ///< ditto, but for given windows time frame virtual double get_current_speed() const; + virtual void get_stats(uint64_t &total_packets, uint64_t &total_bytes) const; private: virtual network_time_seconds time_to_slot(network_time_seconds t) const { return std::floor( t ); } // convert exact time eg 13.7 to rounded time for slot number in history 13 diff --git a/contrib/epee/include/net/network_throttle.hpp b/contrib/epee/include/net/network_throttle.hpp index 5092241a413..02a28632630 100644 --- a/contrib/epee/include/net/network_throttle.hpp +++ b/contrib/epee/include/net/network_throttle.hpp @@ -152,7 +152,8 @@ class i_network_throttle { virtual size_t get_recommended_size_of_planned_transport() const =0; // what should be the recommended limit of data size that we can transport over current network_throttle in near future virtual double get_time_seconds() const =0; // a timer - virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0; + virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0; + virtual void get_stats(uint64_t &total_packets, uint64_t &total_bytes) const =0; }; diff --git a/contrib/epee/src/network_throttle-detail.cpp b/contrib/epee/src/network_throttle-detail.cpp index f89e7aec0c1..72544cbf65c 100644 --- a/contrib/epee/src/network_throttle-detail.cpp +++ b/contrib/epee/src/network_throttle-detail.cpp @@ -136,6 +136,8 @@ network_throttle::network_throttle(const std::string &nameshort, const std::stri m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle m_last_sample_time = 0; m_history.resize(m_window_size); + m_total_packets = 0; + m_total_bytes = 0; } void network_throttle::set_name(const std::string &name) @@ -192,6 +194,8 @@ void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_s calculate_times_struct cts ; calculate_times(packet_size, cts , false, -1); calculate_times_struct cts2; calculate_times(packet_size, cts2, false, 5); m_history.front().m_size += packet_size; + m_total_packets++; + m_total_bytes += packet_size; std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; std::string history_str = oss.str(); @@ -352,6 +356,12 @@ double network_throttle::get_current_speed() const { return bytes_transferred / ((m_history.size() - 1) * m_slot_size); } +void network_throttle::get_stats(uint64_t &total_packets, uint64_t &total_bytes) const { + total_packets = m_total_packets; + total_bytes = m_total_bytes; +} + + } // namespace } // namespace diff --git a/src/common/util.cpp b/src/common/util.cpp index 80b8a9e8175..728efc294fd 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -80,6 +80,7 @@ using namespace epee; #include #include #include +#include #include #undef MONERO_DEFAULT_LOG_CATEGORY @@ -1063,4 +1064,39 @@ std::string get_nix_version_display_string() strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tm); return std::string(buffer); } + + std::string get_human_readable_bytes(uint64_t bytes) + { + // Use 1024 for "kilo", 1024*1024 for "mega" and so on instead of the more modern and standard-conforming + // 1000, 1000*1000 and so on, to be consistent with other Monero code that also uses base 2 units + struct byte_map + { + const char* const format; + const std::uint64_t bytes; + }; + + static constexpr const byte_map sizes[] = + { + {"%.0f B", 1024}, + {"%.2f KB", 1024 * 1024}, + {"%.2f MB", std::uint64_t(1024) * 1024 * 1024}, + {"%.2f GB", std::uint64_t(1024) * 1024 * 1024 * 1024}, + {"%.2f TB", std::uint64_t(1024) * 1024 * 1024 * 1024 * 1024} + }; + + struct bytes_less + { + bool operator()(const byte_map& lhs, const byte_map& rhs) const noexcept + { + return lhs.bytes < rhs.bytes; + } + }; + + const auto size = std::upper_bound( + std::begin(sizes), std::end(sizes) - 1, byte_map{"", bytes}, bytes_less{} + ); + const std::uint64_t divisor = size->bytes / 1024; + return (boost::format(size->format) % (double(bytes) / divisor)).str(); + } + } diff --git a/src/common/util.h b/src/common/util.h index ef2305bf498..77a5a9af623 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -244,4 +244,6 @@ namespace tools void closefrom(int fd); std::string get_human_readable_timestamp(uint64_t ts); + + std::string get_human_readable_bytes(uint64_t bytes); } diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index b324ab99dc6..17b945c9aa9 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -127,6 +127,13 @@ bool t_command_parser_executor::print_connections(const std::vector return m_executor.print_connections(); } +bool t_command_parser_executor::print_net_stats(const std::vector& args) +{ + if (!args.empty()) return false; + + return m_executor.print_net_stats(); +} + bool t_command_parser_executor::print_blockchain_info(const std::vector& args) { if(!args.size()) diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index bec6e4522d0..09801864236 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -148,6 +148,8 @@ class t_command_parser_executor final bool prune_blockchain(const std::vector& args); bool check_blockchain_pruning(const std::vector& args); + + bool print_net_stats(const std::vector& args); }; } // namespace daemonize diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 94e4a8bf100..69ad6ff1021 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -77,6 +77,11 @@ t_command_server::t_command_server( , std::bind(&t_command_parser_executor::print_connections, &m_parser, p::_1) , "Print the current connections." ); + m_command_lookup.set_handler( + "print_net_stats" + , std::bind(&t_command_parser_executor::print_net_stats, &m_parser, p::_1) + , "Print network statistics." + ); m_command_lookup.set_handler( "print_bc" , std::bind(&t_command_parser_executor::print_blockchain_info, &m_parser, p::_1) diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 4ee67f57145..c9ec5109e87 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -627,6 +627,66 @@ bool t_rpc_command_executor::print_connections() { return true; } +bool t_rpc_command_executor::print_net_stats() +{ + cryptonote::COMMAND_RPC_GET_NET_STATS::request net_stats_req; + cryptonote::COMMAND_RPC_GET_NET_STATS::response net_stats_res; + cryptonote::COMMAND_RPC_GET_LIMIT::request limit_req; + cryptonote::COMMAND_RPC_GET_LIMIT::response limit_res; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(net_stats_req, net_stats_res, "get_net_stats", fail_message.c_str())) + { + return true; + } + if (!m_rpc_client->json_rpc_request(limit_req, limit_res, "get_limit", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_net_stats(net_stats_req, net_stats_res) || net_stats_res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, net_stats_res.status); + return true; + } + if (!m_rpc_server->on_get_limit(limit_req, limit_res) || limit_res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, limit_res.status); + return true; + } + } + + uint64_t seconds = (uint64_t)time(NULL) - net_stats_res.start_time; + uint64_t average = seconds > 0 ? net_stats_res.total_bytes_in / seconds : 0; + uint64_t limit = limit_res.limit_down * 1024; // convert to bytes, as limits are always kB/s + double percent = (double)average / (double)limit * 100.0; + tools::success_msg_writer() << boost::format("Received %u bytes (%s) in %u packets, average %s/s = %.2f%% of the limit of %s/s") + % net_stats_res.total_bytes_in + % tools::get_human_readable_bytes(net_stats_res.total_bytes_in) + % net_stats_res.total_packets_in + % tools::get_human_readable_bytes(average) + % percent + % tools::get_human_readable_bytes(limit); + + average = seconds > 0 ? net_stats_res.total_bytes_out / seconds : 0; + limit = limit_res.limit_up * 1024; + percent = (double)average / (double)limit * 100.0; + tools::success_msg_writer() << boost::format("Sent %u bytes (%s) in %u packets, average %s/s = %.2f%% of the limit of %s/s") + % net_stats_res.total_bytes_out + % tools::get_human_readable_bytes(net_stats_res.total_bytes_out) + % net_stats_res.total_packets_out + % tools::get_human_readable_bytes(average) + % percent + % tools::get_human_readable_bytes(limit); + + return true; +} + bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, uint64_t end_block_index) { cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request req; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response res; diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 423132b792a..3c2686b3ff4 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -160,6 +160,8 @@ class t_rpc_command_executor final { bool prune_blockchain(); bool check_blockchain_pruning(); + + bool print_net_stats(); }; } // namespace daemonize diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index ff55167fef5..d976a052efa 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -260,6 +260,23 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_net_stats(const COMMAND_RPC_GET_NET_STATS::request& req, COMMAND_RPC_GET_NET_STATS::response& res, const connection_context *ctx) + { + PERF_TIMER(on_get_net_stats); + // No bootstrap daemon check: Only ever get stats about local server + res.start_time = (uint64_t)m_core.get_start_time(); + { + CRITICAL_REGION_LOCAL(epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in); + epee::net_utils::network_throttle_manager::get_global_throttle_in().get_stats(res.total_packets_in, res.total_bytes_in); + } + { + CRITICAL_REGION_LOCAL(epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_out); + epee::net_utils::network_throttle_manager::get_global_throttle_out().get_stats(res.total_packets_out, res.total_bytes_out); + } + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ class pruned_transaction { transaction& tx; public: diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index fe066b31bcb..8f5d83f1b4c 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -115,6 +115,7 @@ namespace cryptonote MAP_URI_AUTO_JON2_IF("/stop_daemon", on_stop_daemon, COMMAND_RPC_STOP_DAEMON, !m_restricted) MAP_URI_AUTO_JON2("/get_info", on_get_info, COMMAND_RPC_GET_INFO) MAP_URI_AUTO_JON2("/getinfo", on_get_info, COMMAND_RPC_GET_INFO) + MAP_URI_AUTO_JON2_IF("/get_net_stats", on_get_net_stats, COMMAND_RPC_GET_NET_STATS, !m_restricted) MAP_URI_AUTO_JON2("/get_limit", on_get_limit, COMMAND_RPC_GET_LIMIT) MAP_URI_AUTO_JON2_IF("/set_limit", on_set_limit, COMMAND_RPC_SET_LIMIT, !m_restricted) MAP_URI_AUTO_JON2_IF("/out_peers", on_out_peers, COMMAND_RPC_OUT_PEERS, !m_restricted) @@ -179,6 +180,7 @@ namespace cryptonote bool on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res, const connection_context *ctx = NULL); bool on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res, const connection_context *ctx = NULL); bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, const connection_context *ctx = NULL); + bool on_get_net_stats(const COMMAND_RPC_GET_NET_STATS::request& req, COMMAND_RPC_GET_NET_STATS::response& res, const connection_context *ctx = NULL); bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res, const connection_context *ctx = NULL); bool on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res, const connection_context *ctx = NULL); bool on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res, const connection_context *ctx = NULL); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index f65c7c8dd90..dcfcbb2494b 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -84,7 +84,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 4 +#define CORE_RPC_VERSION_MINOR 5 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -1013,6 +1013,39 @@ namespace cryptonote }; + //----------------------------------------------- + struct COMMAND_RPC_GET_NET_STATS + { + struct request_t + { + + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init request; + + + struct response_t + { + std::string status; + uint64_t start_time; + uint64_t total_packets_in; + uint64_t total_bytes_in; + uint64_t total_packets_out; + uint64_t total_bytes_out; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(start_time) + KV_SERIALIZE(total_packets_in) + KV_SERIALIZE(total_bytes_in) + KV_SERIALIZE(total_packets_out) + KV_SERIALIZE(total_bytes_out) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init response; + }; + //----------------------------------------------- struct COMMAND_RPC_STOP_MINING { From 1730a44f9002d51f376a0aadc1804836a2da70df Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 19 Mar 2019 16:30:17 +0000 Subject: [PATCH 10/10] core: improve block rate monitor trigger probabilities The original intent of one false positive a week on average was not met, since what we really want is not the probability of having N blocks in T seconds, but either N blocks of fewer in T seconds, or N blocks or more in T seconds. Some of this could be cached since it calculates the same fairly complex floating point values, but it seems pretty fast already. --- src/cryptonote_core/cryptonote_core.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 387203cc0a7..da413bbe248 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1784,12 +1784,28 @@ namespace cryptonote return f; } //----------------------------------------------------------------------------------------------- - static double probability(unsigned int blocks, unsigned int expected) + static double probability1(unsigned int blocks, unsigned int expected) { // https://www.umass.edu/wsp/resources/poisson/#computing return pow(expected, blocks) / (factorial(blocks) * exp(expected)); } //----------------------------------------------------------------------------------------------- + static double probability(unsigned int blocks, unsigned int expected) + { + double p = 0.0; + if (blocks <= expected) + { + for (unsigned int b = 0; b <= blocks; ++b) + p += probability1(b, expected); + } + else if (blocks > expected) + { + for (unsigned int b = blocks; b <= expected * 3 /* close enough */; ++b) + p += probability1(b, expected); + } + return p; + } + //----------------------------------------------------------------------------------------------- bool core::check_block_rate() { if (m_offline || m_target_blockchain_height > get_current_blockchain_height())