Merge commit '1ed6441' into LokiMergeUpstream

Dockerfile

@@ -25,10 +25,13 @@ RUN set -ex && \
 
 WORKDIR /usr/local
 
+ENV CFLAGS='-fPIC'
+ENV CXXFLAGS='-fPIC'
+
 #Cmake
-ARG CMAKE_VERSION=3.13.0
-ARG CMAKE_VERSION_DOT=v3.13
-ARG CMAKE_HASH=4058b2f1a53c026564e8936698d56c3b352d90df067b195cb749a97a3d273c90 
+ARG CMAKE_VERSION=3.14.0
+ARG CMAKE_VERSION_DOT=v3.14
+ARG CMAKE_HASH=aa76ba67b3c2af1946701f847073f4652af5cbd9f141f221c97af99127e75502
 RUN set -ex \
     && curl -s -O https://cmake.org/files/${CMAKE_VERSION_DOT}/cmake-${CMAKE_VERSION}.tar.gz \
     && echo "${CMAKE_HASH}  cmake-${CMAKE_VERSION}.tar.gz" | sha256sum -c \
@@ -39,41 +42,41 @@ RUN set -ex \
     && make install
 
 ## Boost
-ARG BOOST_VERSION=1_68_0
-ARG BOOST_VERSION_DOT=1.68.0
-ARG BOOST_HASH=7f6130bc3cf65f56a618888ce9d5ea704fa10b462be126ad053e80e553d6d8b7
+ARG BOOST_VERSION=1_69_0
+ARG BOOST_VERSION_DOT=1.69.0
+ARG BOOST_HASH=8f32d4617390d1c2d16f26a27ab60d97807b35440d45891fa340fc2648b04406
 RUN set -ex \
     && curl -s -L -o  boost_${BOOST_VERSION}.tar.bz2 https://dl.bintray.com/boostorg/release/${BOOST_VERSION_DOT}/source/boost_${BOOST_VERSION}.tar.bz2 \
     && echo "${BOOST_HASH}  boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \
     && tar -xvf boost_${BOOST_VERSION}.tar.bz2 \
     && cd boost_${BOOST_VERSION} \
     && ./bootstrap.sh \
-    && ./b2 --build-type=minimal link=static runtime-link=static --with-chrono --with-date_time --with-filesystem --with-program_options --with-regex --with-serialization --with-system --with-thread --with-locale threading=multi threadapi=pthread cflags="-fPIC" cxxflags="-fPIC" stage
+    && ./b2 --build-type=minimal link=static runtime-link=static --with-chrono --with-date_time --with-filesystem --with-program_options --with-regex --with-serialization --with-system --with-thread --with-locale threading=multi threadapi=pthread cflags="$CFLAGS" cxxflags="$CXXFLAGS" stage
 ENV BOOST_ROOT /usr/local/boost_${BOOST_VERSION}
 
 # OpenSSL
-ARG OPENSSL_VERSION=1.1.0j
-ARG OPENSSL_HASH=31bec6c203ce1a8e93d5994f4ed304c63ccf07676118b6634edded12ad1b3246
+ARG OPENSSL_VERSION=1.1.1b
+ARG OPENSSL_HASH=5c557b023230413dfb0756f3137a13e6d726838ccd1430888ad15bfb2b43ea4b
 RUN set -ex \
     && curl -s -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \
     && echo "${OPENSSL_HASH}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c \
     && tar -xzf openssl-${OPENSSL_VERSION}.tar.gz \
     && cd openssl-${OPENSSL_VERSION} \
-    && ./Configure linux-x86_64 no-shared --static -fPIC \
+    && ./Configure linux-x86_64 no-shared --static "$CFLAGS" \
     && make build_generated \
     && make libcrypto.a \
     && make install
 ENV OPENSSL_ROOT_DIR=/usr/local/openssl-${OPENSSL_VERSION}
 
 # ZMQ
-ARG ZMQ_VERSION=v4.2.5
-ARG ZMQ_HASH=d062edd8c142384792955796329baf1e5a3377cd
+ARG ZMQ_VERSION=v4.3.1
+ARG ZMQ_HASH=2cb1240db64ce1ea299e00474c646a2453a8435b
 RUN set -ex \
     && git clone https://github.com/zeromq/libzmq.git -b ${ZMQ_VERSION} --depth=1 \
     && cd libzmq \
     && test `git rev-parse HEAD` = ${ZMQ_HASH} || exit 1 \
     && ./autogen.sh \
-    && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \
+    && ./configure --enable-static --disable-shared \
     && make \
     && make install \
     && ldconfig
@@ -99,39 +102,39 @@ RUN set -ex \
     && mv *.hpp /usr/local/include
 
 # Readline
-ARG READLINE_VERSION=7.0
-ARG READLINE_HASH=750d437185286f40a369e1e4f4764eda932b9459b5ec9a731628393dd3d32334
+ARG READLINE_VERSION=8.0
+ARG READLINE_HASH=e339f51971478d369f8a053a330a190781acb9864cf4c541060f12078948e461
 RUN set -ex \
     && curl -s -O https://ftp.gnu.org/gnu/readline/readline-${READLINE_VERSION}.tar.gz \
     && echo "${READLINE_HASH}  readline-${READLINE_VERSION}.tar.gz" | sha256sum -c \
     && tar -xzf readline-${READLINE_VERSION}.tar.gz \
     && cd readline-${READLINE_VERSION} \
-    && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure \
+    && ./configure \
     && make \
     && make install
 
 # Sodium
-ARG SODIUM_VERSION=1.0.16
-ARG SODIUM_HASH=675149b9b8b66ff44152553fb3ebf9858128363d
+ARG SODIUM_VERSION=1.0.17
+ARG SODIUM_HASH=b732443c442239c2e0184820e9b23cca0de0828c
 RUN set -ex \
     && git clone https://github.com/jedisct1/libsodium.git -b ${SODIUM_VERSION} --depth=1 \
     && cd libsodium \
     && test `git rev-parse HEAD` = ${SODIUM_HASH} || exit 1 \
     && ./autogen.sh \
-    && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure \
+    && ./configure \
     && make \
     && make check \
     && make install
 
 # Udev
-ARG UDEV_VERSION=v3.2.6
-ARG UDEV_HASH=0c35b136c08d64064efa55087c54364608e65ed6
+ARG UDEV_VERSION=v3.2.7
+ARG UDEV_HASH=4758e346a14126fc3a964de5831e411c27ebe487
 RUN set -ex \
     && git clone https://github.com/gentoo/eudev -b ${UDEV_VERSION} \
     && cd eudev \
     && test `git rev-parse HEAD` = ${UDEV_HASH} || exit 1 \
     && ./autogen.sh \
-    && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared \
+    && ./configure --disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared \
     && make \
     && make install
 
@@ -143,7 +146,7 @@ RUN set -ex \
     && cd libusb \
     && test `git rev-parse HEAD` = ${USB_HASH} || exit 1 \
     && ./autogen.sh \
-    && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-shared \
+    && ./configure --disable-shared \
     && make \
     && make install
 
@@ -155,20 +158,20 @@ RUN set -ex \
     && cd hidapi \
     && test `git rev-parse HEAD` = ${HIDAPI_HASH} || exit 1 \
     && ./bootstrap \
-    && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \
+    && ./configure --enable-static --disable-shared \
     && make \
     && make install
 
 # Protobuf
-ARG PROTOBUF_VERSION=v3.6.1
-ARG PROTOBUF_HASH=48cb18e5c419ddd23d9badcfe4e9df7bde1979b2
+ARG PROTOBUF_VERSION=v3.7.0
+ARG PROTOBUF_HASH=582743bf40c5d3639a70f98f183914a2c0cd0680
 RUN set -ex \
     && git clone https://github.com/protocolbuffers/protobuf -b ${PROTOBUF_VERSION} \
     && cd protobuf \
     && test `git rev-parse HEAD` = ${PROTOBUF_HASH} || exit 1 \
     && git submodule update --init --recursive \
     && ./autogen.sh \
-    && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \
+    && ./configure --enable-static --disable-shared \
     && make \
     && make install \
     && ldconfig

contrib/epee/include/net/network_throttle-detail.hpp

@@ -66,6 +66,8 @@ class network_throttle : public i_network_throttle {
 		network_time_seconds m_last_sample_time; // time of last history[0] - so we know when to rotate the buffer
 		network_time_seconds m_start_time; // when we were created
 		bool m_any_packet_yet; // did we yet got any packet to count
+		uint64_t m_total_packets;
+		uint64_t m_total_bytes;
 
 		std::string m_name; // my name for debug and logs
 		std::string m_nameshort; // my name for debug and logs (used in log file name)
@@ -95,6 +97,7 @@ class network_throttle : public i_network_throttle {
 		virtual size_t get_recommended_size_of_planned_transport() const; ///< what should be the size (bytes) of next data block to be transported
 		virtual size_t get_recommended_size_of_planned_transport_window(double force_window) const;  ///< ditto, but for given windows time frame
 		virtual double get_current_speed() const;
+		virtual void get_stats(uint64_t &total_packets, uint64_t &total_bytes) const;
 
 	private:
 		virtual network_time_seconds time_to_slot(network_time_seconds t) const { return std::floor( t ); } // convert exact time eg 13.7 to rounded time for slot number in history 13

contrib/epee/include/net/network_throttle.hpp

@@ -152,7 +152,8 @@ class i_network_throttle {
 		virtual size_t get_recommended_size_of_planned_transport() const =0; // what should be the recommended limit of data size that we can transport over current network_throttle in near future
 
 		virtual double get_time_seconds() const =0; // a timer
-        virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0;
+		virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0;
+		virtual void get_stats(uint64_t &total_packets, uint64_t &total_bytes) const =0;
 
 
 };

contrib/epee/include/storages/portable_storage_base.h

@@ -82,6 +82,7 @@ namespace epee
     struct array_entry_t
     {
       array_entry_t():m_it(m_array.end()){}        
+      array_entry_t(const array_entry_t& other):m_array(other.m_array), m_it(m_array.end()){}
 
       const t_entry_type* get_first_val() const 
       {

contrib/epee/src/network_throttle-detail.cpp

@@ -136,6 +136,8 @@ network_throttle::network_throttle(const std::string &nameshort, const std::stri
 	m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle
 	m_last_sample_time = 0;
 	m_history.resize(m_window_size);
+	m_total_packets = 0;
+	m_total_bytes = 0;
 }
 
 void network_throttle::set_name(const std::string &name) 
@@ -192,6 +194,8 @@ void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_s
 	calculate_times_struct cts ;  calculate_times(packet_size, cts , false, -1);
 	calculate_times_struct cts2;  calculate_times(packet_size, cts2, false, 5);
 	m_history.front().m_size += packet_size;
+	m_total_packets++;
+	m_total_bytes += packet_size;
 
 	std::ostringstream oss; oss << "["; 	for (auto sample: m_history) oss << sample.m_size << " ";	 oss << "]" << std::ends;
 	std::string history_str = oss.str();
@@ -352,6 +356,12 @@ double network_throttle::get_current_speed() const {
 	return bytes_transferred / ((m_history.size() - 1) * m_slot_size);
 }
 
+void network_throttle::get_stats(uint64_t &total_packets, uint64_t &total_bytes) const {
+	total_packets = m_total_packets;
+	total_bytes = m_total_bytes;
+}
+
+
 } // namespace
 } // namespace
 

src/common/dns_utils.cpp

@@ -33,9 +33,9 @@
 
 #include <stdlib.h>
 #include "include_base_utils.h"
+#include "common/threadpool.h"
 #include <random>
 #include <boost/thread/mutex.hpp>
-#include <boost/thread/thread.hpp>
 #include <boost/algorithm/string/join.hpp>
 #include <boost/optional.hpp>
 using namespace epee;
@@ -527,16 +527,16 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std
   size_t first_index = dis(gen);
 
   // send all requests in parallel
-  std::vector<boost::thread> threads(dns_urls.size());
   std::deque<bool> avail(dns_urls.size(), false), valid(dns_urls.size(), false);
+  tools::threadpool& tpool = tools::threadpool::getInstance();
+  tools::threadpool::waiter waiter;
   for (size_t n = 0; n < dns_urls.size(); ++n)
   {
-    threads[n] = boost::thread([n, dns_urls, &records, &avail, &valid](){
+    tpool.submit(&waiter,[n, dns_urls, &records, &avail, &valid](){
       records[n] = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); 
     });
   }
-  for (size_t n = 0; n < dns_urls.size(); ++n)
-    threads[n].join();
+  waiter.wait(&tpool);
 
   size_t cur_index = first_index;
   do

src/common/util.cpp

@@ -81,6 +81,7 @@ using namespace epee;
 #include <boost/filesystem.hpp>
 #include <boost/algorithm/string.hpp>
 #include <boost/asio.hpp>
+#include <boost/format.hpp>
 #include <openssl/sha.h>
 #include "i18n.h"
 
@@ -1081,4 +1082,38 @@ std::string get_nix_version_display_string()
       return std::to_string((uint64_t)(ts / (3600 * 24 * 30.5))) + tr(" months");
     return tr("a long time");
   }
+
+  std::string get_human_readable_bytes(uint64_t bytes)
+  {
+    // Use 1024 for "kilo", 1024*1024 for "mega" and so on instead of the more modern and standard-conforming
+    // 1000, 1000*1000 and so on, to be consistent with other Monero code that also uses base 2 units
+    struct byte_map
+    {
+        const char* const format;
+        const std::uint64_t bytes;
+    };
+
+    static constexpr const byte_map sizes[] =
+    {
+        {"%.0f B", 1024},
+        {"%.2f KB", 1024 * 1024},
+        {"%.2f MB", std::uint64_t(1024) * 1024 * 1024},
+        {"%.2f GB", std::uint64_t(1024) * 1024 * 1024 * 1024},
+        {"%.2f TB", std::uint64_t(1024) * 1024 * 1024 * 1024 * 1024}
+    };
+
+    struct bytes_less
+    {
+        bool operator()(const byte_map& lhs, const byte_map& rhs) const noexcept
+        {
+            return lhs.bytes < rhs.bytes;
+        }
+    };
+
+    const auto size = std::upper_bound(
+        std::begin(sizes), std::end(sizes) - 1, byte_map{"", bytes}, bytes_less{}
+    );
+    const std::uint64_t divisor = size->bytes / 1024;
+    return (boost::format(size->format) % (double(bytes) / divisor)).str();
+  }
 }

src/common/util.h

@@ -246,4 +246,5 @@ namespace tools
 
   std::string get_human_readable_timestamp(uint64_t ts);
   std::string get_human_readable_timespan(std::chrono::seconds seconds);
+  std::string get_human_readable_bytes(uint64_t bytes);
 }

src/cryptonote_core/cryptonote_core.cpp

@@ -1971,12 +1971,28 @@ namespace cryptonote
     return f;
   }
   //-----------------------------------------------------------------------------------------------
-  static double probability(unsigned int blocks, unsigned int expected)
+  static double probability1(unsigned int blocks, unsigned int expected)
   {
     // https://www.umass.edu/wsp/resources/poisson/#computing
     return pow(expected, blocks) / (factorial(blocks) * exp(expected));
   }
   //-----------------------------------------------------------------------------------------------
+  static double probability(unsigned int blocks, unsigned int expected)
+  {
+    double p = 0.0;
+    if (blocks <= expected)
+    {
+      for (unsigned int b = 0; b <= blocks; ++b)
+        p += probability1(b, expected);
+    }
+    else if (blocks > expected)
+    {
+      for (unsigned int b = blocks; b <= expected * 3 /* close enough */; ++b)
+        p += probability1(b, expected);
+    }
+    return p;
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::check_block_rate()
   {
     if (m_offline || m_target_blockchain_height > get_current_blockchain_height())

src/daemon/command_parser_executor.cpp

@@ -128,6 +128,13 @@ bool t_command_parser_executor::print_connections(const std::vector<std::string>
   return m_executor.print_connections();
 }
 
+bool t_command_parser_executor::print_net_stats(const std::vector<std::string>& args)
+{
+  if (!args.empty()) return false;
+
+  return m_executor.print_net_stats();
+}
+
 bool t_command_parser_executor::print_blockchain_info(const std::vector<std::string>& args)
 {
   if(!args.size())

src/daemon/command_parser_executor.h

@@ -160,6 +160,8 @@ class t_command_parser_executor final
   bool prune_blockchain(const std::vector<std::string>& args);
 
   bool check_blockchain_pruning(const std::vector<std::string>& args);
+
+  bool print_net_stats(const std::vector<std::string>& args);
 };
 
 } // namespace daemonize

src/daemon/command_server.cpp

@@ -81,6 +81,11 @@ t_command_server::t_command_server(
     , std::bind(&t_command_parser_executor::print_connections, &m_parser, p::_1)
     , "Print the current connections."
     );
+  m_command_lookup.set_handler(
+      "print_net_stats"
+    , std::bind(&t_command_parser_executor::print_net_stats, &m_parser, p::_1)
+    , "Print network statistics."
+    );
   m_command_lookup.set_handler(
       "print_bc"
     , std::bind(&t_command_parser_executor::print_blockchain_info, &m_parser, p::_1)

src/daemon/daemon.cpp

@@ -116,6 +116,7 @@ t_daemon::t_daemon(t_daemon && other)
   {
     mp_internals = std::move(other.mp_internals);
     other.mp_internals.reset(nullptr);
+    public_rpc_port = other.public_rpc_port;
   }
 }
 
@@ -126,6 +127,7 @@ t_daemon & t_daemon::operator=(t_daemon && other)
   {
     mp_internals = std::move(other.mp_internals);
     other.mp_internals.reset(nullptr);
+    public_rpc_port = other.public_rpc_port;
   }
   return *this;
 }