From ddb8df47f6acf217c37719ab62b94b17455f5aec Mon Sep 17 00:00:00 2001 From: Scott Sallinen Date: Sat, 24 Nov 2018 00:33:56 -0800 Subject: [PATCH 01/15] p2p unlinkable workaround Solves https://github.com/steemit/steem/issues/3180 --- libraries/plugins/p2p/p2p_plugin.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libraries/plugins/p2p/p2p_plugin.cpp b/libraries/plugins/p2p/p2p_plugin.cpp index 8c1a73f171..5bc114c244 100644 --- a/libraries/plugins/p2p/p2p_plugin.cpp +++ b/libraries/plugins/p2p/p2p_plugin.cpp @@ -227,7 +227,12 @@ bool p2p_plugin_impl::handle_block( const graphene::net::block_message& blk_msg, ("e", e.to_detail_string()) ("head", head_block_num)); elog("Error when pushing block:\n${e}", ("e", e.to_detail_string())); - throw; + if (e.code() == 4080000) { + elog("Rethrowing as graphene::net exception"); + FC_THROW_EXCEPTION(graphene::net::unlinkable_block_exception, "Error when pushing block:\n${e}", ("e", e.to_detail_string())); + } else { + throw; + } } } else From e4a4699c2f2d875c9123ca284ec54240f9412da1 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Mon, 10 Dec 2018 13:53:06 -0500 Subject: [PATCH 02/15] Track depth of recursion while binary deserializing, do not pre-allocate object up front --- .../chain/include/steem/chain/steem_fwd.hpp | 8 +- .../steem/chain/steem_object_types.hpp | 19 +- libraries/fc/include/fc/container/flat.hpp | 33 ++- .../fc/include/fc/container/flat_fwd.hpp | 10 +- libraries/fc/include/fc/fixed_string.hpp | 5 +- .../fc/include/fc/interprocess/container.hpp | 16 +- libraries/fc/include/fc/io/enum_type.hpp | 6 +- libraries/fc/include/fc/io/raw.hpp | 233 +++++++++++------- libraries/fc/include/fc/io/raw_fwd.hpp | 77 +++--- libraries/fc/include/fc/io/raw_variant.hpp | 30 ++- libraries/fc/include/fc/network/ip.hpp | 12 +- libraries/fc/include/fc/real128.hpp | 2 +- libraries/fc/include/fc/uint128.hpp | 2 +- .../include/steem/protocol/asset_symbol.hpp | 2 +- .../include/steem/protocol/fixed_string.hpp | 5 +- .../include/steem/protocol/legacy_asset.hpp | 6 +- .../include/steem/protocol/types_fwd.hpp | 14 +- 17 files changed, 279 insertions(+), 201 deletions(-) diff --git a/libraries/chain/include/steem/chain/steem_fwd.hpp b/libraries/chain/include/steem/chain/steem_fwd.hpp index 7a8e9fdfbb..9f339eac74 100644 --- a/libraries/chain/include/steem/chain/steem_fwd.hpp +++ b/libraries/chain/include/steem/chain/steem_fwd.hpp @@ -15,21 +15,21 @@ namespace fc { namespace raw { template inline void pack( Stream& s, const chainbase::oid& id ); template -inline void unpack( Stream& s, chainbase::oid& id ); +inline void unpack( Stream& s, chainbase::oid& id, uint32_t depth = 0 ); template inline void pack( Stream& s, const chainbase::shared_string& ss ); template -inline void unpack( Stream& s, chainbase::shared_string& ss ); +inline void unpack( Stream& s, chainbase::shared_string& ss, uint32_t depth = 0 ); template void pack( Stream& s, const boost::interprocess::deque< E, A >& value ); template -void unpack( Stream& s, boost::interprocess::deque< E, A >& value ); +void unpack( Stream& s, boost::interprocess::deque< E, A >& value, uint32_t depth = 0 ); template void pack( Stream& s, const boost::interprocess::flat_map< K, V, C, A >& value ); template -void unpack( Stream& s, boost::interprocess::flat_map< K, V, C, A >& value ); +void unpack( Stream& s, boost::interprocess::flat_map< K, V, C, A >& value, uint32_t depth = 0 ); } } diff --git a/libraries/chain/include/steem/chain/steem_object_types.hpp b/libraries/chain/include/steem/chain/steem_object_types.hpp index 16807d6112..79d71c6066 100644 --- a/libraries/chain/include/steem/chain/steem_object_types.hpp +++ b/libraries/chain/include/steem/chain/steem_object_types.hpp @@ -207,7 +207,7 @@ void pack( Stream& s, const chainbase::oid& id ) } template -void unpack( Stream& s, chainbase::oid& id ) +void unpack( Stream& s, chainbase::oid& id, uint32_t ) { s.read( (char*)&id._id, sizeof(id._id)); } @@ -220,10 +220,11 @@ void pack( Stream& s, const chainbase::shared_string& ss ) } template< typename Stream > -void unpack( Stream& s, chainbase::shared_string& ss ) +void unpack( Stream& s, chainbase::shared_string& ss, uint32_t depth ) { + depth++; std::string str; - fc::raw::unpack( s, str ); + fc::raw::unpack( s, str, depth ); steem::chain::from_string( ss, str ); } @@ -237,11 +238,12 @@ void pack( Stream& s, const boost::interprocess::deque& dq ) } template< typename Stream, typename E, typename A > -void unpack( Stream& s, boost::interprocess::deque& dq ) +void unpack( Stream& s, boost::interprocess::deque& dq, uint32_t depth ) { + depth++; // This could be optimized std::vector temp; - unpack( s, temp ); + unpack( s, temp, depth ); std::copy( temp.begin(), temp.end(), std::back_inserter(dq) ); } @@ -259,17 +261,18 @@ void pack( Stream& s, const boost::interprocess::flat_map< K, V, C, A >& value ) } template< typename Stream, typename K, typename V, typename C, typename A > -void unpack( Stream& s, boost::interprocess::flat_map< K, V, C, A >& value ) +void unpack( Stream& s, boost::interprocess::flat_map< K, V, C, A >& value, uint32_t depth ) { + depth++; unsigned_int size; - unpack( s, size ); + unpack( s, size, depth ); value.clear(); FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); value.reserve(size.value); for( uint32_t i = 0; i < size.value; ++i ) { std::pair tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, depth ); value.insert( std::move(tmp) ); } } diff --git a/libraries/fc/include/fc/container/flat.hpp b/libraries/fc/include/fc/container/flat.hpp index 48fdf75c29..fce20d4694 100644 --- a/libraries/fc/include/fc/container/flat.hpp +++ b/libraries/fc/include/fc/container/flat.hpp @@ -18,15 +18,16 @@ namespace fc { } } template - inline void unpack( Stream& s, flat_set& value ) { - unsigned_int size; unpack( s, size ); + inline void unpack( Stream& s, flat_set& value, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + unsigned_int size; unpack( s, size, depth ); value.clear(); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); - value.reserve(size.value); for( uint32_t i = 0; i < size.value; ++i ) { T tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, depth ); value.insert( std::move(tmp) ); } } @@ -41,16 +42,17 @@ namespace fc { } } template - inline void unpack( Stream& s, flat_map& value ) + inline void unpack( Stream& s, flat_map& value, uint32_t depth ) { - unsigned_int size; unpack( s, size ); + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + unsigned_int size; unpack( s, size, depth ); value.clear(); FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); - value.reserve(size.value); for( uint32_t i = 0; i < size.value; ++i ) { std::pair tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, depth ); value.insert( std::move(tmp) ); } } @@ -71,13 +73,18 @@ namespace fc { } template - void unpack( Stream& s, bip::vector& value ) { + void unpack( Stream& s, bip::vector& value, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); unsigned_int size; - unpack( s, size ); - value.resize( size ); + unpack( s, size, depth ); if( !std::is_fundamental::value ) { - for( auto& item : value ) - unpack( s, item ); + for ( size_t i = 0; i < size.value; i++ ) + { + T tmp; + unpack( s, tmp, depth ); + value.emplace_back( std::move( tmp ) ); + } } else { s.read( (char*)value.data(), value.size() ); } diff --git a/libraries/fc/include/fc/container/flat_fwd.hpp b/libraries/fc/include/fc/container/flat_fwd.hpp index 98dd9547be..20d5cb175b 100644 --- a/libraries/fc/include/fc/container/flat_fwd.hpp +++ b/libraries/fc/include/fc/container/flat_fwd.hpp @@ -13,17 +13,21 @@ namespace fc { template void pack( Stream& s, const flat_set& value ); template - void unpack( Stream& s, flat_set& value ); + void unpack( Stream& s, flat_set& value, uint32_t depth = 0 ); template void pack( Stream& s, const flat_map& value ); template - void unpack( Stream& s, flat_map& value ) ; + void unpack( Stream& s, flat_map& value, uint32_t depth = 0 ) ; + template + void pack( Stream& s, const flat_map& value ); + template + void unpack( Stream& s, flat_map& value, uint32_t depth = 0 ); template void pack( Stream& s, const bip::vector& value ); template - void unpack( Stream& s, bip::vector& value ); + void unpack( Stream& s, bip::vector& value, uint32_t depth = 0 ); } // namespace raw } // fc diff --git a/libraries/fc/include/fc/fixed_string.hpp b/libraries/fc/include/fc/fixed_string.hpp index 6a30d58a1b..e73c34084a 100644 --- a/libraries/fc/include/fc/fixed_string.hpp +++ b/libraries/fc/include/fc/fixed_string.hpp @@ -105,9 +105,10 @@ namespace fc { } template - inline void unpack( Stream& s, fc::fixed_string& u ) { + inline void unpack( Stream& s, fc::fixed_string& u, uint32_t depth ) { + depth++; unsigned_int size; - fc::raw::unpack( s, size ); + fc::raw::unpack( s, size, depth ); if( size.value > 0 ) { if( size.value > sizeof(Storage) ) { s.read( (char*)&u.data, sizeof(Storage) ); diff --git a/libraries/fc/include/fc/interprocess/container.hpp b/libraries/fc/include/fc/interprocess/container.hpp index cd9259b6b2..fa1c240dab 100644 --- a/libraries/fc/include/fc/interprocess/container.hpp +++ b/libraries/fc/include/fc/interprocess/container.hpp @@ -135,12 +135,18 @@ namespace fc { } } template - inline void unpack( Stream& s, bip::vector& value ) { + inline void unpack( Stream& s, bip::vector& value, uint32_t depth = 0 ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); unsigned_int size; - unpack( s, size ); - value.clear(); value.resize(size); - for( auto& item : value ) - fc::raw::unpack( s, item ); + unpack( s, size, depth ); + value.clear(); + for ( size_t i = 0; i < size.value; i++ ) + { + T tmp; + fc::raw::unpack( s, tmp, depth ); + value.emplace_back( std::move( tmp ) ); + } } } diff --git a/libraries/fc/include/fc/io/enum_type.hpp b/libraries/fc/include/fc/io/enum_type.hpp index e41770ced1..fc546acac7 100644 --- a/libraries/fc/include/fc/io/enum_type.hpp +++ b/libraries/fc/include/fc/io/enum_type.hpp @@ -69,10 +69,12 @@ namespace fc } template - inline void unpack( Stream& s, fc::enum_type& tp ) + inline void unpack( Stream& s, fc::enum_type& tp, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); IntType t; - fc::raw::unpack( s, t ); + fc::raw::unpack( s, t, depth ); tp = t; } } diff --git a/libraries/fc/include/fc/io/raw.hpp b/libraries/fc/include/fc/io/raw.hpp index c143915630..a52e2c1360 100644 --- a/libraries/fc/include/fc/io/raw.hpp +++ b/libraries/fc/include/fc/io/raw.hpp @@ -34,16 +34,18 @@ namespace fc { fc::raw::pack( s, e.get_log() ); } template - inline void unpack( Stream& s, fc::exception& e ) + inline void unpack( Stream& s, fc::exception& e, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); int64_t code; std::string name, what; log_messages msgs; - fc::raw::unpack( s, code ); - fc::raw::unpack( s, name ); - fc::raw::unpack( s, what ); - fc::raw::unpack( s, msgs ); + fc::raw::unpack( s, code, depth ); + fc::raw::unpack( s, name, depth ); + fc::raw::unpack( s, what, depth ); + fc::raw::unpack( s, msgs, depth ); e = fc::exception( fc::move(msgs), code, name, what ); } @@ -54,10 +56,12 @@ namespace fc { fc::raw::pack( s, variant(msg) ); } template - inline void unpack( Stream& s, fc::log_message& msg ) + inline void unpack( Stream& s, fc::log_message& msg, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); fc::variant vmsg; - fc::raw::unpack( s, vmsg ); + fc::raw::unpack( s, vmsg, depth ); msg = vmsg.as(); } @@ -68,10 +72,11 @@ namespace fc { } template - inline void unpack( Stream& s, fc::path& tp ) + inline void unpack( Stream& s, fc::path& tp, uint32_t depth ) { + depth++; std::string p; - fc::raw::unpack( s, p ); + fc::raw::unpack( s, p, depth ); tp = p; } @@ -83,7 +88,7 @@ namespace fc { } template - inline void unpack( Stream& s, fc::time_point_sec& tp ) + inline void unpack( Stream& s, fc::time_point_sec& tp, uint32_t ) { try { uint32_t sec; s.read( (char*)&sec, sizeof(sec) ); @@ -98,7 +103,7 @@ namespace fc { } template - inline void unpack( Stream& s, fc::time_point& tp ) + inline void unpack( Stream& s, fc::time_point& tp, uint32_t ) { try { uint64_t usec; s.read( (char*)&usec, sizeof(usec) ); @@ -113,7 +118,7 @@ namespace fc { } template - inline void unpack( Stream& s, fc::microseconds& usec ) + inline void unpack( Stream& s, fc::microseconds& usec, uint32_t ) { try { uint64_t usec_as_int64; s.read( (char*)&usec_as_int64, sizeof(usec_as_int64) ); @@ -127,7 +132,7 @@ namespace fc { } template - inline void unpack( Stream& s, fc::array& v) + inline void unpack( Stream& s, fc::array& v, uint32_t ) { try { s.read( (char*)&v.data[0], N*sizeof(T) ); } FC_RETHROW_EXCEPTIONS( warn, "fc::array", ("type",fc::get_typename::name())("length",N) ) } @@ -139,7 +144,7 @@ namespace fc { } template - inline void unpack( Stream& s, fc::int_array& v) + inline void unpack( Stream& s, fc::int_array& v, uint32_t ) { try { s.read( (char*)&v.data[0], N*sizeof(T) ); } FC_RETHROW_EXCEPTIONS( warn, "fc::int_array", ("type",fc::get_typename::name())("length",N) ) } @@ -151,10 +156,12 @@ namespace fc { } template - inline void unpack( Stream& s, std::shared_ptr& v) + inline void unpack( Stream& s, std::shared_ptr& v, uint32_t depth ) { try { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); v = std::make_shared(); - fc::raw::unpack( s, *v ); + fc::raw::unpack( s, *v, depth ); } FC_RETHROW_EXCEPTIONS( warn, "std::shared_ptr", ("type",fc::get_typename::name()) ) } template inline void pack( Stream& s, const signed_int& v ) { @@ -177,7 +184,7 @@ namespace fc { }while( val ); } - template inline void unpack( Stream& s, signed_int& vi ) { + template inline void unpack( Stream& s, signed_int& vi, uint32_t ) { uint32_t v = 0, by = 0, limit = 0; char b = 0; do { s.get(b); @@ -189,7 +196,7 @@ namespace fc { vi.value = v&0x01 ? vi.value : -vi.value; vi.value = -vi.value; } - template inline void unpack( Stream& s, unsigned_int& vi ) { + template inline void unpack( Stream& s, unsigned_int& vi, uint32_t ) { uint32_t v = 0, by = 0, limit = 0; char b = 0; do { s.get(b); @@ -200,10 +207,12 @@ namespace fc { vi.value = static_cast(v); } - template inline void unpack( Stream& s, const T& vi ) + template inline void unpack( Stream& s, const T& vi, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); T tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, depth ); FC_ASSERT( vi == tmp ); } @@ -213,7 +222,12 @@ namespace fc { void pack( Stream& s, const safe& v ) { fc::raw::pack( s, v.value ); } template - void unpack( Stream& s, fc::safe& v ) { fc::raw::unpack( s, v.value ); } + void unpack( Stream& s, fc::safe& v, uint32_t depth ) + { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + fc::raw::unpack( s, v.value, depth ); + } template void pack( Stream& s, const fc::fwd& v ) { @@ -221,14 +235,22 @@ namespace fc { } template - void unpack( Stream& s, fc::fwd& v ) { - fc::raw::unpack( *v ); + void unpack( Stream& s, fc::fwd& v, uint32_t depth ) + { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + fc::raw::unpack( *v, depth ); } template void pack( Stream& s, const fc::smart_ref& v ) { fc::raw::pack( s, *v ); } template - void unpack( Stream& s, fc::smart_ref& v ) { fc::raw::unpack( s, *v ); } + void unpack( Stream& s, fc::smart_ref& v, uint32_t depth ) + { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + fc::raw::unpack( s, *v, depth ); + } // optional template @@ -238,10 +260,12 @@ namespace fc { } template - void unpack( Stream& s, fc::optional& v ) + void unpack( Stream& s, fc::optional& v, uint32_t depth ) { try { - bool b; fc::raw::unpack( s, b ); - if( b ) { v = T(); fc::raw::unpack( s, *v ); } + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + bool b; fc::raw::unpack( s, b, depth ); + if( b ) { v = T(); fc::raw::unpack( s, *v, depth ); } } FC_RETHROW_EXCEPTIONS( warn, "optional<${type}>", ("type",fc::get_typename::name() ) ) } // std::vector @@ -250,8 +274,9 @@ namespace fc { if( value.size() ) s.write( &value.front(), (uint32_t)value.size() ); } - template inline void unpack( Stream& s, std::vector& value ) { - unsigned_int size; fc::raw::unpack( s, size ); + template inline void unpack( Stream& s, std::vector& value, uint32_t depth ) { + depth++; + unsigned_int size; fc::raw::unpack( s, size, depth ); FC_ASSERT( size.value < MAX_ARRAY_ALLOC_SIZE ); value.resize(size.value); if( value.size() ) @@ -264,9 +289,10 @@ namespace fc { if( v.size() ) s.write( v.c_str(), v.size() ); } - template inline void unpack( Stream& s, fc::string& v ) { + template inline void unpack( Stream& s, fc::string& v, uint32_t depth ) { + depth++; std::vector tmp; - fc::raw::unpack(s,tmp); + fc::raw::unpack(s,tmp,depth); if( tmp.size() ) v = fc::string(tmp.data(),tmp.data()+tmp.size()); else v = fc::string(); @@ -274,10 +300,11 @@ namespace fc { // bool template inline void pack( Stream& s, const bool& v ) { fc::raw::pack( s, uint8_t(v) ); } - template inline void unpack( Stream& s, bool& v ) + template inline void unpack( Stream& s, bool& v, uint32_t depth ) { + depth++; uint8_t b; - fc::raw::unpack( s, b ); + fc::raw::unpack( s, b, depth ); FC_ASSERT( (b & ~1) == 0 ); v=(b!=0); } @@ -318,7 +345,7 @@ namespace fc { template static inline void pack( Stream& s, const T& v ) { s << v; } template - static inline void unpack( Stream& s, T& v ) { s >> v; } + static inline void unpack( Stream& s, T& v, uint32_t depth = 0 ) { s >> v; } }; template<> @@ -328,7 +355,7 @@ namespace fc { s.write( (char*)&v, sizeof(v) ); } template - static inline void unpack( Stream& s, T& v ) { + static inline void unpack( Stream& s, T& v, uint32_t ) { s.read( (char*)&v, sizeof(v) ); } }; @@ -340,7 +367,7 @@ namespace fc { fc::reflector::visit( pack_object_visitor( v, s ) ); } template - static inline void unpack( Stream& s, T& v ) { + static inline void unpack( Stream& s, T& v, uint32_t ) { fc::reflector::visit( unpack_object_visitor( v, s ) ); } }; @@ -351,9 +378,11 @@ namespace fc { fc::raw::pack(s, (int64_t)v); } template - static inline void unpack( Stream& s, T& v ) { + static inline void unpack( Stream& s, T& v, uint32_t depth = 0 ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); int64_t temp; - fc::raw::unpack(s, temp); + fc::raw::unpack(s, temp, depth); v = (T)temp; } }; @@ -365,8 +394,10 @@ namespace fc { if_class::type>::pack(s,v); } template - static inline void unpack( Stream& s, T& v ) { - if_class::type>::unpack(s,v); + static inline void unpack( Stream& s, T& v, uint32_t depth = 0 ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + if_class::type>::unpack(s,v,depth); } }; template<> @@ -376,12 +407,14 @@ namespace fc { if_enum< typename fc::reflector::is_enum >::pack(s,v); } template - static inline void unpack( Stream& s, T& v ) { - if_enum< typename fc::reflector::is_enum >::unpack(s,v); + static inline void unpack( Stream& s, T& v, uint32_t depth = 0 ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + if_enum< typename fc::reflector::is_enum >::unpack(s,v,depth); } }; - } // namesapce detail + } // namespace detail template inline void pack( Stream& s, const std::unordered_set& value ) { @@ -394,15 +427,16 @@ namespace fc { } } template - inline void unpack( Stream& s, std::unordered_set& value ) { - unsigned_int size; fc::raw::unpack( s, size ); + inline void unpack( Stream& s, std::unordered_set& value, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + unsigned_int size; fc::raw::unpack( s, size, depth ); value.clear(); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); - value.reserve(size.value); for( uint32_t i = 0; i < size.value; ++i ) { T tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, depth ); value.insert( std::move(tmp) ); } } @@ -414,10 +448,12 @@ namespace fc { fc::raw::pack( s, value.second ); } template - inline void unpack( Stream& s, std::pair& value ) + inline void unpack( Stream& s, std::pair& value, uint32_t depth ) { - fc::raw::unpack( s, value.first ); - fc::raw::unpack( s, value.second ); + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + fc::raw::unpack( s, value.first, depth ); + fc::raw::unpack( s, value.second, depth ); } template @@ -431,16 +467,17 @@ namespace fc { } } template - inline void unpack( Stream& s, std::unordered_map& value ) + inline void unpack( Stream& s, std::unordered_map& value, uint32_t depth ) { - unsigned_int size; fc::raw::unpack( s, size ); + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + unsigned_int size; fc::raw::unpack( s, size, depth ); value.clear(); FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); - value.reserve(size.value); for( uint32_t i = 0; i < size.value; ++i ) { std::pair tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, depth ); value.insert( std::move(tmp) ); } } @@ -455,15 +492,17 @@ namespace fc { } } template - inline void unpack( Stream& s, std::map& value ) + inline void unpack( Stream& s, std::map& value, uint32_t depth ) { - unsigned_int size; fc::raw::unpack( s, size ); + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + unsigned_int size; fc::raw::unpack( s, size, depth ); value.clear(); FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); for( uint32_t i = 0; i < size.value; ++i ) { std::pair tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, depth ); value.insert( std::move(tmp) ); } } @@ -480,15 +519,16 @@ namespace fc { } template - inline void unpack( Stream& s, std::deque& value ) { - unsigned_int size; fc::raw::unpack( s, size ); + inline void unpack( Stream& s, std::deque& value, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + unsigned_int size; fc::raw::unpack( s, size, depth ); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); - value.resize(size.value); - auto itr = value.begin(); - auto end = value.end(); - while( itr != end ) { - fc::raw::unpack( s, *itr ); - ++itr; + for ( size_t i = 0; i < size.value; i++ ) + { + T tmp; + fc::raw::unpack( s, tmp, depth ); + value.emplace_back( std::move( tmp ) ); } } @@ -504,15 +544,16 @@ namespace fc { } template - inline void unpack( Stream& s, std::vector& value ) { + inline void unpack( Stream& s, std::vector& value, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); unsigned_int size; fc::raw::unpack( s, size ); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); - value.resize(size.value); - auto itr = value.begin(); - auto end = value.end(); - while( itr != end ) { - fc::raw::unpack( s, *itr ); - ++itr; + for ( size_t i = 0; i < size.value; i++ ) + { + T tmp; + fc::raw::unpack( s, tmp, depth ); + value.emplace_back( std::move( tmp ) ); } } @@ -528,12 +569,14 @@ namespace fc { } template - inline void unpack( Stream& s, std::set& value ) { - unsigned_int size; fc::raw::unpack( s, size ); + inline void unpack( Stream& s, std::set& value, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + unsigned_int size; fc::raw::unpack( s, size, depth ); for( uint64_t i = 0; i < size.value; ++i ) { typename std::set::value_type tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, depth ); value.insert( std::move(tmp) ); } } @@ -550,12 +593,14 @@ namespace fc { } template - inline void unpack( Stream& s, std::multiset& value ) { - unsigned_int size; fc::raw::unpack( s, size ); + inline void unpack( Stream& s, std::multiset& value, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + unsigned_int size; fc::raw::unpack( s, size, depth ); for( uint64_t i = 0; i < size.value; ++i ) { typename std::multiset::value_type tmp; - fc::raw::unpack( s, tmp ); + fc::raw::unpack( s, tmp, depth ); value.insert( std::move(tmp) ); } } @@ -567,9 +612,11 @@ namespace fc { fc::raw::detail::if_reflected< typename fc::reflector::is_defined >::pack(s,v); } template - inline void unpack( Stream& s, T& v ) + inline void unpack( Stream& s, T& v, uint32_t depth ) { try { - fc::raw::detail::if_reflected< typename fc::reflector::is_defined >::unpack(s,v); + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); + fc::raw::detail::if_reflected< typename fc::reflector::is_defined >::unpack(s,v,depth); } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template @@ -595,22 +642,24 @@ namespace fc { } template - inline T unpack_from_vector( const std::vector& s ) + inline T unpack_from_vector( const std::vector& s, uint32_t depth ) { try { + depth++; T tmp; if( s.size() ) { datastream ds( s.data(), size_t(s.size()) ); - fc::raw::unpack(ds,tmp); + fc::raw::unpack(ds,tmp,depth); } return tmp; } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template - inline void unpack_from_vector( const std::vector& s, T& tmp ) + inline void unpack_from_vector( const std::vector& s, T& tmp, uint32_t depth = 0 ) { try { + depth++; if( s.size() ) { datastream ds( s.data(), size_t(s.size()) ); - fc::raw::unpack(ds,tmp); + fc::raw::unpack(ds,tmp,depth); } } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } @@ -621,19 +670,21 @@ namespace fc { } template - inline T unpack_from_char_array( const char* d, uint32_t s ) + inline T unpack_from_char_array( const char* d, uint32_t s, uint32_t depth ) { try { + depth++; T v; datastream ds( d, s ); - fc::raw::unpack(ds,v); + fc::raw::unpack(ds,v,depth); return v; } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } template - inline void unpack_from_char_array( const char* d, uint32_t s, T& v ) + inline void unpack_from_char_array( const char* d, uint32_t s, T& v, uint32_t depth ) { try { + depth++; datastream ds( d, s ); - fc::raw::unpack(ds,v); + fc::raw::unpack(ds,v,depth); return v; } FC_RETHROW_EXCEPTIONS( warn, "error unpacking ${type}", ("type",fc::get_typename::name() ) ) } @@ -671,10 +722,12 @@ namespace fc { sv.visit( pack_static_variant(s) ); } - template void unpack( Stream& s, static_variant& sv ) + template void unpack( Stream& s, static_variant& sv, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); unsigned_int w; - fc::raw::unpack( s, w ); + fc::raw::unpack( s, w, depth ); sv.set_which(w.value); sv.visit( unpack_static_variant(s) ); } diff --git a/libraries/fc/include/fc/io/raw_fwd.hpp b/libraries/fc/include/fc/io/raw_fwd.hpp index 154d9ba79f..646219a516 100644 --- a/libraries/fc/include/fc/io/raw_fwd.hpp +++ b/libraries/fc/include/fc/io/raw_fwd.hpp @@ -12,6 +12,7 @@ #include #define MAX_ARRAY_ALLOC_SIZE (1024*1024*10) +#define MAX_RECURSION_DEPTH (20) namespace fc { class time_point; @@ -33,98 +34,96 @@ namespace fc { inline size_t pack_size( const T& v ); template inline void pack( Stream& s, const fc::fixed_string& u ); - template inline void unpack( Stream& s, fc::fixed_string& u ); + template inline void unpack( Stream& s, fc::fixed_string& u, uint32_t depth = 0 ); template inline void pack( Stream& s, const fc::enum_type& tp ); template - inline void unpack( Stream& s, fc::enum_type& tp ); + inline void unpack( Stream& s, fc::enum_type& tp, uint32_t depth = 0 ); template inline void pack( Stream& s, const std::set& value ); - template inline void unpack( Stream& s, std::set& value ); + template inline void unpack( Stream& s, std::set& value, uint32_t depth = 0 ); template inline void pack( Stream& s, const std::multiset& value ); - template inline void unpack( Stream& s, std::multiset& value ); + template inline void unpack( Stream& s, std::multiset& value, uint32_t depth = 0 ); template inline void pack( Stream& s, const std::unordered_set& value ); - template inline void unpack( Stream& s, std::unordered_set& value ); + template inline void unpack( Stream& s, std::unordered_set& value, uint32_t depth = 0 ); template void pack( Stream& s, const static_variant& sv ); - template void unpack( Stream& s, static_variant& sv ); - - template inline void pack( Stream& s, const flat_set& value ); - template inline void unpack( Stream& s, flat_set& value ); + template void unpack( Stream& s, static_variant& sv, uint32_t depth = 0 ); template inline void pack( Stream& s, const std::deque& value ); - template inline void unpack( Stream& s, std::deque& value ); + template inline void unpack( Stream& s, std::deque& value, uint32_t depth = 0 ); template inline void pack( Stream& s, const std::unordered_map& value ); - template inline void unpack( Stream& s, std::unordered_map& value ); + template inline void unpack( Stream& s, std::unordered_map& value, uint32_t depth = 0 ); template inline void pack( Stream& s, const std::map& value ); - template inline void unpack( Stream& s, std::map& value ); - - template inline void pack( Stream& s, const flat_map& value ); - template inline void unpack( Stream& s, flat_map& value ); + template inline void unpack( Stream& s, std::map& value, uint32_t depth = 0 ); template inline void pack( Stream& s, const std::pair& value ); - template inline void unpack( Stream& s, std::pair& value ); + template inline void unpack( Stream& s, std::pair& value, uint32_t depth = 0 ); template inline void pack( Stream& s, const variant_object& v ); - template inline void unpack( Stream& s, variant_object& v ); + template inline void unpack( Stream& s, variant_object& v, uint32_t depth = 0 ); template inline void pack( Stream& s, const variant& v ); - template inline void unpack( Stream& s, variant& v ); + template inline void unpack( Stream& s, variant& v, uint32_t depth = 0 ); template inline void pack( Stream& s, const path& v ); - template inline void unpack( Stream& s, path& v ); + template inline void unpack( Stream& s, path& v, uint32_t depth = 0 ); template inline void pack( Stream& s, const ip::endpoint& v ); - template inline void unpack( Stream& s, ip::endpoint& v ); + template inline void unpack( Stream& s, ip::endpoint& v, uint32_t depth = 0 ); + template inline void pack( Stream& s, const ip::address& v ); + template inline void unpack( Stream& s, ip::address& v, uint32_t depth = 0 ); - template void unpack( Stream& s, fc::optional& v ); - template void unpack( Stream& s, const T& v ); + template void unpack( Stream& s, fc::optional& v, uint32_t depth = 0 ); + template void unpack( Stream& s, const T& v, uint32_t depth = 0 ); template void pack( Stream& s, const fc::optional& v ); template void pack( Stream& s, const safe& v ); - template void unpack( Stream& s, fc::safe& v ); + template void unpack( Stream& s, fc::safe& v, uint32_t depth = 0 ); - template void unpack( Stream& s, time_point& ); + template void unpack( Stream& s, microseconds&, uint32_t depth = 0 ); + template void pack( Stream& s, const microseconds& ); + template void unpack( Stream& s, time_point&, uint32_t depth = 0 ); template void pack( Stream& s, const time_point& ); - template void unpack( Stream& s, time_point_sec& ); + template void unpack( Stream& s, time_point_sec&, uint32_t depth = 0 ); template void pack( Stream& s, const time_point_sec& ); - template void unpack( Stream& s, std::string& ); + template void unpack( Stream& s, std::string&, uint32_t depth = 0 ); template void pack( Stream& s, const std::string& ); - template void unpack( Stream& s, fc::ecc::public_key& ); + template void unpack( Stream& s, fc::ecc::public_key&, uint32_t depth = 0 ); template void pack( Stream& s, const fc::ecc::public_key& ); - template void unpack( Stream& s, fc::ecc::private_key& ); + template void unpack( Stream& s, fc::ecc::private_key&, uint32_t depth = 0 ); template void pack( Stream& s, const fc::ecc::private_key& ); template inline void pack( Stream& s, const T& v ); - template inline void unpack( Stream& s, T& v ); + template inline void unpack( Stream& s, T& v, uint32_t depth = 0 ); template inline void pack( Stream& s, const std::vector& v ); - template inline void unpack( Stream& s, std::vector& v ); + template inline void unpack( Stream& s, std::vector& v, uint32_t depth = 0 ); template inline void pack( Stream& s, const signed_int& v ); - template inline void unpack( Stream& s, signed_int& vi ); + template inline void unpack( Stream& s, signed_int& vi, uint32_t depth = 0 ); template inline void pack( Stream& s, const unsigned_int& v ); - template inline void unpack( Stream& s, unsigned_int& vi ); + template inline void unpack( Stream& s, unsigned_int& vi, uint32_t depth = 0 ); template inline void pack( Stream& s, const char* v ); template inline void pack( Stream& s, const std::vector& value ); - template inline void unpack( Stream& s, std::vector& value ); + template inline void unpack( Stream& s, std::vector& value, uint32_t depth = 0 ); template inline void pack( Stream& s, const fc::array& v); - template inline void unpack( Stream& s, fc::array& v); + template inline void unpack( Stream& s, fc::array& v, uint32_t depth = 0); template inline void pack( Stream& s, const fc::int_array& v); - template inline void unpack( Stream& s, fc::int_array& v); + template inline void unpack( Stream& s, fc::int_array& v, uint32_t depth = 0); template inline void pack( Stream& s, const bool& v ); - template inline void unpack( Stream& s, bool& v ); + template inline void unpack( Stream& s, bool& v, uint32_t depth = 0 ); template inline std::vector pack_to_vector( const T& v ); - template inline T unpack_from_vector( const std::vector& s ); - template inline T unpack_from_char_array( const char* d, uint32_t s ); - template inline void unpack_from_char_array( const char* d, uint32_t s, T& v ); + template inline T unpack_from_vector( const std::vector& s, uint32_t depth = 0 ); + template inline T unpack_from_char_array( const char* d, uint32_t s, uint32_t depth = 0 ); + template inline void unpack_from_char_array( const char* d, uint32_t s, T& v, uint32_t depth = 0 ); } } diff --git a/libraries/fc/include/fc/io/raw_variant.hpp b/libraries/fc/include/fc/io/raw_variant.hpp index 24fe3f6e36..53b8898539 100644 --- a/libraries/fc/include/fc/io/raw_variant.hpp +++ b/libraries/fc/include/fc/io/raw_variant.hpp @@ -53,10 +53,12 @@ namespace fc { namespace raw { v.visit( variant_packer(s) ); } template - inline void unpack( Stream& s, variant& v ) + inline void unpack( Stream& s, variant& v, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); uint8_t t; - unpack( s, t ); + unpack( s, t, depth ); switch( t ) { case variant::null_type: @@ -64,49 +66,49 @@ namespace fc { namespace raw { case variant::int64_type: { int64_t val; - raw::unpack(s,val); + raw::unpack(s,val,depth); v = val; return; } case variant::uint64_type: { uint64_t val; - raw::unpack(s,val); + raw::unpack(s,val,depth); v = val; return; } case variant::double_type: { double val; - raw::unpack(s,val); + raw::unpack(s,val,depth); v = val; return; } case variant::bool_type: { bool val; - raw::unpack(s,val); + raw::unpack(s,val,depth); v = val; return; } case variant::string_type: { fc::string val; - raw::unpack(s,val); + raw::unpack(s,val,depth); v = fc::move(val); return; } case variant::array_type: { variants val; - raw::unpack(s,val); + raw::unpack(s,val,depth); v = fc::move(val); return; } case variant::object_type: { variant_object val; - raw::unpack(s,val); + raw::unpack(s,val,depth); v = fc::move(val); return; } @@ -127,10 +129,12 @@ namespace fc { namespace raw { } } template - inline void unpack( Stream& s, variant_object& v ) + inline void unpack( Stream& s, variant_object& v, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); unsigned_int vs; - unpack( s, vs ); + unpack( s, vs, depth ); mutable_variant_object mvo; mvo.reserve(vs.value); @@ -138,8 +142,8 @@ namespace fc { namespace raw { { fc::string key; fc::variant value; - fc::raw::unpack(s,key); - fc::raw::unpack(s,value); + fc::raw::unpack(s,key,depth); + fc::raw::unpack(s,value,depth); mvo.set( fc::move(key), fc::move(value) ); } v = fc::move(mvo); diff --git a/libraries/fc/include/fc/network/ip.hpp b/libraries/fc/include/fc/network/ip.hpp index 43fc5a442c..41f772f167 100644 --- a/libraries/fc/include/fc/network/ip.hpp +++ b/libraries/fc/include/fc/network/ip.hpp @@ -88,10 +88,11 @@ namespace fc { fc::raw::pack( s, uint32_t(v) ); } template - inline void unpack( Stream& s, ip::address& v ) + inline void unpack( Stream& s, ip::address& v, uint32_t depth ) { + depth++; uint32_t _ip; - fc::raw::unpack( s, _ip ); + fc::raw::unpack( s, _ip, depth ); v = ip::address(_ip); } @@ -102,12 +103,13 @@ namespace fc { fc::raw::pack( s, v.port() ); } template - inline void unpack( Stream& s, ip::endpoint& v ) + inline void unpack( Stream& s, ip::endpoint& v, uint32_t depth ) { + depth++; ip::address a; uint16_t p; - fc::raw::unpack( s, a ); - fc::raw::unpack( s, p ); + fc::raw::unpack( s, a, depth ); + fc::raw::unpack( s, p, depth ); v = ip::endpoint(a,p); } diff --git a/libraries/fc/include/fc/real128.hpp b/libraries/fc/include/fc/real128.hpp index 3a7d26dbef..4dd528e8c8 100644 --- a/libraries/fc/include/fc/real128.hpp +++ b/libraries/fc/include/fc/real128.hpp @@ -44,7 +44,7 @@ namespace fc { template inline void pack( Stream& s, const real128& value_to_pack ) { s.write( (char*)&value_to_pack, sizeof(value_to_pack) ); } template - inline void unpack( Stream& s, real128& value_to_unpack ) { s.read( (char*)&value_to_unpack, sizeof(value_to_unpack) ); } + inline void unpack( Stream& s, real128& value_to_unpack, uint32_t ) { s.read( (char*)&value_to_unpack, sizeof(value_to_unpack) ); } } diff --git a/libraries/fc/include/fc/uint128.hpp b/libraries/fc/include/fc/uint128.hpp index 7d4394bf03..98432cd69a 100644 --- a/libraries/fc/include/fc/uint128.hpp +++ b/libraries/fc/include/fc/uint128.hpp @@ -134,7 +134,7 @@ namespace fc template inline void pack( Stream& s, const uint128& u ) { s.write( (char*)&u, sizeof(u) ); } template - inline void unpack( Stream& s, uint128& u ) { s.read( (char*)&u, sizeof(u) ); } + inline void unpack( Stream& s, uint128& u, uint32_t ) { s.read( (char*)&u, sizeof(u) ); } } size_t city_hash_size_t(const char *buf, size_t len); diff --git a/libraries/protocol/include/steem/protocol/asset_symbol.hpp b/libraries/protocol/include/steem/protocol/asset_symbol.hpp index d84f90e271..d74d8fc71f 100644 --- a/libraries/protocol/include/steem/protocol/asset_symbol.hpp +++ b/libraries/protocol/include/steem/protocol/asset_symbol.hpp @@ -181,7 +181,7 @@ inline void pack( Stream& s, const steem::protocol::asset_symbol_type& sym ) } template< typename Stream > -inline void unpack( Stream& s, steem::protocol::asset_symbol_type& sym ) +inline void unpack( Stream& s, steem::protocol::asset_symbol_type& sym, uint32_t ) { uint64_t ser = 0; s.read( (char*) &ser, 4 ); diff --git a/libraries/protocol/include/steem/protocol/fixed_string.hpp b/libraries/protocol/include/steem/protocol/fixed_string.hpp index 669c4ffe71..ca1c536842 100644 --- a/libraries/protocol/include/steem/protocol/fixed_string.hpp +++ b/libraries/protocol/include/steem/protocol/fixed_string.hpp @@ -197,10 +197,11 @@ inline void pack( Stream& s, const steem::protocol::fixed_string_impl< Storage > } template< typename Stream, typename Storage > -inline void unpack( Stream& s, steem::protocol::fixed_string_impl< Storage >& u ) +inline void unpack( Stream& s, steem::protocol::fixed_string_impl< Storage >& u, uint32_t depth ) { + depth++; std::string str; - unpack( s, str ); + unpack( s, str, depth ); u = str; } diff --git a/libraries/protocol/include/steem/protocol/legacy_asset.hpp b/libraries/protocol/include/steem/protocol/legacy_asset.hpp index 33da29ac15..37de5071f0 100644 --- a/libraries/protocol/include/steem/protocol/legacy_asset.hpp +++ b/libraries/protocol/include/steem/protocol/legacy_asset.hpp @@ -77,7 +77,7 @@ inline void pack( Stream& s, const steem::protocol::legacy_steem_asset_symbol_ty } template< typename Stream > -inline void unpack( Stream& s, steem::protocol::legacy_steem_asset_symbol_type& sym ) +inline void unpack( Stream& s, steem::protocol::legacy_steem_asset_symbol_type& sym, uint32_t depth ) { // 994240: "account_creation_fee": "0.1 STEEM" // 1021529: "account_creation_fee": "10.0 STEEM" @@ -87,10 +87,10 @@ inline void unpack( Stream& s, steem::protocol::legacy_steem_asset_symbol_type& // 4338089: "account_creation_fee": "0.001 0.001" // 4626205: "account_creation_fee": "6.000 6.000" // 4632595: "account_creation_fee": "6.000 6.000" - + depth++; uint64_t ser = 0; - fc::raw::unpack( s, ser ); + fc::raw::unpack( s, ser, depth ); switch( ser ) { case STEEM_SYMBOL_LEGACY_SER_1: diff --git a/libraries/protocol/include/steem/protocol/types_fwd.hpp b/libraries/protocol/include/steem/protocol/types_fwd.hpp index 0c23036b8b..851d7547ea 100644 --- a/libraries/protocol/include/steem/protocol/types_fwd.hpp +++ b/libraries/protocol/include/steem/protocol/types_fwd.hpp @@ -1,7 +1,8 @@ #pragma once +#include +#include namespace fc { -class uint128; class variant; } // fc @@ -16,25 +17,20 @@ struct legacy_steem_asset; namespace fc { namespace raw { -template -inline void pack( Stream& s, const uint128& u ); -template -inline void unpack( Stream& s, uint128& u ); - template< typename Stream, typename Storage > inline void pack( Stream& s, const steem::protocol::fixed_string_impl< Storage >& u ); template< typename Stream, typename Storage > -inline void unpack( Stream& s, steem::protocol::fixed_string_impl< Storage >& u ); +inline void unpack( Stream& s, steem::protocol::fixed_string_impl< Storage >& u, uint32_t depth = 0 ); template< typename Stream > inline void pack( Stream& s, const steem::protocol::asset_symbol_type& sym ); template< typename Stream > -inline void unpack( Stream& s, steem::protocol::asset_symbol_type& sym ); +inline void unpack( Stream& s, steem::protocol::asset_symbol_type& sym, uint32_t depth = 0 ); template< typename Stream > inline void pack( Stream& s, const steem::protocol::legacy_steem_asset_symbol_type& sym ); template< typename Stream > -inline void unpack( Stream& s, steem::protocol::legacy_steem_asset_symbol_type& sym ); +inline void unpack( Stream& s, steem::protocol::legacy_steem_asset_symbol_type& sym, uint32_t depth = 0 ); } // raw From ef8135523bb7de6f31ca23c53eba59e8eb4f12ea Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Mon, 10 Dec 2018 11:14:14 -0800 Subject: [PATCH 03/15] Add an account auth behavior test --- tests/tests/operation_tests.cpp | 101 ++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp index 9b26cacbd7..ad512cd441 100644 --- a/tests/tests/operation_tests.cpp +++ b/tests/tests/operation_tests.cpp @@ -7503,5 +7503,106 @@ BOOST_AUTO_TEST_CASE( create_claimed_account_apply ) FC_LOG_AND_RETHROW() } +BOOST_AUTO_TEST_CASE( account_auth_tests ) +{ + try + { + ACTORS( (alice)(bob)(charlie) ) + generate_block(); + + fund( "alice", ASSET( "20.000 TESTS" ) ); + fund( "bob", ASSET( "20.000 TESTS" ) ); + fund( "charlie", ASSET( "20.000 TESTS" ) ); + vest( STEEM_INIT_MINER_NAME, "alice" , ASSET( "10.000 TESTS" ) ); + vest( STEEM_INIT_MINER_NAME, "bob" , ASSET( "10.000 TESTS" ) ); + vest( STEEM_INIT_MINER_NAME, "charlie" , ASSET( "10.000 TESTS" ) ); + generate_block(); + + private_key_type bob_active_private_key = bob_private_key; + private_key_type bob_posting_private_key = generate_private_key( "bob_posting" ); + private_key_type charlie_active_private_key = charlie_private_key; + private_key_type charlie_posting_private_key = generate_private_key( "charlie_posting" ); + + db_plugin->debug_update( [=]( database& db ) + { + db.modify( db.get< account_authority_object, by_account >( "alice"), [&]( account_authority_object& auth ) + { + auth.active.add_authority( "bob", 1 ); + auth.posting.add_authority( "charlie", 1 ); + }); + + db.modify( db.get< account_authority_object, by_account >( "bob" ), [&]( account_authority_object& auth ) + { + auth.posting = authority( 1, bob_posting_private_key.get_public_key(), 1 ); + }); + + db.modify( db.get< account_authority_object, by_account >( "charlie" ), [&]( account_authority_object& auth ) + { + auth.posting = authority( 1, charlie_posting_private_key.get_public_key(), 1 ); + }); + }); + + generate_block(); + + signed_transaction tx; + transfer_operation transfer; + + transfer.from = "alice"; + transfer.to = "bob"; + transfer.amount = ASSET( "1.000 TESTS" ); + tx.operations.push_back( transfer ); + tx.set_expiration( db->head_block_time() + STEEM_MAX_TIME_UNTIL_EXPIRATION ); + tx.signatures.clear(); + sign( tx, bob_active_private_key ); + db->push_transaction( tx, 0 ); + + generate_block(); + tx.set_expiration( db->head_block_time() + STEEM_MAX_TIME_UNTIL_EXPIRATION ); + tx.signatures.clear(); + sign( tx, bob_posting_private_key ); + STEEM_REQUIRE_THROW( db->push_transaction( tx, 0 ), tx_missing_active_auth ); + + generate_block(); + tx.set_expiration( db->head_block_time() + STEEM_MAX_TIME_UNTIL_EXPIRATION ); + tx.signatures.clear(); + sign( tx, charlie_active_private_key ); + STEEM_REQUIRE_THROW( db->push_transaction( tx, 0 ), tx_missing_active_auth ); + + generate_block(); + tx.set_expiration( db->head_block_time() + STEEM_MAX_TIME_UNTIL_EXPIRATION ); + tx.signatures.clear(); + sign( tx, charlie_posting_private_key ); + STEEM_REQUIRE_THROW( db->push_transaction( tx, 0 ), tx_missing_active_auth ); + + custom_json_operation json; + json.required_posting_auths.insert( "alice" ); + json.json = "{\"foo\":\"bar\"}"; + tx.operations.clear(); + tx.signatures.clear(); + tx.operations.push_back( json ); + sign( tx, bob_active_private_key ); + STEEM_REQUIRE_THROW( db->push_transaction( tx, 0 ), tx_missing_posting_auth ); + + generate_block(); + tx.set_expiration( db->head_block_time() + STEEM_MAX_TIME_UNTIL_EXPIRATION ); + tx.signatures.clear(); + sign( tx, bob_posting_private_key ); + db->push_transaction( tx, 0 ); + + generate_block(); + tx.set_expiration( db->head_block_time() + STEEM_MAX_TIME_UNTIL_EXPIRATION ); + tx.signatures.clear(); + sign( tx, charlie_active_private_key ); + STEEM_REQUIRE_THROW( db->push_transaction( tx, 0 ), tx_missing_posting_auth ); + + generate_block(); + tx.set_expiration( db->head_block_time() + STEEM_MAX_TIME_UNTIL_EXPIRATION ); + tx.signatures.clear(); + sign( tx, charlie_posting_private_key ); + db->push_transaction( tx, 0 ); + } + FC_LOG_AND_RETHROW() +} + BOOST_AUTO_TEST_SUITE_END() #endif From ee8d297d165e8c03fc2ae1d17bdfcc19179f44b3 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Mon, 10 Dec 2018 16:31:29 -0500 Subject: [PATCH 04/15] Added depth assertion for templated types, removed object pre-allocation as well --- libraries/chain/include/steem/chain/steem_object_types.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/chain/include/steem/chain/steem_object_types.hpp b/libraries/chain/include/steem/chain/steem_object_types.hpp index 79d71c6066..fea8cc5533 100644 --- a/libraries/chain/include/steem/chain/steem_object_types.hpp +++ b/libraries/chain/include/steem/chain/steem_object_types.hpp @@ -241,6 +241,7 @@ template< typename Stream, typename E, typename A > void unpack( Stream& s, boost::interprocess::deque& dq, uint32_t depth ) { depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); // This could be optimized std::vector temp; unpack( s, temp, depth ); @@ -264,11 +265,11 @@ template< typename Stream, typename K, typename V, typename C, typename A > void unpack( Stream& s, boost::interprocess::flat_map< K, V, C, A >& value, uint32_t depth ) { depth++; + FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); unsigned_int size; unpack( s, size, depth ); value.clear(); FC_ASSERT( size.value*(sizeof(K)+sizeof(V)) < MAX_ARRAY_ALLOC_SIZE ); - value.reserve(size.value); for( uint32_t i = 0; i < size.value; ++i ) { std::pair tmp; From 4b7767d00ba6f81d43fdbecf024e30cf236249c4 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Thu, 13 Dec 2018 16:49:31 -0500 Subject: [PATCH 05/15] When unpacking containers clear the structures, added unpack serialization tests, cleaned up unpack forward declarations --- .../steem/chain/steem_object_types.hpp | 1 + .../fc/include/fc/container/deque_fwd.hpp | 2 +- libraries/fc/include/fc/container/flat.hpp | 1 + libraries/fc/include/fc/io/raw.hpp | 4 + libraries/fc/include/fc/io/raw_fwd.hpp | 3 - libraries/fc/include/fc/io/raw_variant.hpp | 1 - tests/tests/serialization_tests.cpp | 107 ++++++++++++++++++ 7 files changed, 114 insertions(+), 5 deletions(-) diff --git a/libraries/chain/include/steem/chain/steem_object_types.hpp b/libraries/chain/include/steem/chain/steem_object_types.hpp index fea8cc5533..64fe793ce5 100644 --- a/libraries/chain/include/steem/chain/steem_object_types.hpp +++ b/libraries/chain/include/steem/chain/steem_object_types.hpp @@ -245,6 +245,7 @@ void unpack( Stream& s, boost::interprocess::deque& dq, uint32_t depth ) // This could be optimized std::vector temp; unpack( s, temp, depth ); + dq.clear(); std::copy( temp.begin(), temp.end(), std::back_inserter(dq) ); } diff --git a/libraries/fc/include/fc/container/deque_fwd.hpp b/libraries/fc/include/fc/container/deque_fwd.hpp index 80359ea67c..246114cd5e 100644 --- a/libraries/fc/include/fc/container/deque_fwd.hpp +++ b/libraries/fc/include/fc/container/deque_fwd.hpp @@ -8,6 +8,6 @@ namespace fc { template void pack( Stream& s, const std::deque& value ); template - void unpack( Stream& s, std::deque& value ); + void unpack( Stream& s, std::deque& value, uint32_t depth = 0 ); } } // namespace fc diff --git a/libraries/fc/include/fc/container/flat.hpp b/libraries/fc/include/fc/container/flat.hpp index fce20d4694..c24bb193f4 100644 --- a/libraries/fc/include/fc/container/flat.hpp +++ b/libraries/fc/include/fc/container/flat.hpp @@ -78,6 +78,7 @@ namespace fc { FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); unsigned_int size; unpack( s, size, depth ); + value.clear(); if( !std::is_fundamental::value ) { for ( size_t i = 0; i < size.value; i++ ) { diff --git a/libraries/fc/include/fc/io/raw.hpp b/libraries/fc/include/fc/io/raw.hpp index a52e2c1360..798aa45186 100644 --- a/libraries/fc/include/fc/io/raw.hpp +++ b/libraries/fc/include/fc/io/raw.hpp @@ -524,6 +524,7 @@ namespace fc { FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); unsigned_int size; fc::raw::unpack( s, size, depth ); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); + value.clear(); for ( size_t i = 0; i < size.value; i++ ) { T tmp; @@ -549,6 +550,7 @@ namespace fc { FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); unsigned_int size; fc::raw::unpack( s, size ); FC_ASSERT( size.value*sizeof(T) < MAX_ARRAY_ALLOC_SIZE ); + value.clear(); for ( size_t i = 0; i < size.value; i++ ) { T tmp; @@ -573,6 +575,7 @@ namespace fc { depth++; FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); unsigned_int size; fc::raw::unpack( s, size, depth ); + value.clear(); for( uint64_t i = 0; i < size.value; ++i ) { typename std::set::value_type tmp; @@ -597,6 +600,7 @@ namespace fc { depth++; FC_ASSERT( depth <= MAX_RECURSION_DEPTH ); unsigned_int size; fc::raw::unpack( s, size, depth ); + value.clear(); for( uint64_t i = 0; i < size.value; ++i ) { typename std::multiset::value_type tmp; diff --git a/libraries/fc/include/fc/io/raw_fwd.hpp b/libraries/fc/include/fc/io/raw_fwd.hpp index 646219a516..80e19760b9 100644 --- a/libraries/fc/include/fc/io/raw_fwd.hpp +++ b/libraries/fc/include/fc/io/raw_fwd.hpp @@ -53,9 +53,6 @@ namespace fc { template void pack( Stream& s, const static_variant& sv ); template void unpack( Stream& s, static_variant& sv, uint32_t depth = 0 ); - template inline void pack( Stream& s, const std::deque& value ); - template inline void unpack( Stream& s, std::deque& value, uint32_t depth = 0 ); - template inline void pack( Stream& s, const std::unordered_map& value ); template inline void unpack( Stream& s, std::unordered_map& value, uint32_t depth = 0 ); diff --git a/libraries/fc/include/fc/io/raw_variant.hpp b/libraries/fc/include/fc/io/raw_variant.hpp index 53b8898539..f3f0b77a97 100644 --- a/libraries/fc/include/fc/io/raw_variant.hpp +++ b/libraries/fc/include/fc/io/raw_variant.hpp @@ -137,7 +137,6 @@ namespace fc { namespace raw { unpack( s, vs, depth ); mutable_variant_object mvo; - mvo.reserve(vs.value); for( uint32_t i = 0; i < vs.value; ++i ) { fc::string key; diff --git a/tests/tests/serialization_tests.cpp b/tests/tests/serialization_tests.cpp index 0b24fda02f..9693b85141 100644 --- a/tests/tests/serialization_tests.cpp +++ b/tests/tests/serialization_tests.cpp @@ -646,5 +646,112 @@ BOOST_AUTO_TEST_CASE( asset_symbol_type_test ) FC_LOG_AND_RETHROW(); } +BOOST_AUTO_TEST_CASE( unpack_clear_test ) +{ + try + { + std::stringstream ss1; + std::stringstream ss2; + + signed_block b1; + + for ( int i = 0; i < 10; i++ ) + { + signed_transaction tx; + + vote_operation op; + op.voter = "alice"; + op.author = "bob"; + op.permlink = "permlink1"; + op.weight = STEEM_100_PERCENT; + tx.operations.push_back( op ); + + vote_operation op2; + op2.voter = "charlie"; + op2.author = "sam"; + op2.permlink = "permlink2"; + op2.weight = STEEM_100_PERCENT; + tx.operations.push_back( op2 ); + + tx.ref_block_num = 1000; + tx.ref_block_prefix = 1000000000; + tx.expiration = fc::time_point_sec( 1514764800 + i ); + + b1.transactions.push_back( tx ); + } + + signed_block b2; + + for ( int i = 0; i < 20; i++ ) + { + signed_transaction tx; + vote_operation op; + op.voter = "dave"; + op.author = "greg"; + op.permlink = "foobar"; + op.weight = STEEM_100_PERCENT/2; + tx.ref_block_num = 4000; + tx.ref_block_prefix = 4000000000; + tx.expiration = fc::time_point_sec( 1714764800 + i ); + tx.operations.push_back( op ); + + b2.transactions.push_back( tx ); + } + + fc::raw::pack( ss2, b2 ); + fc::raw::pack( ss1, b1 ); + + signed_block unpacked_block; + fc::raw::unpack( ss2, unpacked_block ); + + // This operation should completely overwrite signed block 'b2' + fc::raw::unpack( ss1, unpacked_block ); + + BOOST_REQUIRE( b1.transactions.size() == unpacked_block.transactions.size() ); + for ( size_t i = 0; i < unpacked_block.transactions.size(); i++ ) + { + signed_transaction tx = unpacked_block.transactions[ i ]; + BOOST_REQUIRE( unpacked_block.transactions[ i ].operations.size() == b1.transactions[ i ].operations.size() ); + + vote_operation op = tx.operations[ 0 ].get< vote_operation >(); + BOOST_REQUIRE( op.voter == "alice" ); + BOOST_REQUIRE( op.author == "bob" ); + BOOST_REQUIRE( op.permlink == "permlink1" ); + BOOST_REQUIRE( op.weight == STEEM_100_PERCENT ); + + vote_operation op2 = tx.operations[ 1 ].get< vote_operation >(); + BOOST_REQUIRE( op2.voter == "charlie" ); + BOOST_REQUIRE( op2.author == "sam" ); + BOOST_REQUIRE( op2.permlink == "permlink2" ); + BOOST_REQUIRE( op2.weight == STEEM_100_PERCENT ); + + BOOST_REQUIRE( tx.ref_block_num == 1000 ); + BOOST_REQUIRE( tx.ref_block_prefix == 1000000000 ); + BOOST_REQUIRE( tx.expiration == fc::time_point_sec( 1514764800 + i ) ); + } + } + FC_LOG_AND_RETHROW(); +} + +BOOST_AUTO_TEST_CASE( unpack_recursion_test ) +{ + try + { + std::stringstream ss; + int recursion_level = 100000; + uint64_t allocation_per_level = 500000; + + for ( int i = 0; i < recursion_level; i++ ) + { + fc::raw::pack( ss, unsigned_int( allocation_per_level ) ); + fc::raw::pack( ss, static_cast< uint8_t >( variant::array_type ) ); + } + + std::vector< fc::variant > v; + STEEM_REQUIRE_THROW( fc::raw::unpack( ss, v ), fc::assert_exception ); + } + FC_LOG_AND_RETHROW(); +} + BOOST_AUTO_TEST_SUITE_END() #endif From aaacaaf9566978a4fe2c834918f6ddad1233609d Mon Sep 17 00:00:00 2001 From: Seungwon Eugene Jeong Date: Wed, 9 Jan 2019 18:48:04 +0000 Subject: [PATCH 06/15] Fix MacOS build instruction Fix Issue #3225 Fix "Could NOT find zlib (missing: ZLIB_LIBRARIES)" --- doc/building.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/building.md b/doc/building.md index b7c3b05e60..4e7e8d7ff8 100644 --- a/doc/building.md +++ b/doc/building.md @@ -185,12 +185,15 @@ Install Homebrew by following the instructions here: http://brew.sh/ openssl \ snappy \ zlib \ + bzip2 \ python3 pip3 install --user jinja2 Note: brew recently updated to boost 1.61.0, which is not yet supported by steem. Until then, this will allow you to install boost 1.60.0. +You may also need to install zlib and bzip2 libraries manually. +In that case, change the directories for `export` accordingly. *Optional.* To use TCMalloc in LevelDB: @@ -208,11 +211,11 @@ steem. Until then, this will allow you to install boost 1.60.0. ### Compile - export OPENSSL_ROOT_DIR=$(brew --prefix)/Cellar/openssl/1.0.2h_1/ export BOOST_ROOT=$(brew --prefix)/Cellar/boost@1.60/1.60.0/ - export SNAPPY_LIBRARIES=$(brew --prefix)/Cellar/snappy/1.1.7_1/lib/ - export SNAPPY_INCLUDE_DIR=$(brew --prefix)/Cellar/snappy/1.1.7_1/include/ - export ZLIB_LIBRARIES=$(brew --prefix)/Cellar/zlib/1.2.11/lib/ + export OPENSSL_ROOT_DIR=$(brew --prefix)/Cellar/openssl/1.0.2q/ + export SNAPPY_ROOT_DIR=$(brew --prefix)/Cellar/snappy/1.1.7_1 + export ZLIB_ROOT_DIR=$(brew --prefix)/Cellar/zlib/1.2.11 + export BZIP2_ROOT_DIR=$(brew --prefix)/Cellar/bzip2/1.0.6_1 git checkout stable git submodule update --init --recursive mkdir build && cd build From f398604b17dd72819ddf0dff05873ea26100c93d Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Tue, 22 Jan 2019 12:27:29 -0500 Subject: [PATCH 07/15] Limited recursive depth while parsing json --- libraries/fc/include/fc/io/json.hpp | 25 ++-- libraries/fc/include/fc/io/json_relaxed.hpp | 84 ++++++------ libraries/fc/src/io/json.cpp | 140 +++++++++++--------- 3 files changed, 138 insertions(+), 111 deletions(-) diff --git a/libraries/fc/include/fc/io/json.hpp b/libraries/fc/include/fc/io/json.hpp index 8a46d134fe..9e8d14eedb 100644 --- a/libraries/fc/include/fc/io/json.hpp +++ b/libraries/fc/include/fc/io/json.hpp @@ -2,6 +2,8 @@ #include #include +#define JSON_MAX_RECURSION_DEPTH (50) + namespace fc { class ostream; @@ -28,19 +30,19 @@ namespace fc legacy_generator = 1 }; - static ostream& to_stream( ostream& out, const fc::string&); + static ostream& to_stream( ostream& out, const fc::string& ); static ostream& to_stream( ostream& out, const variant& v, output_formatting format = stringify_large_ints_and_doubles ); static ostream& to_stream( ostream& out, const variants& v, output_formatting format = stringify_large_ints_and_doubles ); static ostream& to_stream( ostream& out, const variant_object& v, output_formatting format = stringify_large_ints_and_doubles ); - static variant from_stream( buffered_istream& in, parse_type ptype = legacy_parser ); + static variant from_stream( buffered_istream& in, parse_type ptype = legacy_parser, uint32_t depth = 0 ); - static variant from_string( const string& utf8_str, parse_type ptype = legacy_parser ); - static variants variants_from_string( const string& utf8_str, parse_type ptype = legacy_parser ); + static variant from_string( const string& utf8_str, parse_type ptype = legacy_parser, uint32_t depth = 0 ); + static variants variants_from_string( const string& utf8_str, parse_type ptype = legacy_parser, uint32_t depth = 0 ); static string to_string( const variant& v, output_formatting format = stringify_large_ints_and_doubles ); static string to_pretty_string( const variant& v, output_formatting format = stringify_large_ints_and_doubles ); - static bool is_valid( const std::string& json_str, parse_type ptype = legacy_parser ); + static bool is_valid( const std::string& json_str, parse_type ptype = legacy_parser, uint32_t depth = 0 ); template static void save_to_file( const T& v, const fc::path& fi, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles ) @@ -49,28 +51,29 @@ namespace fc } static void save_to_file( const variant& v, const fc::path& fi, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles ); - static variant from_file( const fc::path& p, parse_type ptype = legacy_parser ); + static variant from_file( const fc::path& p, parse_type ptype = legacy_parser, uint32_t depth = 0 ); template - static T from_file( const fc::path& p, parse_type ptype = legacy_parser ) + static T from_file( const fc::path& p, parse_type ptype = legacy_parser, uint32_t depth = 0 ) { - return json::from_file(p, ptype).as(); + depth++; + return json::from_file( p, ptype, depth ).as(); } template - static string to_string( const T& v, output_formatting format = stringify_large_ints_and_doubles ) + static string to_string( const T& v, output_formatting format = stringify_large_ints_and_doubles ) { return to_string( variant(v), format ); } template - static string to_pretty_string( const T& v, output_formatting format = stringify_large_ints_and_doubles ) + static string to_pretty_string( const T& v, output_formatting format = stringify_large_ints_and_doubles ) { return to_pretty_string( variant(v), format ); } template - static void save_to_file( const T& v, const std::string& p, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles ) + static void save_to_file( const T& v, const std::string& p, bool pretty = true, output_formatting format = stringify_large_ints_and_doubles ) { save_to_file( variant(v), fc::path(p), pretty ); } diff --git a/libraries/fc/include/fc/io/json_relaxed.hpp b/libraries/fc/include/fc/io/json_relaxed.hpp index e4876f259b..211c677d31 100644 --- a/libraries/fc/include/fc/io/json_relaxed.hpp +++ b/libraries/fc/include/fc/io/json_relaxed.hpp @@ -21,11 +21,12 @@ namespace fc { namespace json_relaxed { template - variant variant_from_stream( T& in ); + variant variant_from_stream( T& in, uint32_t depth ); template - fc::string tokenFromStream( T& in ) + fc::string tokenFromStream( T& in, uint32_t depth ) { + depth++; fc::stringstream token; try { @@ -36,7 +37,7 @@ namespace fc { namespace json_relaxed switch( c = in.peek() ) { case '\\': - token << parseEscape( in ); + token << parseEscape( in, depth ); break; case '\t': case ' ': @@ -81,8 +82,9 @@ namespace fc { namespace json_relaxed } template - fc::string quoteStringFromStream( T& in ) + fc::string quoteStringFromStream( T& in, uint32_t depth = 0 ) { + depth++; fc::stringstream token; try { @@ -140,7 +142,7 @@ namespace fc { namespace json_relaxed FC_THROW_EXCEPTION( parse_error_exception, "unexpected EOF in string '${token}'", ("token", token.str() ) ); else if( allow_escape && (c == '\\') ) - token << parseEscape( in ); + token << parseEscape( in, depth ); else { in.get(); @@ -163,7 +165,7 @@ namespace fc { namespace json_relaxed FC_THROW_EXCEPTION( parse_error_exception, "unexpected EOF in string '${token}'", ("token", token.str() ) ); else if( allow_escape && (c == '\\') ) - token << parseEscape( in ); + token << parseEscape( in, depth ); else if( (c == '\r') | (c == '\n') ) FC_THROW_EXCEPTION( parse_error_exception, "unexpected EOL in string '${token}'", ("token", token.str() ) ); @@ -179,10 +181,11 @@ namespace fc { namespace json_relaxed } template - fc::string stringFromStream( T& in ) + fc::string stringFromStream( T& in, uint32_t depth = 0 ) { try { + depth++; char c = in.peek(), c2; switch( c ) @@ -192,7 +195,7 @@ namespace fc { namespace json_relaxed FC_THROW_EXCEPTION( parse_error_exception, "expected: '\"' at beginning of string, got '\''" ); // falls through case '"': - return quoteStringFromStream( in ); + return quoteStringFromStream( in, depth ); case 'r': if( strict ) FC_THROW_EXCEPTION( parse_error_exception, "raw strings not supported in strict mode" ); @@ -205,11 +208,11 @@ namespace fc { namespace json_relaxed case '\'': if( strict ) FC_THROW_EXCEPTION( parse_error_exception, "raw strings not supported in strict mode" ); - return quoteStringFromStream( in ); + return quoteStringFromStream( in, depth ); default: if( strict ) FC_THROW_EXCEPTION( parse_error_exception, "unquoted strings not supported in strict mode" ); - return c+tokenFromStream( in ); + return c+tokenFromStream( in, depth ); } break; case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': @@ -225,7 +228,7 @@ namespace fc { namespace json_relaxed case '_': case '-': case '.': case '+': case '/': if( strict ) FC_THROW_EXCEPTION( parse_error_exception, "unquoted strings not supported in strict mode" ); - return tokenFromStream( in ); + return tokenFromStream( in, depth ); default: FC_THROW_EXCEPTION( parse_error_exception, "expected: string" ); } @@ -343,7 +346,7 @@ namespace fc { namespace json_relaxed } template - fc::variant parseNumberOrStr( const fc::string& token ) + fc::variant parseNumberOrStr( const fc::string& token, uint32_t depth = 0 ) { try { //ilog( (token) ); size_t i = 0, n = token.length(); @@ -562,8 +565,10 @@ namespace fc { namespace json_relaxed } FC_CAPTURE_AND_RETHROW( (token) ) } template - variant_object objectFromStream( T& in ) + variant_object objectFromStream( T& in, uint32_t depth = 0 ) { + depth++; + FC_ASSERT( depth <= JSON_MAX_RECURSION_DEPTH ); mutable_variant_object obj; try { @@ -573,7 +578,7 @@ namespace fc { namespace json_relaxed "Expected '{', but read '${char}'", ("char",string(&c, &c + 1)) ); in.get(); - skip_white_space(in); + skip_white_space( in, depth ); while( in.peek() != '}' ) { if( in.peek() == ',' ) @@ -581,19 +586,19 @@ namespace fc { namespace json_relaxed in.get(); continue; } - if( skip_white_space(in) ) continue; - string key = json_relaxed::stringFromStream( in ); - skip_white_space(in); + if( skip_white_space( in, depth ) ) continue; + string key = json_relaxed::stringFromStream( in, depth ); + skip_white_space( in, depth ); if( in.peek() != ':' ) { FC_THROW_EXCEPTION( parse_error_exception, "Expected ':' after key \"${key}\"", ("key", key) ); } in.get(); - auto val = json_relaxed::variant_from_stream( in ); + auto val = json_relaxed::variant_from_stream( in, depth ); obj(std::move(key),std::move(val)); - skip_white_space(in); + skip_white_space( in, depth ); } if( in.peek() == '}' ) { @@ -613,15 +618,17 @@ namespace fc { namespace json_relaxed } template - variants arrayFromStream( T& in ) + variants arrayFromStream( T& in, uint32_t depth = 0 ) { + depth++; + FC_ASSERT( depth <= JSON_MAX_RECURSION_DEPTH ); variants ar; try { if( in.peek() != '[' ) FC_THROW_EXCEPTION( parse_error_exception, "Expected '['" ); in.get(); - skip_white_space(in); + skip_white_space( in, depth ); while( in.peek() != ']' ) { @@ -630,9 +637,9 @@ namespace fc { namespace json_relaxed in.get(); continue; } - if( skip_white_space(in) ) continue; - ar.push_back( json_relaxed::variant_from_stream(in) ); - skip_white_space(in); + if( skip_white_space( in, depth ) ) continue; + ar.push_back( json_relaxed::variant_from_stream( in, depth ) ); + skip_white_space( in, depth ); } if( in.peek() != ']' ) FC_THROW_EXCEPTION( parse_error_exception, "Expected ']' after parsing ${variant}", @@ -645,19 +652,20 @@ namespace fc { namespace json_relaxed } template - variant numberFromStream( T& in ) + variant numberFromStream( T& in, uint32_t depth = 0 ) { try { - fc::string token = tokenFromStream(in); - variant result = json_relaxed::parseNumberOrStr( token ); + depth++; + fc::string token = tokenFromStream( in, depth ); + variant result = json_relaxed::parseNumberOrStr( token, depth ); if( strict && !(result.is_int64() || result.is_uint64() || result.is_double()) ) FC_THROW_EXCEPTION( parse_error_exception, "expected: number" ); return result; } FC_CAPTURE_AND_RETHROW() } template - variant wordFromStream( T& in ) + variant wordFromStream( T& in, uint32_t depth = 0 ) { - fc::string token = tokenFromStream(in); + fc::string token = tokenFromStream( in, depth ); FC_ASSERT( token.length() > 0 ); @@ -686,9 +694,11 @@ namespace fc { namespace json_relaxed } template - variant variant_from_stream( T& in ) + variant variant_from_stream( T& in, uint32_t depth ) { - skip_white_space(in); + depth++; + FC_ASSERT( depth <= JSON_MAX_RECURSION_DEPTH ); + skip_white_space( in, depth ); variant var; while( signed char c = in.peek() ) { @@ -701,11 +711,11 @@ namespace fc { namespace json_relaxed in.get(); continue; case '"': - return json_relaxed::stringFromStream( in ); + return json_relaxed::stringFromStream( in, depth ); case '{': - return json_relaxed::objectFromStream( in ); + return json_relaxed::objectFromStream( in, depth ); case '[': - return json_relaxed::arrayFromStream( in ); + return json_relaxed::arrayFromStream( in, depth ); case '-': case '+': case '.': @@ -719,7 +729,7 @@ namespace fc { namespace json_relaxed case '7': case '8': case '9': - return json_relaxed::numberFromStream( in ); + return json_relaxed::numberFromStream( in, depth ); // null, true, false, or 'warning' / string case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': @@ -730,13 +740,13 @@ namespace fc { namespace json_relaxed case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': case '/': - return json_relaxed::wordFromStream( in ); + return json_relaxed::wordFromStream( in, depth ); case 0x04: // ^D end of transmission case EOF: FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" ); default: FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", - ("c", c)("s", stringFromToken(in)) ); + ("c", c)("s", stringFromToken( in, depth )) ); } } return variant(); diff --git a/libraries/fc/src/io/json.cpp b/libraries/fc/src/io/json.cpp index 288f40d763..b3c27933f9 100644 --- a/libraries/fc/src/io/json.cpp +++ b/libraries/fc/src/io/json.cpp @@ -15,16 +15,16 @@ namespace fc { // forward declarations of provided functions - template variant variant_from_stream( T& in ); - template char parseEscape( T& in ); - template fc::string stringFromStream( T& in ); - template bool skip_white_space( T& in ); - template fc::string stringFromToken( T& in ); - template variant_object objectFromStream( T& in ); - template variants arrayFromStream( T& in ); - template variant number_from_stream( T& in ); - template variant token_from_stream( T& in ); - void escape_string( const string& str, ostream& os ); + template variant variant_from_stream( T& in, uint32_t depth = 0 ); + template char parseEscape( T& in, uint32_t depth = 0 ); + template fc::string stringFromStream( T& in, uint32_t depth = 0 ); + template bool skip_white_space( T& in, uint32_t depth = 0 ); + template fc::string stringFromToken( T& in, uint32_t depth = 0 ); + template variant_object objectFromStream( T& in, uint32_t depth = 0 ); + template variants arrayFromStream( T& in, uint32_t depth = 0 ); + template variant number_from_stream( T& in, uint32_t depth = 0 ); + template variant token_from_stream( T& in, uint32_t depth = 0 ); + void escape_string( const string& str, ostream& os, uint32_t depth = 0 ); template void to_stream( T& os, const variants& a, json::output_formatting format ); template void to_stream( T& os, const variant_object& o, json::output_formatting format ); template void to_stream( T& os, const variant& v, json::output_formatting format ); @@ -36,7 +36,7 @@ namespace fc namespace fc { template - char parseEscape( T& in ) + char parseEscape( T& in, uint32_t ) { if( in.peek() == '\\' ) { @@ -65,7 +65,7 @@ namespace fc } template - bool skip_white_space( T& in ) + bool skip_white_space( T& in, uint32_t ) { bool skipped = false; while( true ) @@ -86,7 +86,7 @@ namespace fc } template - fc::string stringFromStream( T& in ) + fc::string stringFromStream( T& in, uint32_t depth ) { fc::stringstream token; try @@ -104,7 +104,7 @@ namespace fc switch( c = in.peek() ) { case '\\': - token << parseEscape( in ); + token << parseEscape( in, depth ); break; case 0x04: FC_THROW_EXCEPTION( parse_error_exception, "EOF before closing '\"' in string '${token}'", @@ -123,7 +123,7 @@ namespace fc ("token", token.str() ) ); } template - fc::string stringFromToken( T& in ) + fc::string stringFromToken( T& in, uint32_t depth ) { fc::stringstream token; try @@ -135,7 +135,7 @@ namespace fc switch( c = in.peek() ) { case '\\': - token << parseEscape( in ); + token << parseEscape( in, depth ); break; case '\t': case ' ': @@ -168,8 +168,10 @@ namespace fc } template - variant_object objectFromStream( T& in ) + variant_object objectFromStream( T& in, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= JSON_MAX_RECURSION_DEPTH ); mutable_variant_object obj; try { @@ -179,7 +181,7 @@ namespace fc "Expected '{', but read '${char}'", ("char",string(&c, &c + 1)) ); in.get(); - skip_white_space(in); + skip_white_space( in, depth ); while( in.peek() != '}' ) { if( in.peek() == ',' ) @@ -187,19 +189,19 @@ namespace fc in.get(); continue; } - if( skip_white_space(in) ) continue; - string key = stringFromStream( in ); - skip_white_space(in); + if( skip_white_space( in, depth ) ) continue; + string key = stringFromStream( in, depth ); + skip_white_space( in, depth ); if( in.peek() != ':' ) { FC_THROW_EXCEPTION( parse_error_exception, "Expected ':' after key \"${key}\"", ("key", key) ); } in.get(); - auto val = variant_from_stream( in ); + auto val = variant_from_stream( in, depth ); obj(std::move(key),std::move(val)); - skip_white_space(in); + skip_white_space( in, depth ); } if( in.peek() == '}' ) { @@ -219,15 +221,17 @@ namespace fc } template - variants arrayFromStream( T& in ) + variants arrayFromStream( T& in, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= JSON_MAX_RECURSION_DEPTH ); variants ar; try { if( in.peek() != '[' ) FC_THROW_EXCEPTION( parse_error_exception, "Expected '['" ); in.get(); - skip_white_space(in); + skip_white_space( in, depth ); while( in.peek() != ']' ) { @@ -236,9 +240,9 @@ namespace fc in.get(); continue; } - if( skip_white_space(in) ) continue; - ar.push_back( variant_from_stream(in) ); - skip_white_space(in); + if( skip_white_space( in, depth ) ) continue; + ar.push_back( variant_from_stream( in, depth ) ); + skip_white_space( in, depth ); } if( in.peek() != ']' ) FC_THROW_EXCEPTION( parse_error_exception, "Expected ']' after parsing ${variant}", @@ -251,8 +255,9 @@ namespace fc } template - variant number_from_stream( T& in ) + variant number_from_stream( T& in, uint32_t depth ) { + depth++; fc::stringstream ss; bool dot = false; @@ -291,7 +296,7 @@ namespace fc default: if( isalnum( c ) ) { - return ss.str() + stringFromToken( in ); + return ss.str() + stringFromToken( in, depth ); } done = true; break; @@ -314,8 +319,9 @@ namespace fc return to_uint64(str); } template - variant token_from_stream( T& in ) + variant token_from_stream( T& in, uint32_t depth ) { + depth++; std::stringstream ss; ss.exceptions( std::ifstream::badbit ); bool received_eof = false; @@ -379,16 +385,18 @@ namespace fc // make out ("falfe") // A strict JSON parser would signal this as an error, but we // will just treat the malformed token as an un-quoted string. - return str + stringFromToken(in);; + return str + stringFromToken( in, depth ); } } } template - variant variant_from_stream( T& in ) + variant variant_from_stream( T& in, uint32_t depth ) { - skip_white_space(in); + depth++; + FC_ASSERT( depth <= JSON_MAX_RECURSION_DEPTH ); + skip_white_space( in, depth ); variant var; while( true ) { @@ -402,11 +410,11 @@ namespace fc in.get(); continue; case '"': - return stringFromStream( in ); + return stringFromStream( in, depth ); case '{': - return objectFromStream( in ); + return objectFromStream( in, depth ); case '[': - return arrayFromStream( in ); + return arrayFromStream( in, depth ); case '-': case '.': case '0': @@ -419,19 +427,19 @@ namespace fc case '7': case '8': case '9': - return number_from_stream( in ); + return number_from_stream( in, depth ); // null, true, false, or 'warning' / string case 'n': case 't': case 'f': - return token_from_stream( in ); + return token_from_stream( in, depth ); case 0x04: // ^D end of transmission case EOF: case 0: FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" ); default: FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"", - ("c", c)("s", stringFromToken(in)) ); + ("c", c)("s", stringFromToken( in, depth )) ); } } return variant(); @@ -457,8 +465,10 @@ namespace fc } } - variant json::from_string( const std::string& utf8_str, parse_type ptype ) + variant json::from_string( const std::string& utf8_str, parse_type ptype, uint32_t depth ) { try { + depth++; + FC_ASSERT( depth <= JSON_MAX_RECURSION_DEPTH ); check_string_depth( utf8_str ); fc::stringstream in( utf8_str ); @@ -466,20 +476,22 @@ namespace fc switch( ptype ) { case legacy_parser: - return variant_from_stream( in ); + return variant_from_stream( in, depth ); case legacy_parser_with_string_doubles: - return variant_from_stream( in ); + return variant_from_stream( in, depth ); case strict_parser: - return json_relaxed::variant_from_stream( in ); + return json_relaxed::variant_from_stream( in, depth ); case relaxed_parser: - return json_relaxed::variant_from_stream( in ); + return json_relaxed::variant_from_stream( in, depth ); default: FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); } } FC_RETHROW_EXCEPTIONS( warn, "", ("str",utf8_str) ) } - variants json::variants_from_string( const std::string& utf8_str, parse_type ptype ) + variants json::variants_from_string( const std::string& utf8_str, parse_type ptype, uint32_t depth ) { try { + depth++; + FC_ASSERT( depth <= JSON_MAX_RECURSION_DEPTH ); check_string_depth( utf8_str ); variants result; fc::stringstream in( utf8_str ); @@ -488,7 +500,7 @@ namespace fc while( true ) { // result.push_back( variant_from_stream( in )); - result.push_back(json_relaxed::variant_from_stream( in )); + result.push_back(json_relaxed::variant_from_stream( in, depth )); } } catch ( const fc::eof_exception& ){} return result; @@ -511,7 +523,7 @@ namespace fc * * All other characters are printed as UTF8. */ - void escape_string( const string& str, ostream& os ) + void escape_string( const string& str, ostream& os, uint32_t ) { os << '"'; for( auto itr = str.begin(); itr != str.end(); ++itr ) @@ -796,7 +808,7 @@ namespace fc fc::to_stream( o, v, format ); } } - variant json::from_file( const fc::path& p, parse_type ptype ) + variant json::from_file( const fc::path& p, parse_type ptype, uint32_t depth ) { //auto tmp = std::make_shared( p, ifstream::binary ); //auto tmp = std::make_shared( p.generic_string().c_str(), std::ios::binary ); @@ -805,29 +817,31 @@ namespace fc switch( ptype ) { case legacy_parser: - return variant_from_stream( bi ); + return variant_from_stream( bi, depth ); case legacy_parser_with_string_doubles: - return variant_from_stream( bi ); + return variant_from_stream( bi, depth ); case strict_parser: - return json_relaxed::variant_from_stream( bi ); + return json_relaxed::variant_from_stream( bi, depth ); case relaxed_parser: - return json_relaxed::variant_from_stream( bi ); + return json_relaxed::variant_from_stream( bi, depth ); default: FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); } } - variant json::from_stream( buffered_istream& in, parse_type ptype ) + variant json::from_stream( buffered_istream& in, parse_type ptype, uint32_t depth ) { + depth++; + FC_ASSERT( depth <= JSON_MAX_RECURSION_DEPTH ); switch( ptype ) { case legacy_parser: - return variant_from_stream( in ); + return variant_from_stream( in, depth ); case legacy_parser_with_string_doubles: - return variant_from_stream( in ); + return variant_from_stream( in, depth ); case strict_parser: - return json_relaxed::variant_from_stream( in ); + return json_relaxed::variant_from_stream( in, depth ); case relaxed_parser: - return json_relaxed::variant_from_stream( in ); + return json_relaxed::variant_from_stream( in, depth ); default: FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); } @@ -849,23 +863,23 @@ namespace fc return out; } - bool json::is_valid( const std::string& utf8_str, parse_type ptype ) + bool json::is_valid( const std::string& utf8_str, parse_type ptype, uint32_t depth ) { if( utf8_str.size() == 0 ) return false; fc::stringstream in( utf8_str ); switch( ptype ) { case legacy_parser: - variant_from_stream( in ); + variant_from_stream( in, depth ); break; case legacy_parser_with_string_doubles: - variant_from_stream( in ); + variant_from_stream( in, depth ); break; case strict_parser: - json_relaxed::variant_from_stream( in ); + json_relaxed::variant_from_stream( in, depth ); break; case relaxed_parser: - json_relaxed::variant_from_stream( in ); + json_relaxed::variant_from_stream( in, depth ); break; default: FC_ASSERT( false, "Unknown JSON parser type {ptype}", ("ptype", ptype) ); From 018adbdb9ec9b1ebb8b79c14baa4e7301be7dbee Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Tue, 22 Jan 2019 12:28:01 -0500 Subject: [PATCH 08/15] Increase recursive json limit to 200 --- libraries/fc/include/fc/io/json.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/fc/include/fc/io/json.hpp b/libraries/fc/include/fc/io/json.hpp index 9e8d14eedb..c26b5cb55a 100644 --- a/libraries/fc/include/fc/io/json.hpp +++ b/libraries/fc/include/fc/io/json.hpp @@ -2,7 +2,7 @@ #include #include -#define JSON_MAX_RECURSION_DEPTH (50) +#define JSON_MAX_RECURSION_DEPTH (200) namespace fc { From f640b420e16f7e5c4ba9bf622f935dd033df3e91 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Tue, 22 Jan 2019 12:51:31 -0500 Subject: [PATCH 09/15] Increment depth on wordFromStream --- libraries/fc/include/fc/io/json_relaxed.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/fc/include/fc/io/json_relaxed.hpp b/libraries/fc/include/fc/io/json_relaxed.hpp index 211c677d31..f4c3f8c244 100644 --- a/libraries/fc/include/fc/io/json_relaxed.hpp +++ b/libraries/fc/include/fc/io/json_relaxed.hpp @@ -665,6 +665,7 @@ namespace fc { namespace json_relaxed template variant wordFromStream( T& in, uint32_t depth = 0 ) { + depth++; fc::string token = tokenFromStream( in, depth ); FC_ASSERT( token.length() > 0 ); From 975931eba9f3eaeb6080185fa3f58644872785b1 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Tue, 5 Mar 2019 10:17:34 -0800 Subject: [PATCH 10/15] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a9a02b373..70548299b9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,7 @@ If there is an existing feature that is not working correctly, or a glitch in th ## Enhancement Suggestions -Do **not** use the issue tracker to suggest enhancements or improvements to the platform. The best place for these discussions is on Steemit.com. If there is a well vetted idea that has the support of the community that you feel should be considered by the development team, please email it to [sneak@steemit.com](mailto:sneak@steemit.com) for review. +Do **not** use the issue tracker to suggest enhancements or improvements to the platform. The best place for these discussions is on Steemit.com. If there is a well vetted idea that has the support of the community that you feel should be considered by the development team, please email it to [suggestions@steemit.com](mailto:suggestions@steemit.com) for review. ## Implementation Discussion From 644834b52ef141d3c3e5e5244bd775e1ceb9e8ee Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 8 Mar 2019 09:44:49 -0800 Subject: [PATCH 11/15] Add RC start block and account whitelisting for test net #3168 --- libraries/plugins/rc/rc_plugin.cpp | 59 ++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/libraries/plugins/rc/rc_plugin.cpp b/libraries/plugins/rc/rc_plugin.cpp index ee79fac825..35ed0ffe92 100644 --- a/libraries/plugins/rc/rc_plugin.cpp +++ b/libraries/plugins/rc/rc_plugin.cpp @@ -15,6 +15,8 @@ #include +#include + #define STEEM_RC_REGEN_TIME (60*60*24*5) // 2020.748973 VESTS == 1.000 STEEM when HF20 occurred on mainnet // TODO: What should this value be for testnet? @@ -73,6 +75,10 @@ class rc_plugin_impl std::map< account_name_type, int64_t > _account_to_max_rc; uint32_t _enable_at_block = 1; +#ifdef IS_TEST_NET + std::set< account_name_type > _whitelist; +#endif + boost::signals2::connection _post_apply_block_conn; boost::signals2::connection _pre_apply_transaction_conn; boost::signals2::connection _post_apply_transaction_conn; @@ -222,7 +228,12 @@ void use_account_rcs( const dynamic_global_property_object& gpo, const account_name_type& account_name, int64_t rc, - rc_plugin_skip_flags skip ) + rc_plugin_skip_flags skip +#ifdef IS_TEST_NET + , + set< account_name_type > whitelist +#endif + ) { if( account_name == account_name_type() ) { @@ -235,6 +246,10 @@ void use_account_rcs( return; } +#ifdef IS_TEST_NET + if( whitelist.count( account_name ) ) return; +#endif + // ilog( "use_account_rcs( ${n}, ${rc} )", ("n", account_name)("rc", rc) ); const account_object& account = db.get< account_object, by_name >( account_name ); const rc_account_object& rc_account = db.get< rc_account_object, by_name >( account_name ); @@ -323,7 +338,12 @@ void rc_plugin_impl::on_post_apply_transaction( const transaction_notification& } tx_info.resource_user = get_resource_user( note.transaction ); - use_account_rcs( _db, gpo, tx_info.resource_user, total_cost, _skip ); + use_account_rcs( _db, gpo, tx_info.resource_user, total_cost, _skip +#ifdef IS_TEST_NET + , + _whitelist +#endif + ); std::shared_ptr< exp_rc_data > export_data = steem::plugins::block_data_export::find_export_data< exp_rc_data >( STEEM_RC_PLUGIN_NAME ); @@ -1028,7 +1048,12 @@ void rc_plugin_impl::on_post_apply_optional_action( const optional_action_notifi } opt_action_info.resource_user = get_resource_user( note.action ); - use_account_rcs( _db, gpo, opt_action_info.resource_user, total_cost, _skip ); + use_account_rcs( _db, gpo, opt_action_info.resource_user, total_cost, _skip +#ifdef IS_TEST_NET + , + _whitelist +#endif + ); std::shared_ptr< exp_rc_data > export_data = steem::plugins::block_data_export::find_export_data< exp_rc_data >( STEEM_RC_PLUGIN_NAME ); @@ -1065,10 +1090,18 @@ void rc_plugin::set_program_options( options_description& cli, options_descripti cfg.add_options() ("rc-skip-reject-not-enough-rc", bpo::value()->default_value( false ), "Skip rejecting transactions when account has insufficient RCs. This is not recommended." ) ("rc-compute-historical-rc", bpo::value()->default_value( false ), "Generate historical resource credits" ) +#ifdef IS_TEST_NET + ("rc-start-at-block", bpo::value()->default_value(0), "Start calculating RCs at a specific block" ) + ("rc-account-whitelist", bpo::value< vector >()->composing(), "Ignore RC calculations for the whitelist" ) +#endif ; cli.add_options() ("rc-skip-reject-not-enough-rc", bpo::bool_switch()->default_value( false ), "Skip rejecting transactions when account has insufficient RCs. This is not recommended." ) ("rc-compute-historical-rc", bpo::bool_switch()->default_value( false ), "Generate historical resource credits" ) +#ifdef IS_TEST_NET + ("rc-start-at-block", bpo::value()->default_value(0), "Start calculating RCs at a specific block" ) + ("rc-account-whitelist", bpo::value< vector >()->composing(), "Ignore RC calculations for the whitelist" ) +#endif ; } @@ -1118,6 +1151,26 @@ void rc_plugin::plugin_initialize( const boost::program_options::variables_map& { my->_enable_at_block = STEEM_HF20_BLOCK_NUM; } +#else + uint32_t start_block = options.at( "rc-start-at-block" ).as(); + if( start_block > 0 ) + { + my->_enable_at_block = start_block; + } + + if( options.count( "rc-account-whitelist" ) > 0 ) + { + auto accounts = options.at( "rc-account-whitelist" ).as< vector< string > > (); + for( auto& arg : accounts ) + { + vector< string > names; + boost::split( names, arg, boost::is_any_of( " \t" ) ); + for( const std::string& name : names ) + my->_whitelist.insert( account_name_type( name ) ); + } + + ilog( "Ignoring RC's for accounts: ${w}", ("w", my->_whitelist) ); + } #endif appbase::app().get_plugin< chain::chain_plugin >().report_state_options( name(), state_opts ); From 977dea5aafe148a237bc27a693b50678f571083b Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Thu, 21 Mar 2019 09:37:02 -0700 Subject: [PATCH 12/15] Reduce cache size by 75% --- libraries/chain/database.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 1fa3d08b2a..b4855911d7 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -3165,7 +3165,7 @@ void database::_apply_block( const signed_block& next_block ) // last call of applying a block because it is the only thing that is not // reversible. migrate_irreversible_state(); - trim_cache( 40000000 ); + trim_cache( 10000000 ); } FC_CAPTURE_LOG_AND_RETHROW( (next_block.block_num()) ) } struct process_header_visitor From 6775bbfcbe52b4e32df40e8e8109654fc284dcfd Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 22 Mar 2019 14:27:06 -0700 Subject: [PATCH 13/15] Reduce cache by another 50% --- libraries/chain/database.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index b4855911d7..a95eee73d8 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -3165,7 +3165,7 @@ void database::_apply_block( const signed_block& next_block ) // last call of applying a block because it is the only thing that is not // reversible. migrate_irreversible_state(); - trim_cache( 10000000 ); + trim_cache( 5000000 ); } FC_CAPTURE_LOG_AND_RETHROW( (next_block.block_num()) ) } struct process_header_visitor From 46fe5f7c7999c5bfe35bf73bf84e2d6496652476 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Mon, 25 Mar 2019 09:21:04 -0700 Subject: [PATCH 14/15] Reduce another 10% --- libraries/chain/database.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index a95eee73d8..29041a6de5 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -3165,7 +3165,7 @@ void database::_apply_block( const signed_block& next_block ) // last call of applying a block because it is the only thing that is not // reversible. migrate_irreversible_state(); - trim_cache( 5000000 ); + trim_cache( 4500000 ); } FC_CAPTURE_LOG_AND_RETHROW( (next_block.block_num()) ) } struct process_header_visitor From 04ce691ede8c21870496e3ccbe8a2047e517a87a Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Tue, 26 Mar 2019 15:17:13 -0400 Subject: [PATCH 15/15] Fix MIRA Linux build --- libraries/mira/test/test_objects.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/mira/test/test_objects.hpp b/libraries/mira/test/test_objects.hpp index 08996fa904..d6976c944a 100644 --- a/libraries/mira/test/test_objects.hpp +++ b/libraries/mira/test/test_objects.hpp @@ -264,26 +264,26 @@ typedef mira::multi_index_container< chainbase::allocator< account_object > > account_index; -//FC_REFLECT( book::id_type, (_id) ) +FC_REFLECT( book::id_type, (_id) ) FC_REFLECT( book, (id)(a)(b) ) CHAINBASE_SET_INDEX_TYPE( book, book_index ) -//FC_REFLECT( single_index_object::id_type, (_id) ) +FC_REFLECT( single_index_object::id_type, (_id) ) FC_REFLECT( single_index_object, (id) ) CHAINBASE_SET_INDEX_TYPE( single_index_object, single_index_index ) -//FC_REFLECT( test_object::id_type, (_id) ) +FC_REFLECT( test_object::id_type, (_id) ) FC_REFLECT( test_object, (id)(val)(name) ) CHAINBASE_SET_INDEX_TYPE( test_object, test_object_index ) -//FC_REFLECT( test_object2::id_type, (_id) ) +FC_REFLECT( test_object2::id_type, (_id) ) FC_REFLECT( test_object2, (id)(val) ) CHAINBASE_SET_INDEX_TYPE( test_object2, test_object2_index ) -//FC_REFLECT( test_object3::id_type, (_id) ) +FC_REFLECT( test_object3::id_type, (_id) ) FC_REFLECT( test_object3, (id)(val)(val2)(val3) ) CHAINBASE_SET_INDEX_TYPE( test_object3, test_object3_index ) -//FC_REFLECT( account_object::id_type, (_id) ) +FC_REFLECT( account_object::id_type, (_id) ) FC_REFLECT( account_object, (id)(name) ) CHAINBASE_SET_INDEX_TYPE( account_object, account_index )