diff --git a/Dockerfile b/Dockerfile index e1ea159d29..4820714a6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ RUN \ pkg-config \ python3 \ python3-dev \ + python3-jinja2 \ python3-pip \ nginx \ fcgiwrap \ diff --git a/contrib/config-for-docker.ini b/contrib/config-for-docker.ini index ff2eb8dba7..80fdabf47b 100644 --- a/contrib/config-for-docker.ini +++ b/contrib/config-for-docker.ini @@ -1,3 +1,7 @@ +# this will need to increase depending on which plugins/apis you use +# 16 GB should be sufficient for a consensus node +shared-file-size = 16G + # Endpoint for P2P node to listen on # p2p-endpoint = diff --git a/contrib/fullnode.config.ini b/contrib/fullnode.config.ini index 3fd5eff6e6..b710e10e28 100644 --- a/contrib/fullnode.config.ini +++ b/contrib/fullnode.config.ini @@ -1,5 +1,5 @@ # Set larger shared-file-size than default -shared-file-size = 50G +shared-file-size = 48G # Set an API to be publicly available, may be specified multiple times public-api = database_api login_api account_by_key_api network_broadcast_api tag_api follow_api market_history_api raw_block_api @@ -11,16 +11,18 @@ enable-plugin = witness account_history account_by_key tags follow market_histor bcd-trigger = [[0,10],[85,300]] # Defines a range of accounts to track as a json pair ["from","to"] [from,to] -# track-account-range = +# track-account-range = + +history-whitelist-ops = transfer_operation transfer_to_vesting_operation withdraw_vesting_operation interest_operation transfer_to_savings_operation transfer_from_savings_operation cancel_transfer_from_savings_operation escrow_transfer_operation escrow_approve_operation escrow_dispute_operation escrow_release_operation fill_convert_request_operation fill_order_operation claim_reward_balance_operation author_reward_operation curation_reward_operation fill_vesting_withdraw_operation fill_transfer_from_savings_operation delegate_vesting_shares_operation return_vesting_delegation_operation comment_benefactor_reward_operation # Ignore posting operations, only track transfers and account updates -# filter-posting-ops = +# filter-posting-ops = # Database edits to apply on startup (may specify multiple times) -# edit-script = +# edit-script = # RPC endpoint of a trusted validating node (required) -# trusted-node = +# trusted-node = # Set the maximum size of cached feed for an account @@ -33,7 +35,7 @@ bucket-size = [15,60,300,3600,86400] history-per-size = 5760 # Defines a range of accounts to private messages to/from as a json pair ["from","to"] [from,to) -# pm-account-range = +# pm-account-range = # Enable block production, even if the chain is stale. enable-stale-production = false @@ -42,25 +44,25 @@ enable-stale-production = false required-participation = false # name of witness controlled by this node (e.g. initwitness ) -# witness = +# witness = # name of miner and its private key (e.g. ["account","WIF PRIVATE KEY"] ) -# miner = +# miner = # Number of threads to use for proof of work mining -# mining-threads = +# mining-threads = # WIF PRIVATE KEY to be used by one or more witnesses or miners -# private-key = +# private-key = # Account creation fee to be voted on upon successful POW - Minimum fee is 100.000 STEEM (written as 100000) -# miner-account-creation-fee = +# miner-account-creation-fee = # Maximum block size (in bytes) to be voted on upon successful POW - Max block size must be between 128 KB and 750 MB -# miner-maximum-block-size = +# miner-maximum-block-size = # SBD interest rate to be vote on upon successful POW - Default interest rate is 10% (written as 1000) -# miner-sbd-interest-rate = +# miner-sbd-interest-rate = # declare an appender named "stderr" that writes messages to the console [log.console_appender.stderr] @@ -74,7 +76,7 @@ filename=logs/p2p/p2p.log # route any messages logged to the default logger to the "stderr" logger we # declared above, if they are info level are higher [logger.default] -level=info +level=debug appenders=stderr # route messages sent to the "p2p" logger to stderr too diff --git a/contrib/steemd.run b/contrib/steemd.run index d2ebfd37ea..6df514f52d 100644 --- a/contrib/steemd.run +++ b/contrib/steemd.run @@ -26,7 +26,7 @@ fi # if user did pass in desired seed nodes, use # the ones the user specified: -if [[ ! -z "$STEEMD_SEED_NODES" ]]; then +if [[ ! -z "$STEEMD_SEED_NODES" ]]; then for NODE in $STEEMD_SEED_NODES ; do ARGS+=" --seed-node=$NODE" done @@ -47,9 +47,9 @@ fi if [[ ! -z "$TRACK_ACCOUNT" ]]; then if [[ ! "$USE_WAY_TOO_MUCH_RAM" ]]; then - ARGS+=" --enable-plugin=witness account_history" + ARGS+=" --enable-plugin=account_history" fi - ARGS+=" --track-account-range=[\"$TRACK_ACCOUNT\",\"$TRACK_ACCOUNT\"]" + ARGS+=" --track-account-range=[\"$TRACK_ACCOUNT\",\"$TRACK_ACCOUNT\"]" fi NOW=`date +%s` diff --git a/doc/building.md b/doc/building.md index 7cd8ac57c6..1fb01eae86 100644 --- a/doc/building.md +++ b/doc/building.md @@ -51,7 +51,9 @@ will build out of the box without further effort: libssl-dev \ libtool \ make \ - pkg-config + pkg-config \ + python3 \ + python3-jinja2 # Boost packages (also required) sudo apt-get install -y \ @@ -108,7 +110,9 @@ Here are the required packages: libreadline-dev \ libbz2-dev \ python-dev \ - perl + perl \ + python3 \ + python3-jinja2 The Boost provided in the Ubuntu 14.04 package manager (Boost 1.55) is too old. Steem requires Boost 1.58 (as in Ubuntu 16.04) and works with versions up to 1.60 (including). @@ -162,7 +166,6 @@ Install Homebrew by following the instructions here: http://brew.sh/ brew doctor brew update - brew tap homebrew/versions ### Install steem dependencies: @@ -171,10 +174,11 @@ Install Homebrew by following the instructions here: http://brew.sh/ automake \ cmake \ git \ - homebrew/versions/boost160 \ + boost160 \ libtool \ openssl \ - python3 + python3 \ + python3-jinja2 Note: brew recently updated to boost 1.61.0, which is not yet supported by steem. Until then, this will allow you to install boost 1.60.0. @@ -196,7 +200,7 @@ steem. Until then, this will allow you to install boost 1.60.0. ### Compile export OPENSSL_ROOT_DIR=$(brew --prefix)/Cellar/openssl/1.0.2h_1/ - export BOOST_ROOT=$(brew --prefix)/Cellar/boost160/1.60.0/ + export BOOST_ROOT=$(brew --prefix)/Cellar/boost@1.60/1.60.0/ git submodule update --init --recursive mkdir build && cd build cmake -DBOOST_ROOT="$BOOST_ROOT" -DCMAKE_BUILD_TYPE=Release .. diff --git a/doc/quickstart.md b/doc/quickstart.md index 62dfeff51c..354371f21b 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -47,17 +47,21 @@ docker run -d --env TRACK_ACCOUNT="yourexchangeid" steemit/steem Please make sure that you have enough resources available. Check `shared-file-size =` in your `config.ini` to reflect your needs. -Set it to at least 20% more than current size. +Set it to at least 25% more than current size. Provided values are expected to grow significantly over time. -Blockchain data takes over **12GB** of storage space. +Blockchain data takes over **16GB** of storage space. #### Full node -Shared memory file for full node uses over **45GB** +Shared memory file for full node uses over **65GB** + +#### Exchange node +Shared memory file for exchange node users over **16GB** +(tracked history for single account) #### Seed node -Shared memory file for seed node uses over **3GB** +Shared memory file for seed node uses over **5.5GB** #### Other use cases Shared memory file size varies, depends on your specific configuration but it is expected to be somewhere between "seed node" and "full node" usage. diff --git a/external_plugins/CMakeLists.txt b/external_plugins/CMakeLists.txt index d0f11ab492..6b4e26333b 100644 --- a/external_plugins/CMakeLists.txt +++ b/external_plugins/CMakeLists.txt @@ -1,16 +1,10 @@ # for each subdirectory containing a CMakeLists.txt, add that subdirectory -set( ENV{STEEMIT_EXTERNAL_PLUGINS} "" ) file( GLOB children RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} * ) foreach( child ${children} ) if( IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/${child}" ) if( EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${child}/CMakeLists.txt" ) add_subdirectory( "${child}" ) - set( ENV{STEEMIT_EXTERNAL_PLUGINS} $ENV{STEEMIT_EXTERNAL_PLUGINS} ${child} ) endif() endif() endforeach() - -# add_subdirectory( hello_api ) -# add_subdirectory( hello_arguments ) -# add_subdirectory( hello_objects ) \ No newline at end of file diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index b457f44a82..caac3c8b61 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -301,7 +301,7 @@ namespace detail { { try { - _chain_db->open(_data_dir / "blockchain", _shared_dir, 0, _shared_file_size, chainbase::database::read_write );\ + _chain_db->open(_data_dir / "blockchain", _shared_dir, STEEMIT_INIT_SUPPLY, _shared_file_size, chainbase::database::read_write );\ } catch( fc::assert_exception& ) { @@ -314,7 +314,7 @@ namespace detail { catch( chain::block_log_exception& ) { wlog( "Error opening block log. Having to resync from network..." ); - _chain_db->open( _data_dir / "blockchain", _shared_dir, 0, _shared_file_size, chainbase::database::read_write ); + _chain_db->open( _data_dir / "blockchain", _shared_dir, STEEMIT_INIT_SUPPLY, _shared_file_size, chainbase::database::read_write ); } } } @@ -328,7 +328,7 @@ namespace detail { else { ilog( "Starting Steem node in read mode." ); - _chain_db->open( _data_dir / "blockchain", _shared_dir, 0, _shared_file_size, chainbase::database::read_only ); + _chain_db->open( _data_dir / "blockchain", _shared_dir, STEEMIT_INIT_SUPPLY, _shared_file_size, chainbase::database::read_only ); if( _options->count( "read-forward-rpc" ) ) { diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp index 027bc69238..a07ddefc81 100755 --- a/libraries/app/database_api.cpp +++ b/libraries/app/database_api.cpp @@ -33,10 +33,7 @@ class database_api_impl : public std::enable_shared_from_this ~database_api_impl(); // Subscriptions - void set_subscribe_callback( std::function cb, bool clear_filter ); - void set_pending_transaction_callback( std::function cb ); void set_block_applied_callback( std::function cb ); - void cancel_all_subscriptions(); // Blocks and transactions optional get_block_header(uint32_t block_num)const; @@ -77,9 +74,6 @@ class database_api_impl : public std::enable_shared_from_this // signal handlers void on_applied_block( const chain::signed_block& b ); - mutable fc::bloom_filter _subscribe_filter; - std::function _subscribe_callback; - std::function _pending_trx_callback; std::function _block_applied_callback; steemit::chain::database& _db; @@ -114,41 +108,6 @@ void find_accounts( set& accounts, const discussion& d ) { // // ////////////////////////////////////////////////////////////////////// -void database_api::set_subscribe_callback( std::function cb, bool clear_filter ) -{ - my->_db.with_read_lock( [&]() - { - my->set_subscribe_callback( cb, clear_filter ); - }); -} - -void database_api_impl::set_subscribe_callback( std::function cb, bool clear_filter ) -{ - _subscribe_callback = cb; - if( clear_filter || !cb ) - { - static fc::bloom_parameters param; - param.projected_element_count = 10000; - param.false_positive_probability = 1.0/10000; - param.maximum_size = 1024*8*8*2; - param.compute_optimal_parameters(); - _subscribe_filter = fc::bloom_filter(param); - } -} - -void database_api::set_pending_transaction_callback( std::function cb ) -{ - my->_db.with_read_lock( [&]() - { - my->set_pending_transaction_callback( cb ); - }); -} - -void database_api_impl::set_pending_transaction_callback( std::function cb ) -{ - _pending_trx_callback = cb; -} - void database_api::set_block_applied_callback( std::function cb ) { my->_db.with_read_lock( [&]() @@ -175,19 +134,6 @@ void database_api_impl::set_block_applied_callback( std::function_db.with_read_lock( [&]() - { - my->cancel_all_subscriptions(); - }); -} - -void database_api_impl::cancel_all_subscriptions() -{ - set_subscribe_callback( std::function(), true); -} - ////////////////////////////////////////////////////////////////////// // // // Constructors // @@ -338,7 +284,7 @@ price database_api::get_current_median_history_price()const dynamic_global_property_api_obj database_api_impl::get_dynamic_global_properties()const { - return _db.get(dynamic_global_property_id_type()); + return dynamic_global_property_api_obj( _db.get( dynamic_global_property_id_type() ), _db ); } witness_schedule_api_obj database_api::get_witness_schedule()const @@ -2029,6 +1975,7 @@ state database_api::get_state( string path )const case operation::tag::value: case operation::tag::value: case operation::tag::value: + case operation::tag::value: default: eacnt.other_history[item.first] = item.second; } @@ -2357,7 +2304,7 @@ state database_api::get_state( string path )const annotated_signed_transaction database_api::get_transaction( transaction_id_type id )const { #ifdef SKIP_BY_TX_ID - return annotated_signed_transaction(); + FC_ASSERT( false, "This node's operator has disabled operation indexing by transaction_id" ); #else return my->_db.with_read_lock( [&](){ const auto& idx = my->_db.get_index().indices().get(); diff --git a/libraries/app/impacted.cpp b/libraries/app/impacted.cpp index f877f5da0e..1ebb991975 100644 --- a/libraries/app/impacted.cpp +++ b/libraries/app/impacted.cpp @@ -265,6 +265,13 @@ struct get_impacted_account_visitor _impacted.insert( op.benefactor ); _impacted.insert( op.author ); } + + void operator()( const producer_reward_operation& op ) + { + _impacted.insert( op.producer ); + } + + //void operator()( const operation& op ){} }; void operation_get_impacted_accounts( const operation& op, flat_set& result ) diff --git a/libraries/app/include/steemit/app/database_api.hpp b/libraries/app/include/steemit/app/database_api.hpp index 3020565887..e0475937b3 100755 --- a/libraries/app/include/steemit/app/database_api.hpp +++ b/libraries/app/include/steemit/app/database_api.hpp @@ -115,15 +115,7 @@ class database_api // Subscriptions // /////////////////// - void set_subscribe_callback( std::function cb, bool clear_filter ); - void set_pending_transaction_callback( std::function cb ); void set_block_applied_callback( std::function cb ); - /** - * @brief Stop receiving any notifications - * - * This unsubscribes from all subscribed markets and objects. - */ - void cancel_all_subscriptions(); vector get_trending_tags( string after_tag, uint32_t limit )const; @@ -457,10 +449,7 @@ FC_REFLECT_ENUM( steemit::app::withdraw_route_type, (incoming)(outgoing)(all) ); FC_API(steemit::app::database_api, // Subscriptions - (set_subscribe_callback) - (set_pending_transaction_callback) (set_block_applied_callback) - (cancel_all_subscriptions) // tags (get_trending_tags) diff --git a/libraries/app/include/steemit/app/steem_api_objects.hpp b/libraries/app/include/steemit/app/steem_api_objects.hpp index 64fecf38f9..75372cd701 100644 --- a/libraries/app/include/steemit/app/steem_api_objects.hpp +++ b/libraries/app/include/steemit/app/steem_api_objects.hpp @@ -42,7 +42,6 @@ using namespace steemit::chain; typedef chain::change_recovery_account_request_object change_recovery_account_request_api_obj; typedef chain::block_summary_object block_summary_api_obj; typedef chain::comment_vote_object comment_vote_api_obj; -typedef chain::dynamic_global_property_object dynamic_global_property_api_obj; typedef chain::convert_request_object convert_request_api_obj; typedef chain::escrow_object escrow_api_obj; typedef chain::liquidity_reward_balance_object liquidity_reward_balance_api_obj; @@ -478,6 +477,34 @@ struct signed_block_api_obj : public signed_block vector< transaction_id_type > transaction_ids; }; +struct dynamic_global_property_api_obj : public dynamic_global_property_object +{ + dynamic_global_property_api_obj( const dynamic_global_property_object& gpo, const chain::database& db ) : + dynamic_global_property_object( gpo ) + { + if( db.has_index< witness::reserve_ratio_index >() ) + { + const auto& r = db.find( witness::reserve_ratio_id_type() ); + + if( BOOST_LIKELY( r != nullptr ) ) + { + current_reserve_ratio = r->current_reserve_ratio; + average_block_size = r->average_block_size; + max_virtual_bandwidth = r->max_virtual_bandwidth; + } + } + } + + dynamic_global_property_api_obj( const dynamic_global_property_object& gpo ) : + dynamic_global_property_object( gpo ) {} + + dynamic_global_property_api_obj() {} + + uint32_t current_reserve_ratio = 0; + uint64_t average_block_size = 0; + uint128_t max_virtual_bandwidth = 0; +}; + } } // steemit::app FC_REFLECT( steemit::app::comment_api_obj, @@ -568,3 +595,9 @@ FC_REFLECT_DERIVED( steemit::app::signed_block_api_obj, (steemit::protocol::sign (signing_key) (transaction_ids) ) + +FC_REFLECT_DERIVED( steemit::app::dynamic_global_property_api_obj, (steemit::chain::dynamic_global_property_object), + (current_reserve_ratio) + (average_block_size) + (max_virtual_bandwidth) + ) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 46a2396466..6c5861d435 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -535,7 +535,7 @@ void database::_maybe_warn_multiple_production( uint32_t height )const witness_time_pairs.push_back( std::make_pair( b->data.witness, b->data.timestamp ) ); } - ilog( "Encountered block num collision at block ${n} due to a fork, witnesses are:", ("n", height)("w", witness_time_pairs) ); + ilog( "Encountered block num collision at block ${n} due to a fork, witnesses are: ${w}", ("n", height)("w", witness_time_pairs) ); } return; } @@ -1193,7 +1193,6 @@ void database::clear_witness_votes( const account_object& a ) remove(current); } - #warning( "TODO: Remove this check after HF 19" ) if( has_hardfork( STEEMIT_HARDFORK_0_6__104 ) ) modify( a, [&](account_object& acc ) { @@ -1665,8 +1664,7 @@ void database::process_comment_cashout() { fc::microseconds decay_rate; - #warning( "TODO: Remove temp reward fund after HF 19" ) - if( rfo.name == STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME || has_hardfork( STEEMIT_HARDFORK_0_19__1051 ) ) + if( has_hardfork( STEEMIT_HARDFORK_0_19__1051 ) ) decay_rate = STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19; else decay_rate = STEEMIT_RECENT_RSHARES_DECAY_RATE_HF17; @@ -1692,15 +1690,12 @@ void database::process_comment_cashout() // add all rshares about to be cashed out to the reward funds. This ensures equal satoshi per rshare payment if( has_hardfork( STEEMIT_HARDFORK_0_17__771 ) ) { - const auto& linear = get< reward_fund_object, by_name >( STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME ); - while( current != cidx.end() && current->cashout_time <= head_block_time() ) { if( current->net_rshares > 0 ) { const auto& rf = get_reward_fund( *current ); funds[ rf.id._id ].recent_claims += util::evaluate_reward_curve( current->net_rshares.value, rf.author_reward_curve, rf.content_constant ); - funds[ STEEMIT_TEMP_LINEAR_REWARD_FUND_ID ].recent_claims += util::evaluate_reward_curve( current->net_rshares.value, linear.author_reward_curve, linear.content_constant ); } ++current; @@ -1829,7 +1824,9 @@ void database::process_funds() p.virtual_supply += asset( new_steem, STEEM_SYMBOL ); }); - create_vesting( get_account( cwit.owner ), asset( witness_reward, STEEM_SYMBOL ) ); + const auto& producer_reward = create_vesting( get_account( cwit.owner ), asset( witness_reward, STEEM_SYMBOL ) ); + push_virtual_operation( producer_reward_operation( cwit.owner, producer_reward ) ); + } else { @@ -1914,7 +1911,8 @@ asset database::get_producer_reward() /// pay witness in vesting shares if( props.head_block_number >= STEEMIT_START_MINER_VOTING_BLOCK || (witness_account.vesting_shares.amount.value == 0) ) { // const auto& witness_obj = get_witness( props.current_witness ); - create_vesting( witness_account, pay ); + const auto& producer_reward = create_vesting( witness_account, pay ); + push_virtual_operation( producer_reward_operation( witness_account.name, producer_reward ) ); } else { @@ -2565,12 +2563,20 @@ void database::apply_block( const signed_block& next_block, uint32_t skip ) void database::show_free_memory( bool force ) { - uint32_t free_gb = uint32_t(get_free_memory() / (1024*1024*1024)); + uint32_t free_gb = uint32_t( get_free_memory() / (1024*1024*1024) ); if( force || (free_gb < _last_free_gb_printed) || (free_gb > _last_free_gb_printed+1) ) { ilog( "Free memory is now ${n}G", ("n", free_gb) ); _last_free_gb_printed = free_gb; } + + if( free_gb == 0 ) + { + uint32_t free_mb = uint32_t( get_free_memory() / (1024*1024) ); + + if( free_mb <= 100 && head_block_num() % 10 == 0 ) + elog( "Free memory is now ${n}M. Increase shared file size immediately!" , ("n", free_mb) ); + } } void database::_apply_block( const signed_block& next_block ) @@ -2844,18 +2850,18 @@ void database::_apply_transaction(const signed_transaction& trx) { const auto& tapos_block_summary = get< block_summary_object >( trx.ref_block_num ); //Verify TaPoS block summary has correct ID prefix, and that this block's time is not past the expiration - FC_ASSERT( trx.ref_block_prefix == tapos_block_summary.block_id._hash[1], + STEEMIT_ASSERT( trx.ref_block_prefix == tapos_block_summary.block_id._hash[1], transaction_tapos_exception, "", ("trx.ref_block_prefix", trx.ref_block_prefix) ("tapos_block_summary",tapos_block_summary.block_id._hash[1])); } fc::time_point_sec now = head_block_time(); - FC_ASSERT( trx.expiration <= now + fc::seconds(STEEMIT_MAX_TIME_UNTIL_EXPIRATION), "", - ("trx.expiration",trx.expiration)("now",now)("max_til_exp",STEEMIT_MAX_TIME_UNTIL_EXPIRATION)); + STEEMIT_ASSERT( trx.expiration <= now + fc::seconds(STEEMIT_MAX_TIME_UNTIL_EXPIRATION), transaction_expiration_exception, + "", ("trx.expiration",trx.expiration)("now",now)("max_til_exp",STEEMIT_MAX_TIME_UNTIL_EXPIRATION)); if( has_hardfork( STEEMIT_HARDFORK_0_9 ) ) // Simple solution to pending trx bug when now == trx.expiration - FC_ASSERT( now < trx.expiration, "", ("now",now)("trx.exp",trx.expiration) ); - FC_ASSERT( now <= trx.expiration, "", ("now",now)("trx.exp",trx.expiration) ); + STEEMIT_ASSERT( now < trx.expiration, transaction_expiration_exception, "", ("now",now)("trx.exp",trx.expiration) ); + STEEMIT_ASSERT( now <= trx.expiration, transaction_expiration_exception, "", ("now",now)("trx.exp",trx.expiration) ); } //Insert transaction into unique transactions database. @@ -2923,7 +2929,6 @@ void database::create_block_summary(const signed_block& next_block) void database::update_global_dynamic_data( const signed_block& b ) { try { - auto block_size = fc::raw::pack_size(b); const dynamic_global_property_object& _dgp = get_dynamic_global_properties(); @@ -2969,41 +2974,6 @@ void database::update_global_dynamic_data( const signed_block& b ) dgp.head_block_id = b.id(); dgp.time = b.timestamp; dgp.current_aslot += missed_blocks+1; - dgp.average_block_size = (99 * dgp.average_block_size + block_size)/100; - - /** - * About once per minute the average network use is consulted and used to - * adjust the reserve ratio. Anything above 50% usage reduces the ratio by - * half which should instantly bring the network from 50% to 25% use unless - * the demand comes from users who have surplus capacity. In other words, - * a 50% reduction in reserve ratio does not result in a 50% reduction in usage, - * it will only impact users who where attempting to use more than 50% of their - * capacity. - * - * When the reserve ratio is at its max (10,000) a 50% reduction will take 3 to - * 4 days to return back to maximum. When it is at its minimum it will return - * back to its prior level in just a few minutes. - * - * If the network reserve ratio falls under 100 then it is probably time to - * increase the capacity of the network. - */ - if( dgp.head_block_number % 20 == 0 ) - { - if( ( !has_hardfork( STEEMIT_HARDFORK_0_12__179 ) && dgp.average_block_size > dgp.maximum_block_size / 2 ) || - ( has_hardfork( STEEMIT_HARDFORK_0_12__179 ) && dgp.average_block_size > dgp.maximum_block_size / 4 ) ) - { - dgp.current_reserve_ratio /= 2; /// exponential back up - } - else - { /// linear growth... not much fine grain control near full capacity - dgp.current_reserve_ratio++; - } - - if( has_hardfork( STEEMIT_HARDFORK_0_2 ) && dgp.current_reserve_ratio > STEEMIT_MAX_RESERVE_RATIO ) - dgp.current_reserve_ratio = STEEMIT_MAX_RESERVE_RATIO; - } - dgp.max_virtual_bandwidth = (dgp.maximum_block_size * dgp.current_reserve_ratio * - STEEMIT_BANDWIDTH_PRECISION * STEEMIT_BANDWIDTH_AVERAGE_WINDOW_SECONDS) / STEEMIT_BLOCK_INTERVAL; } ); if( !(get_node_properties().skip_flags & skip_undo_history_check) ) @@ -3368,7 +3338,8 @@ void database::adjust_balance( const account_object& a, const asset& delta ) acnt.sbd_seconds = 0; acnt.sbd_last_interest_payment = head_block_time(); - push_virtual_operation( interest_operation( a.name, interest_paid ) ); + if(interest > 0) + push_virtual_operation( interest_operation( a.name, interest_paid ) ); modify( get_dynamic_global_properties(), [&]( dynamic_global_property_object& props) { @@ -3412,7 +3383,8 @@ void database::adjust_savings_balance( const account_object& a, const asset& del acnt.savings_sbd_seconds = 0; acnt.savings_sbd_last_interest_payment = head_block_time(); - push_virtual_operation( interest_operation( a.name, interest_paid ) ); + if(interest > 0) + push_virtual_operation( interest_operation( a.name, interest_paid ) ); modify( get_dynamic_global_properties(), [&]( dynamic_global_property_object& props) { @@ -3798,23 +3770,9 @@ void database::apply_hardfork( uint32_t hardfork ) rfo.curation_reward_curve = curve_id::quadratic_curation; }); - auto linear_rf = create< reward_fund_object >( [&]( reward_fund_object& rfo ) - { - rfo.name = STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME; - rfo.last_update = head_block_time(); - rfo.content_constant = 0; - rfo.percent_curation_rewards = STEEMIT_1_PERCENT * 25; - rfo.percent_content_rewards = 0; - rfo.reward_balance = asset( 0, STEEM_SYMBOL ); - rfo.recent_claims = 0; - rfo.author_reward_curve = curve_id::linear; - rfo.curation_reward_curve = curve_id::square_root; - }); - // As a shortcut in payout processing, we use the id as an array index. // The IDs must be assigned this way. The assertion is a dummy check to ensure this happens. FC_ASSERT( post_rf.id._id == 0 ); - FC_ASSERT( linear_rf.id._id == STEEMIT_TEMP_LINEAR_REWARD_FUND_ID ); modify( gpo, [&]( dynamic_global_property_object& g ) { @@ -3877,40 +3835,15 @@ void database::apply_hardfork( uint32_t hardfork ) gpo.vote_power_reserve_rate = 10; }); - const auto& linear = get< reward_fund_object, by_name >( STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME ); modify( get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ), [&]( reward_fund_object &rfo ) { - #warning( "TODO: Replace with constant after HF 19" ) - rfo.recent_claims = linear.recent_claims; +#ifndef IS_TEST_NET + rfo.recent_claims = STEEMIT_HF_19_RECENT_CLAIMS; +#endif rfo.author_reward_curve = curve_id::linear; rfo.curation_reward_curve = curve_id::square_root; }); - #warning( "TODO: Remove weight conversion after HF 19" ) - const auto& cidx = get_index< comment_index, by_cashout_time >(); - const auto& vidx = get_index< comment_vote_index, by_comment_voter >(); - - /* - * Iterator through all comments that have not yet been paid, setting their total vote weight to the sqrt weight - * and update their votes as well. - */ - for( auto c_itr = cidx.begin(); c_itr != cidx.end() && c_itr->cashout_time < fc::time_point_sec::maximum(); ++c_itr ) - { - modify( *c_itr, [&]( comment_object& c ) - { - c.total_vote_weight = c.total_sqrt_vote_weight; - }); - - for( auto v_itr = vidx.lower_bound( c_itr->id ); v_itr != vidx.end() && v_itr->comment == c_itr->id; ++v_itr ) - { - modify( *v_itr, [&]( comment_vote_object& v ) - { - v.weight = v.sqrt_weight; - }); - } - } - - #warning( "TODO: Remove if 0 delegation opjects are not created in pre HF19 consensus" ) /* Remove all 0 delegation objects */ vector< const vesting_delegation_object* > to_remove; const auto& delegation_idx = get_index< vesting_delegation_index, by_id >(); diff --git a/libraries/chain/hardfork.d/0_19.hf b/libraries/chain/hardfork.d/0_19.hf index 2b78f1689f..7d0935cc95 100644 --- a/libraries/chain/hardfork.d/0_19.hf +++ b/libraries/chain/hardfork.d/0_19.hf @@ -15,4 +15,8 @@ #define STEEMIT_HARDFORK_0_19_VERSION hardfork_version( 0, 19 ) +#define STEEMIT_HF_19_RECENT_CLAIMS (fc::uint128_t(uint64_t(140797515942543623ull))) +#define STEEMIT_HARDFORK_0_19_ACTUAL_TIME (fc::time_point_sec(1497970809)) // Tue, 20 Jun 2017 15:00:09 UTC +#define STEEMIT_HF_19_SQRT_PRE_CALC (fc::time_point_sec( STEEMIT_HARDFORK_0_19_ACTUAL_TIME - STEEMIT_CASHOUT_WINDOW_SECONDS )) + #endif diff --git a/libraries/chain/include/steemit/chain/comment_object.hpp b/libraries/chain/include/steemit/chain/comment_object.hpp index 7ae09db327..5b3f3c9bf3 100644 --- a/libraries/chain/include/steemit/chain/comment_object.hpp +++ b/libraries/chain/include/steemit/chain/comment_object.hpp @@ -78,8 +78,6 @@ namespace steemit { namespace chain { time_point_sec cashout_time; /// 24 hours from the weighted average of vote time time_point_sec max_cashout_time; uint64_t total_vote_weight = 0; /// the total weight of voting rewards, used to calculate pro-rata share of curation payouts - #warning( "Remove total_sqrt_vote_weight after HF 19" ) - uint64_t total_sqrt_vote_weight = 0; /// Temporary total weight of votes for post hf19 curation reward curve uint16_t reward_weight = 0; @@ -122,8 +120,6 @@ namespace steemit { namespace chain { account_id_type voter; comment_id_type comment; uint64_t weight = 0; ///< defines the score this vote receives, used by vote payout calc. 0 if a negative vote or changed votes. - #warning( "Remove sqrt_weight after HF 19. It is a temporary field" ) - uint64_t sqrt_weight = 0; ///< Temporary vote weight for post hf19 curation curve int64_t rshares = 0; ///< The number of rshares this vote is responsible for int16_t vote_percent = 0; ///< The percent weight of the vote time_point_sec last_update; ///< The time of the last update of the vote @@ -254,13 +250,13 @@ FC_REFLECT( steemit::chain::comment_object, (depth)(children) (net_rshares)(abs_rshares)(vote_rshares) (children_abs_rshares)(cashout_time)(max_cashout_time) - (total_vote_weight)(total_sqrt_vote_weight)(reward_weight)(total_payout_value)(curator_payout_value)(beneficiary_payout_value)(author_rewards)(net_votes)(root_comment) + (total_vote_weight)(reward_weight)(total_payout_value)(curator_payout_value)(beneficiary_payout_value)(author_rewards)(net_votes)(root_comment) (max_accepted_payout)(percent_steem_dollars)(allow_replies)(allow_votes)(allow_curation_rewards) (beneficiaries) ) CHAINBASE_SET_INDEX_TYPE( steemit::chain::comment_object, steemit::chain::comment_index ) FC_REFLECT( steemit::chain::comment_vote_object, - (id)(voter)(comment)(weight)(sqrt_weight)(rshares)(vote_percent)(last_update)(num_changes) + (id)(voter)(comment)(weight)(rshares)(vote_percent)(last_update)(num_changes) ) CHAINBASE_SET_INDEX_TYPE( steemit::chain::comment_vote_object, steemit::chain::comment_vote_index ) diff --git a/libraries/chain/include/steemit/chain/database_exceptions.hpp b/libraries/chain/include/steemit/chain/database_exceptions.hpp index fa71fce399..67927ef129 100644 --- a/libraries/chain/include/steemit/chain/database_exceptions.hpp +++ b/libraries/chain/include/steemit/chain/database_exceptions.hpp @@ -75,6 +75,9 @@ namespace steemit { namespace chain { FC_DECLARE_DERIVED_EXCEPTION( plugin_exception, steemit::chain::chain_exception, 4100000, "plugin exception" ) FC_DECLARE_DERIVED_EXCEPTION( block_log_exception, steemit::chain::chain_exception, 4110000, "block log exception" ) + FC_DECLARE_DERIVED_EXCEPTION( transaction_expiration_exception, steemit::chain::transaction_exception, 4030100, "transaction expiration exception" ) + FC_DECLARE_DERIVED_EXCEPTION( transaction_tapos_exception, steemit::chain::transaction_exception, 4030200, "transaction tapos exception" ) + FC_DECLARE_DERIVED_EXCEPTION( pop_empty_chain, steemit::chain::undo_database_exception, 4070001, "there are no blocks to pop" ) STEEMIT_DECLARE_OP_BASE_EXCEPTIONS( transfer ); diff --git a/libraries/chain/include/steemit/chain/db_with.hpp b/libraries/chain/include/steemit/chain/db_with.hpp index 8e9f30befb..89c413c6f0 100644 --- a/libraries/chain/include/steemit/chain/db_with.hpp +++ b/libraries/chain/include/steemit/chain/db_with.hpp @@ -75,12 +75,22 @@ struct pending_transactions_restorer _db._push_transaction( tx ); } } + catch( const transaction_exception& e ) + { + dlog( "Pending transaction became invalid after switching to block ${b} ${n} ${t}", + ("b", _db.head_block_id())("n", _db.head_block_num())("t", _db.head_block_time()) ); + dlog( "The invalid transaction caused exception ${e}", ("e", e.to_detail_string()) ); + dlog( "${t}", ("t", tx) ); + } catch( const fc::exception& e ) { - //wlog( "Pending transaction became invalid after switching to block ${b} ${t}", ("b", _db.head_block_id())("t",_db.head_block_time()) ); - //wlog( "The invalid pending transaction caused exception ${e}", ("e", e.to_detail_string() ) ); - + /* + dlog( "Pending transaction became invalid after switching to block ${b} ${n} ${t}", + ("b", _db.head_block_id())("n", _db.head_block_num())("t", _db.head_block_time()) ); + dlog( "The invalid pending transaction caused exception ${e}", ("e", e.to_detail_string() ) ); + dlog( "${t}", ("t", tx) ); + */ } } } diff --git a/libraries/chain/include/steemit/chain/global_property_object.hpp b/libraries/chain/include/steemit/chain/global_property_object.hpp index 075feceaca..fcb6ea093b 100644 --- a/libraries/chain/include/steemit/chain/global_property_object.hpp +++ b/libraries/chain/include/steemit/chain/global_property_object.hpp @@ -82,16 +82,6 @@ namespace steemit { namespace chain { uint16_t sbd_print_rate = STEEMIT_100_PERCENT; - /** - * Average block size is updated every block to be: - * - * average_block_size = (99 * average_block_size + new_block_size) / 100 - * - * This property is used to update the current_reserve_ratio to maintain approximately - * 50% or less utilization of network capacity. - */ - uint32_t average_block_size = 0; - /** * Maximum block size is decided by the set of active witnesses which change every round. * Each witness posts what they think the maximum size should be as part of their witness @@ -117,25 +107,6 @@ namespace steemit { namespace chain { uint32_t last_irreversible_block_num = 0; - /** - * The maximum bandwidth the blockchain can support is: - * - * max_bandwidth = maximum_block_size * STEEMIT_BANDWIDTH_AVERAGE_WINDOW_SECONDS / STEEMIT_BLOCK_INTERVAL - * - * The maximum virtual bandwidth is: - * - * max_bandwidth * current_reserve_ratio - */ - uint64_t max_virtual_bandwidth = 0; - - /** - * Any time average_block_size <= 50% maximum_block_size this value grows by 1 until it - * reaches STEEMIT_MAX_RESERVE_RATIO. Any time average_block_size is greater than - * 50% it falls by 1%. Upward adjustments happen once per round, downward adjustments - * happen every block. - */ - uint64_t current_reserve_ratio = 1; - /** * The number of votes regenerated per day. Any user voting slower than this rate will be * "wasting" voting power through spillover; any user voting faster than this rate will have @@ -176,14 +147,11 @@ FC_REFLECT( steemit::chain::dynamic_global_property_object, (pending_rewarded_vesting_steem) (sbd_interest_rate) (sbd_print_rate) - (average_block_size) (maximum_block_size) (current_aslot) (recent_slots_filled) (participation_count) (last_irreversible_block_num) - (max_virtual_bandwidth) - (current_reserve_ratio) (vote_power_reserve_rate) ) CHAINBASE_SET_INDEX_TYPE( steemit::chain::dynamic_global_property_object, steemit::chain::dynamic_global_property_index ) diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index a1296449cd..b4cf5be836 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -122,16 +122,6 @@ void account_create_evaluator::do_apply( const account_create_operation& o ) ("f", wso.median_props.account_creation_fee * asset( STEEMIT_CREATE_ACCOUNT_WITH_STEEM_MODIFIER, STEEM_SYMBOL ) ) ("p", o.fee) ); } - else if( _db.has_hardfork( STEEMIT_HARDFORK_0_17__818 ) ) - { - #warning( "TODO: Remove after HF19. Entire conditional might be able to be removed." ) - if( _db.is_producing() ) - FC_ASSERT( false, "account_create_operation is temporarily disabled. Please use account_create_with_delegation_operation instead" ); - const witness_schedule_object& wso = _db.get_witness_schedule_object(); - FC_ASSERT( o.fee >= wso.median_props.account_creation_fee * asset( STEEMIT_CREATE_ACCOUNT_WITH_STEEM_MODIFIER, STEEM_SYMBOL ), "Insufficient Fee: ${f} required, ${p} provided.", - ("f", wso.median_props.account_creation_fee * asset( STEEMIT_CREATE_ACCOUNT_WITH_STEEM_MODIFIER, STEEM_SYMBOL ) ) - ("p", o.fee) ); - } else if( _db.has_hardfork( STEEMIT_HARDFORK_0_1 ) ) { const witness_schedule_object& wso = _db.get_witness_schedule_object(); @@ -272,9 +262,7 @@ void account_create_with_delegation_evaluator::do_apply( const account_create_wi auth.last_owner_update = fc::time_point_sec::min(); }); - #warning( "TODO: Check if not creating 0 delegation objects in HF19 passes consensus. Expectation is it should work." ) - if( ( _db.has_hardfork( STEEMIT_HARDFORK_0_19__997 ) && o.delegation.amount > 0 ) - || !_db.has_hardfork( STEEMIT_HARDFORK_0_19__997 ) ) + if( o.delegation.amount > 0 || !_db.has_hardfork( STEEMIT_HARDFORK_0_19__997 ) ) { _db.create< vesting_delegation_object >( [&]( vesting_delegation_object& vdo ) { @@ -383,10 +371,9 @@ void delete_comment_evaluator::do_apply( const delete_comment_operation& o ) if( _db.has_hardfork( STEEMIT_HARDFORK_0_19__876 ) ) FC_ASSERT( comment.cashout_time != fc::time_point_sec::maximum() ); - #warning( "TODO: Remove is_producing check after HF19. Conditional may be removed entirely after HF." ) - if( _db.is_producing() || _db.has_hardfork( STEEMIT_HARDFORK_0_19__977 ) ) { + if( _db.has_hardfork( STEEMIT_HARDFORK_0_19__977 ) ) FC_ASSERT( comment.net_rshares <= 0, "Cannot delete a comment with net positive votes." ); - } + if( comment.net_rshares > 0 ) return; const auto& vote_idx = _db.get_index().indices().get(); @@ -1305,7 +1292,6 @@ void vote_evaluator::do_apply( const vote_operation& o ) old_rshares = util::evaluate_reward_curve( old_rshares ); uint64_t max_vote_weight = 0; - uint64_t sqrt_max_vote_weight = 0; /** this verifies uniqueness of voter * @@ -1358,24 +1344,11 @@ void vote_evaluator::do_apply( const vote_operation& o ) if( _db.has_hardfork( STEEMIT_HARDFORK_0_17__774 ) ) { const auto& reward_fund = _db.get_reward_fund( comment ); - uint64_t old_weight = util::evaluate_reward_curve( old_vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ).to_uint64(); - uint64_t new_weight = util::evaluate_reward_curve( comment.vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ).to_uint64(); + auto curve = !_db.has_hardfork( STEEMIT_HARDFORK_0_19__1052 ) && comment.created > STEEMIT_HF_19_SQRT_PRE_CALC + ? curve_id::square_root : reward_fund.curation_reward_curve; + uint64_t old_weight = util::evaluate_reward_curve( old_vote_rshares.value, curve, reward_fund.content_constant ).to_uint64(); + uint64_t new_weight = util::evaluate_reward_curve( comment.vote_rshares.value, curve, reward_fund.content_constant ).to_uint64(); cv.weight = new_weight - old_weight; - #warning( "TODO: After HF19 caclulate which comment was the first to be paid on sqrt curation and activate the logic based on time" ) - -#ifndef IS_TEST_NET -/* - * Disabling this check so we can test the precalculation logic. - * It is not needed on live because the precalculation is only needed pre HF19 - */ - if( !_db.has_hardfork( STEEMIT_HARDFORK_0_19__1052 ) ) -#endif - { - old_weight = util::evaluate_reward_curve( old_vote_rshares.value, curve_id::square_root ).to_uint64(); - new_weight = util::evaluate_reward_curve( comment.vote_rshares.value, curve_id::square_root ).to_uint64(); - cv.sqrt_weight = new_weight - old_weight; - sqrt_max_vote_weight = cv.sqrt_weight; - } } else if ( _db.has_hardfork( STEEMIT_HARDFORK_0_1 ) ) { @@ -1402,15 +1375,6 @@ void vote_evaluator::do_apply( const vote_operation& o ) w *= delta_t; w /= STEEMIT_REVERSE_AUCTION_WINDOW_SECONDS; cv.weight = w.to_uint64(); -#ifndef IS_TEST_NET - if( _db.has_hardfork( STEEMIT_HARDFORK_0_17 ) ) -#endif - { - uint128_t w(sqrt_max_vote_weight); - w *= delta_t; - w /= STEEMIT_REVERSE_AUCTION_WINDOW_SECONDS; - cv.sqrt_weight = w.to_uint64(); - } } } else @@ -1424,7 +1388,6 @@ void vote_evaluator::do_apply( const vote_operation& o ) _db.modify( comment, [&]( comment_object& c ) { c.total_vote_weight += max_vote_weight; - c.total_sqrt_vote_weight += sqrt_max_vote_weight; }); } if( !_db.has_hardfork( STEEMIT_HARDFORK_0_17__774) ) @@ -1523,7 +1486,6 @@ void vote_evaluator::do_apply( const vote_operation& o ) _db.modify( comment, [&]( comment_object& c ) { c.total_vote_weight -= itr->weight; - c.total_sqrt_vote_weight -= itr->sqrt_weight; }); _db.modify( *itr, [&]( comment_vote_object& cv ) @@ -1532,7 +1494,6 @@ void vote_evaluator::do_apply( const vote_operation& o ) cv.vote_percent = o.weight; cv.last_update = _db.head_block_time(); cv.weight = 0; - cv.sqrt_weight = 0; cv.num_changes += 1; }); @@ -2160,9 +2121,12 @@ void claim_reward_balance_evaluator::do_apply( const claim_reward_balance_operat { const auto& acnt = _db.get_account( op.account ); - FC_ASSERT( op.reward_steem <= acnt.reward_steem_balance, "Cannot claim that much STEEM." ); - FC_ASSERT( op.reward_sbd <= acnt.reward_sbd_balance, "Cannot claim that much SBD." ); - FC_ASSERT( op.reward_vests <= acnt.reward_vesting_balance, "Cannot claim that much VESTS." ); + FC_ASSERT( op.reward_steem <= acnt.reward_steem_balance, "Cannot claim that much STEEM. Claim: ${c} Actual: ${a}", + ("c", op.reward_steem)("a", acnt.reward_steem_balance) ); + FC_ASSERT( op.reward_sbd <= acnt.reward_sbd_balance, "Cannot claim that much SBD. Claim: ${c} Actual: ${a}", + ("c", op.reward_sbd)("a", acnt.reward_sbd_balance) ); + FC_ASSERT( op.reward_vests <= acnt.reward_vesting_balance, "Cannot claim that much VESTS. Claim: ${c} Actual: ${a}", + ("c", op.reward_vests)("a", acnt.reward_vesting_balance) ); asset reward_vesting_steem_to_move = asset( 0, STEEM_SYMBOL ); if( op.reward_vests == acnt.reward_vesting_balance ) @@ -2260,26 +2224,14 @@ void delegate_vesting_shares_evaluator::do_apply( const delegate_vesting_shares_ { auto delta = delegation->vesting_shares - op.vesting_shares; - if( _db.has_hardfork( STEEMIT_HARDFORK_0_19__971 ) ) + if( op.vesting_shares.amount > 0 ) { - if( op.vesting_shares.amount > 0 ) - { - FC_ASSERT( delta >= min_update, "Steem Power decrease is not enough of a difference. min_update: ${min}", ("min", min_update) ); - FC_ASSERT( op.vesting_shares >= min_delegation, "Delegation must be removed or leave minimum delegation amount of ${v}", ("v", min_delegation) ); - } - else - { - FC_ASSERT( delegation->vesting_shares.amount > 0, "Delegation would set vesting_shares to zero, but it is already zero"); - } + FC_ASSERT( delta >= min_update, "Steem Power decrease is not enough of a difference. min_update: ${min}", ("min", min_update) ); + FC_ASSERT( op.vesting_shares >= min_delegation, "Delegation must be removed or leave minimum delegation amount of ${v}", ("v", min_delegation) ); } - #warning( "TODO: Check and remove conditional after HF19" ) else { - FC_ASSERT( delta >= min_update, "Steem Power decrease is not enough of a difference. min_update: ${min}", ("min", min_update) ); - if( op.vesting_shares.amount > 0 ) - { - FC_ASSERT( op.vesting_shares >= min_delegation, "Delegation must be removed or leave minimum delegation amount of ${v}", ("v", min_delegation) ); - } + FC_ASSERT( delegation->vesting_shares.amount > 0, "Delegation would set vesting_shares to zero, but it is already zero"); } _db.create< vesting_delegation_expiration_object >( [&]( vesting_delegation_expiration_object& obj ) diff --git a/libraries/manifest/CMakeLists.txt b/libraries/manifest/CMakeLists.txt index 164146536f..bd17f4dbee 100644 --- a/libraries/manifest/CMakeLists.txt +++ b/libraries/manifest/CMakeLists.txt @@ -1,55 +1,110 @@ file(GLOB HEADERS "include/steemit/manifest/*.hpp") -################# external plugins ################### +# helpful hints from +# https://blog.kangz.net/posts/2016/05/26/integrating-a-code-generator-with-cmake/ +# https://stackoverflow.com/questions/26074450/how-to-include-generated-files-for-compilation-with-cmake -# external_plugins target depends on all plugins -add_library( steemit_external_plugins - external_plugins.cpp - ) +############################################################################## +# First, we generate plugins.json by reading all the plugins.json files. +# We run the Python script once at configure time with execute_process() +# to figure out which files will be read (using the --print-dependencies option), +# then we run it again at build time with add_custom_command() to +# actually read the files. +# +# This way it will be properly regenerated. +# -# generate file based on comments -file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/inc/mf_external_plugins.inc" "// this file is autogenerated\n#pragma once\n#define STEEMIT_EXTERNAL_PLUGIN_LIST \\\n ") - -foreach(pin $ENV{STEEMIT_EXTERNAL_PLUGINS}) - file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/inc/mf_external_plugins.inc" "( ${pin} )") - target_link_libraries( steemit_external_plugins "${pin}" ) -endforeach() +execute_process( + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR} + python3 -m steem_manifest.plugins + ${CMAKE_CURRENT_SOURCE_DIR}/../plugins + -o ${CMAKE_CURRENT_BINARY_DIR}/template_context/plugins.json + --print-dependencies + OUTPUT_VARIABLE PLUGINS_DEPS + RESULT_VARIABLE RETURN_VALUE +) -file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/inc/mf_external_plugins.inc" "\n") +if( NOT RETURN_VALUE EQUAL 0 ) + message(FATAL_ERROR "Could not get plugin dependencies") +endif() +message( STATUS "PLUGINS_DEPS: ${PLUGINS_DEPS}" ) -################# internal plugins ################### +add_custom_command( + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR} + python3 -m steem_manifest.plugins + ${CMAKE_CURRENT_SOURCE_DIR}/../plugins + -o ${CMAKE_CURRENT_BINARY_DIR}/template_context/plugins.json + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/template_context/plugins.json + DEPENDS ${PLUGINS_DEPS} ${CMAKE_CURRENT_SOURCE_DIR}/steem_manifest/plugins.py +) -add_library( steemit_internal_plugins - internal_plugins.cpp - ) +############################################################################## +# We instantiate jinja2 templates with the values in plugins.json. +# Just like generating plugins.json, we use execute_process() to do dry runs +# at configure time to figure out the dependencies and outputs. +# Then we use add_custom_command() to execute at build time. -file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/inc/mf_internal_plugins.inc" "// this file is autogenerated\n#pragma once\n#define STEEMIT_INTERNAL_PLUGIN_LIST \\\n ") +execute_process( + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR} + python3 -m steem_manifest.build + -j ${CMAKE_CURRENT_BINARY_DIR}/template_context + -t ${CMAKE_CURRENT_SOURCE_DIR}/templates + -o ${CMAKE_CURRENT_BINARY_DIR}/gensrc + --print-dependencies + OUTPUT_VARIABLE MANIFEST_DEPS + RESULT_VARIABLE RETURN_VALUE +) -foreach(pin $ENV{STEEMIT_INTERNAL_PLUGINS}) - file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/inc/mf_internal_plugins.inc" "( ${pin} )") - target_link_libraries( steemit_internal_plugins "steemit_${pin}" ) -endforeach() +if( NOT RETURN_VALUE EQUAL 0 ) + message(FATAL_ERROR "Could not get manifest dependencies") +endif() +message( STATUS "MANIFEST_DEPS: ${MANIFEST_DEPS}" ) -file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/inc/mf_internal_plugins.inc" "\n") +execute_process( + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR} + python3 -m steem_manifest.build + -j ${CMAKE_CURRENT_BINARY_DIR}/template_context + -t ${CMAKE_CURRENT_SOURCE_DIR}/templates + -o ${CMAKE_CURRENT_BINARY_DIR}/gensrc + --print-outputs + OUTPUT_VARIABLE MANIFEST_OUTPUTS + RESULT_VARIABLE RETURN_VALUE +) -################# common ################### +if( NOT RETURN_VALUE EQUAL 0 ) + message(FATAL_ERROR "Could not get manifest outputs") +endif() +message( STATUS "MANIFEST_OUTPUTS: ${MANIFEST_OUTPUTS}" ) -add_library( steemit_mf_plugins - mf_plugins.cpp - ${HEADERS} - ${CMAKE_CURRENT_BINARY_DIR}/inc/mf_internal_plugins.inc - ${CMAKE_CURRENT_BINARY_DIR}/inc/mf_external_plugins.inc - ) -target_link_libraries( steemit_mf_plugins fc ) +add_custom_command( + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR} + python3 -m steem_manifest.build + -j ${CMAKE_CURRENT_BINARY_DIR}/template_context + -t ${CMAKE_CURRENT_SOURCE_DIR}/templates + -o ${CMAKE_CURRENT_BINARY_DIR}/gensrc + OUTPUT ${MANIFEST_OUTPUTS} + DEPENDS ${MANIFEST_DEPS} ${CMAKE_CURRENT_SOURCE_DIR}/steem_manifest/build.py +) -target_include_directories( steemit_mf_plugins - PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" - PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/inc" ) +############################################################################## +# Now we need to repeat the two above add_custom_command() as execute_process() +# because CMakeLists.txt is one of the generated files, and it must be +# created at configure time! +# -INSTALL( TARGETS - steemit_mf_plugins +execute_process( + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR} + python3 -m steem_manifest.plugins + ${CMAKE_CURRENT_SOURCE_DIR}/../plugins + -o ${CMAKE_CURRENT_BINARY_DIR}/template_context/plugins.json +) - RUNTIME DESTINATION bin - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib +execute_process( + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR} + python3 -m steem_manifest.build + -j ${CMAKE_CURRENT_BINARY_DIR}/template_context + -t ${CMAKE_CURRENT_SOURCE_DIR}/templates + -o ${CMAKE_CURRENT_BINARY_DIR}/gensrc ) + +add_subdirectory( ${CMAKE_CURRENT_BINARY_DIR}/gensrc/plugins ${CMAKE_CURRENT_BINARY_DIR}/genbuild ) diff --git a/libraries/manifest/external_plugins.cpp b/libraries/manifest/external_plugins.cpp deleted file mode 100644 index beaf2ef5bd..0000000000 --- a/libraries/manifest/external_plugins.cpp +++ /dev/null @@ -1 +0,0 @@ -// this empty source file is part of the cmake hackery to build a lib containing all plugins diff --git a/libraries/manifest/internal_plugins.cpp b/libraries/manifest/internal_plugins.cpp deleted file mode 100644 index beaf2ef5bd..0000000000 --- a/libraries/manifest/internal_plugins.cpp +++ /dev/null @@ -1 +0,0 @@ -// this empty source file is part of the cmake hackery to build a lib containing all plugins diff --git a/libraries/manifest/steem_manifest/__init__.py b/libraries/manifest/steem_manifest/__init__.py new file mode 100644 index 0000000000..5ac1a5cf51 --- /dev/null +++ b/libraries/manifest/steem_manifest/__init__.py @@ -0,0 +1,10 @@ + +# Explanation of why the imports in this file are commented out: +# +# If you import a module in __init__.py and call it with python -m, it will +# spew warnings in Python 3.6 and potentially break earlier versions. +# +# See https://bugs.python.org/issue27487 + +# from . import build +# from . import plugins diff --git a/libraries/manifest/steem_manifest/build.py b/libraries/manifest/steem_manifest/build.py new file mode 100644 index 0000000000..33ec9763d9 --- /dev/null +++ b/libraries/manifest/steem_manifest/build.py @@ -0,0 +1,105 @@ + +import jinja2 + +import argparse +import json +import os +import sys + +def needs_overwrite( target_path, text ): + if not os.path.exists( target_path ): + return True + with open(target_path, "r") as f: + current_text = f.read() + if current_text == text: + return False + return True + +def overwrite_if_different( target_path, text ): + if not needs_overwrite( target_path, text ): + return + with open(target_path, "w") as f: + f.write(text) + return + +def build(template_dir="templates", output_dir=None, ctx=None, do_write=True, deps=None, outputs=None): + os.makedirs(output_dir, exist_ok=True) + for root, dirs, files in os.walk(template_dir): + dirs_to_remove = [d for d in dirs if d.startswith(".")] + for d in dirs_to_remove: + dirs.remove(d) + for f in files: + fpath = os.path.join(root, f) + rpath = os.path.relpath(fpath, template_dir) + target_path = os.path.join(output_dir, rpath) + target_dir = os.path.dirname(target_path) + os.makedirs(target_dir, exist_ok=True) + with open(fpath, "r") as infile: + infile_text = infile.read() + if f.endswith(".j2"): + template = jinja2.Template( infile_text ) + outfile_text = template.render( **ctx ) + target_path = target_path[:-3] + else: + outfile_text = infile_text + if do_write: + overwrite_if_different( target_path, outfile_text ) + if deps is not None: + deps.append(fpath) + if outputs is not None: + outputs.append(target_path) + return + +def load_context(json_dir, ctx=None, do_read=True, deps=None): + if ctx is None: + ctx = {} + for root, dirs, files in os.walk(json_dir): + dirs_to_remove = [d for d in dirs if d.startswith(".")] + for d in dirs_to_remove: + dirs.remove(d) + dirs.sort() + for f in sorted(files): + if not f.endswith(".json"): + continue + fpath = os.path.join(root, f) + if do_read: + with open(fpath, "r") as infile: + obj = json.load(infile) + ctx.update(obj) + if deps is not None: + deps.append(fpath) + return ctx + +def main(argv): + + parser = argparse.ArgumentParser( description="Build the manifest library" ) + parser.add_argument( "--json-dir", "-j", type=str, required=True, help="Location of JSON template context files") + parser.add_argument( "--template-dir", "-t", type=str, required=True, help="Location of template files" ) + parser.add_argument( "--output-dir", "-o", type=str, required=True, help="Output location" ) + parser.add_argument( "--print-dependencies", action="store_true", help="Print dependencies and exit") + parser.add_argument( "--print-outputs", action="store_true", help="Print dependencies and exit") + args = parser.parse_args() + + deps = [] + outputs = [] + + do_write = not (args.print_dependencies or args.print_outputs) + + ctx = load_context(args.json_dir, do_read=do_write, deps=deps) + + build( + template_dir=args.template_dir, + output_dir=args.output_dir, + ctx=ctx, + do_write=do_write, + deps=deps, + outputs=outputs, + ) + + if args.print_dependencies: + print(";".join(deps)) + if args.print_outputs: + print(";".join(outputs)) + +if __name__ == "__main__": + main(sys.argv) diff --git a/libraries/manifest/steem_manifest/plugins.py b/libraries/manifest/steem_manifest/plugins.py new file mode 100755 index 0000000000..6eb084c46e --- /dev/null +++ b/libraries/manifest/steem_manifest/plugins.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import sys + +def find_plugin_filenames(basedir): + for root, dirs, files in os.walk(basedir): + dirs_to_remove = [d for d in dirs if d.startswith(".")] + for d in dirs_to_remove: + dirs.remove(d) + if "plugin.json" in files: + yield os.path.join(root, "plugin.json") + +def find_plugins(basedir): + for plugin_json in find_plugin_filenames(basedir): + with open(plugin_json, "r") as f: + yield json.load(f) + +def process_plugin(ctx, plugin): + ctx["plugins"].append(plugin) + for iext in plugin.get("index_extensions", []): + ctx["index_extensions"].append(iext) + return + +def main(argv): + + parser = argparse.ArgumentParser( description="Find plugins" ) + + parser.add_argument("plugin_dir", metavar="DIR", type=str, nargs="+", + help="Dir(s) to search for templates") + parser.add_argument("-o", "--output", metavar="FILE", type=str, default="-", + help="Output file") + parser.add_argument( "--print-dependencies", action="store_true", help="Print dependencies and exit") + + args = parser.parse_args() + + if args.print_dependencies: + deps = [] + for d in args.plugin_dir: + for plugin in find_plugin_filenames(d): + deps.append(plugin) + print(";".join(deps)) + return + + if args.output == "-": + outfile = sys.stdout + else: + os.makedirs(os.path.dirname(args.output), exist_ok=True) + outfile = open(args.output, "w") + + ctx = { + "plugins" : [], + "index_extensions" : [], + } + for d in args.plugin_dir: + for plugin in find_plugins(d): + process_plugin(ctx, plugin) + outfile.write(json.dumps(ctx)) + outfile.write("\n") + outfile.close() + +if __name__ == "__main__": + main(sys.argv) diff --git a/libraries/manifest/templates/plugins/CMakeLists.txt.j2 b/libraries/manifest/templates/plugins/CMakeLists.txt.j2 new file mode 100644 index 0000000000..7080dbb522 --- /dev/null +++ b/libraries/manifest/templates/plugins/CMakeLists.txt.j2 @@ -0,0 +1,52 @@ + +# The steemit_plugins target links with all plugins +add_library( steemit_plugins + plugins.cpp ) + +target_link_libraries( steemit_plugins +{% for plugin in plugins %} + {{ plugin["plugin_project"] }} +{% endfor %} +) + +# The `steemit_mf_plugins` target has prototypes for querying plugins +# as abstract, but does not itself define the plugins. +# +# The reason for this design is that if we try to use `plugins` +# we will have this circular dependency: +# +# ``` +# xxx_plugin ===> steemit::app ===> plugins ===> xxx_plugin +# ``` +# +# To break the dependency, we create `mf_plugins` which only +# has a "weak" reference to the plugin: +# +# ``` +# xxx_plugin ===> steemit::app ===> mf_plugins ---> xxx_plugin +# ``` +# +# This weak reference is implemented by a declaring a +# `create_xxx_plugin()` global function which returns a +# `shared_ptr< abstract_plugin >`. The weak reference isn't +# resolved until link time. So as long as we link with both +# `steemit_mf_plugins` and `steemit_plugins` we are fine. +# + +add_library( steemit_mf_plugins + mf_plugins.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/steemit/manifest/plugins.hpp +) + +target_include_directories( steemit_mf_plugins + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" + ) +target_link_libraries( steemit_mf_plugins fc ) + +INSTALL( TARGETS + steemit_mf_plugins + + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) diff --git a/libraries/manifest/templates/plugins/include/steemit/manifest/plugins.hpp b/libraries/manifest/templates/plugins/include/steemit/manifest/plugins.hpp new file mode 100644 index 0000000000..bcbab144a4 --- /dev/null +++ b/libraries/manifest/templates/plugins/include/steemit/manifest/plugins.hpp @@ -0,0 +1,21 @@ + +#pragma once + +#include +#include +#include + +namespace steemit { namespace app { + +class abstract_plugin; +class application; + +} } + +namespace steemit { namespace plugin { + +void initialize_plugin_factories(); +std::shared_ptr< steemit::app::abstract_plugin > create_plugin( const std::string& name, steemit::app::application* app ); +std::vector< std::string > get_available_plugins(); + +} } diff --git a/libraries/manifest/mf_plugins.cpp b/libraries/manifest/templates/plugins/mf_plugins.cpp.j2 similarity index 50% rename from libraries/manifest/mf_plugins.cpp rename to libraries/manifest/templates/plugins/mf_plugins.cpp.j2 index 74cce72059..ff767ebdc3 100644 --- a/libraries/manifest/mf_plugins.cpp +++ b/libraries/manifest/templates/plugins/mf_plugins.cpp.j2 @@ -2,26 +2,23 @@ #include #include -#include -#include - -#define STEEMIT_DECLARE_PLUGIN_CREATOR( r, data, x ) \ - std::shared_ptr< steemit::app::abstract_plugin > BOOST_PP_CAT( create_, BOOST_PP_CAT( x, _plugin ) )( steemit::app::application* app ); namespace steemit { namespace plugin { -BOOST_PP_SEQ_FOR_EACH( STEEMIT_DECLARE_PLUGIN_CREATOR, _, STEEMIT_INTERNAL_PLUGIN_LIST ) -BOOST_PP_SEQ_FOR_EACH( STEEMIT_DECLARE_PLUGIN_CREATOR, _, STEEMIT_EXTERNAL_PLUGIN_LIST ) +{% for plugin in plugins %} +std::shared_ptr< steemit::app::abstract_plugin > create_{{ plugin["plugin_name"] }}_plugin( steemit::app::application* app ); +{% endfor %} boost::container::flat_map< std::string, std::function< std::shared_ptr< steemit::app::abstract_plugin >( steemit::app::application* app ) > > plugin_factories_by_name; -#define STEEMIT_REGISTER_PLUGIN_FACTORY( r, data, x ) \ - plugin_factories_by_name[ #x ] = []( steemit::app::application* app ) -> std::shared_ptr< steemit::app::abstract_plugin >{ return BOOST_PP_CAT( create_, BOOST_PP_CAT( x, _plugin( app ) ) ); }; - void initialize_plugin_factories() { - BOOST_PP_SEQ_FOR_EACH( STEEMIT_REGISTER_PLUGIN_FACTORY, _, STEEMIT_INTERNAL_PLUGIN_LIST ) - BOOST_PP_SEQ_FOR_EACH( STEEMIT_REGISTER_PLUGIN_FACTORY, _, STEEMIT_EXTERNAL_PLUGIN_LIST ) + {% for plugin in plugins %} + plugin_factories_by_name[ "{{ plugin["plugin_name"] }}" ] = []( steemit::app::application* app ) -> std::shared_ptr< steemit::app::abstract_plugin > + { + return create_{{ plugin["plugin_name"] }}_plugin( app ); + }; + {% endfor %} } std::shared_ptr< steemit::app::abstract_plugin > create_plugin( const std::string& name, steemit::app::application* app ) diff --git a/libraries/manifest/templates/plugins/plugins.cpp b/libraries/manifest/templates/plugins/plugins.cpp new file mode 100644 index 0000000000..34b39c985f --- /dev/null +++ b/libraries/manifest/templates/plugins/plugins.cpp @@ -0,0 +1 @@ +// empty cpp file for plugin project diff --git a/libraries/net/include/graphene/net/config.hpp b/libraries/net/include/graphene/net/config.hpp index d14bd825d1..8847cd87c7 100644 --- a/libraries/net/include/graphene/net/config.hpp +++ b/libraries/net/include/graphene/net/config.hpp @@ -62,11 +62,11 @@ * our peers and save a copy in a cache were we will find it if * a peer requests it. We expire out old items out of the cache * after this number of blocks go by. - * + * * Recently lowered from 30 to match the default expiration time * the web wallet imposes on transactions. */ -#define GRAPHENE_NET_MESSAGE_CACHE_DURATION_IN_BLOCKS 5 +#define GRAPHENE_NET_MESSAGE_CACHE_DURATION_IN_BLOCKS 20 /** * We prevent a peer from offering us a list of blocks which, if we fetched them @@ -87,12 +87,12 @@ * is being flooded -- typically transactions will be fetched as soon * as we find out about them, so only one item will be requested * at a time. - * + * * No tests have been done to find the optimal value for this * parameter, so consider increasing or decreasing it if performance * during flooding is lacking. */ -#define GRAPHENE_NET_MAX_ITEMS_PER_PEER_DURING_NORMAL_OPERATION 1 +#define GRAPHENE_NET_MAX_ITEMS_PER_PEER_DURING_NORMAL_OPERATION 1 /** * Instead of fetching all item IDs from a peer, then fetching all blocks diff --git a/libraries/plugins/CMakeLists.txt b/libraries/plugins/CMakeLists.txt index ab0cf557cc..6b4e26333b 100644 --- a/libraries/plugins/CMakeLists.txt +++ b/libraries/plugins/CMakeLists.txt @@ -1,12 +1,10 @@ # for each subdirectory containing a CMakeLists.txt, add that subdirectory -set( ENV{STEEMIT_INTERNAL_PLUGINS} "" ) file( GLOB children RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} * ) foreach( child ${children} ) if( IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/${child}" ) if( EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${child}/CMakeLists.txt" ) add_subdirectory( "${child}" ) - set( ENV{STEEMIT_INTERNAL_PLUGINS} "$ENV{STEEMIT_INTERNAL_PLUGINS};${child}" ) endif() endif() endforeach() diff --git a/libraries/plugins/account_by_key/account_by_key_plugin.cpp b/libraries/plugins/account_by_key/account_by_key_plugin.cpp index dc3a8858c0..a1bfcb4c70 100644 --- a/libraries/plugins/account_by_key/account_by_key_plugin.cpp +++ b/libraries/plugins/account_by_key/account_by_key_plugin.cpp @@ -50,6 +50,11 @@ struct pre_operation_visitor _plugin.my->clear_cache(); } + void operator()( const account_create_with_delegation_operation& op )const + { + _plugin.my->clear_cache(); + } + void operator()( const account_update_operation& op )const { _plugin.my->clear_cache(); @@ -103,6 +108,12 @@ struct post_operation_visitor if( acct_itr ) _plugin.my->update_key_lookup( *acct_itr ); } + void operator()( const account_create_with_delegation_operation& op )const + { + auto acct_itr = _plugin.database().find< account_authority_object, by_account >( op.new_account_name ); + if( acct_itr ) _plugin.my->update_key_lookup( *acct_itr ); + } + void operator()( const account_update_operation& op )const { auto acct_itr = _plugin.database().find< account_authority_object, by_account >( op.account ); diff --git a/libraries/plugins/account_by_key/plugin.json b/libraries/plugins/account_by_key/plugin.json new file mode 100644 index 0000000000..f9b5c7b605 --- /dev/null +++ b/libraries/plugins/account_by_key/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "account_by_key", + "plugin_project": "steemit_account_by_key" +} diff --git a/libraries/plugins/account_history/account_history_plugin.cpp b/libraries/plugins/account_history/account_history_plugin.cpp index 337580c791..126bd54e08 100644 --- a/libraries/plugins/account_history/account_history_plugin.cpp +++ b/libraries/plugins/account_history/account_history_plugin.cpp @@ -11,6 +11,10 @@ #include #include +#include + +#define STEEM_NAMESPACE_PREFIX "steemit::protocol::" + namespace steemit { namespace account_history { namespace detail @@ -35,7 +39,9 @@ class account_history_plugin_impl account_history_plugin& _self; flat_map< account_name_type, account_name_type > _tracked_accounts; - bool _filter_content = false; + bool _filter_content = false; + bool _blacklist = false; + flat_set< string > _op_list; }; account_history_plugin_impl::~account_history_plugin_impl() @@ -45,7 +51,9 @@ account_history_plugin_impl::~account_history_plugin_impl() struct operation_visitor { - operation_visitor( database& db, const operation_notification& note, const operation_object*& n, account_name_type i ):_db(db),_note(note),new_obj(n),item(i){}; + operation_visitor( database& db, const operation_notification& note, const operation_object*& n, account_name_type i ) + :_db(db), _note(note), new_obj(n), item(i) {} + typedef void result_type; database& _db; @@ -53,11 +61,6 @@ struct operation_visitor const operation_object*& new_obj; account_name_type item; - /// ignore these ops - /* - */ - - template void operator()( Op&& )const { @@ -94,79 +97,28 @@ struct operation_visitor } }; +struct operation_visitor_filter : operation_visitor +{ + operation_visitor_filter( database& db, const operation_notification& note, const operation_object*& n, account_name_type i, const flat_set< string >& filter, bool blacklist ): + operation_visitor( db, note, n, i ), _filter( filter ), _blacklist( blacklist ) {} + const flat_set< string >& _filter; + bool _blacklist; -struct operation_visitor_filter : operation_visitor { - operation_visitor_filter( database& db, const operation_notification& note, const operation_object*& n, account_name_type i ):operation_visitor(db,note,n,i){} - - void operator()( const comment_operation& )const {} - void operator()( const vote_operation& )const {} - void operator()( const delete_comment_operation& )const{} - void operator()( const custom_json_operation& )const {} - void operator()( const custom_operation& )const {} - void operator()( const curation_reward_operation& )const {} - void operator()( const fill_order_operation& )const {} - void operator()( const limit_order_create_operation& )const {} - void operator()( const limit_order_cancel_operation& )const {} - void operator()( const pow_operation& )const {} - - void operator()( const transfer_operation& op )const - { - operation_visitor::operator()( op ); - } - - void operator()( const transfer_to_vesting_operation& op )const - { - operation_visitor::operator()( op ); - } - - void operator()( const account_create_operation& op )const - { - operation_visitor::operator()( op ); - } - - void operator()( const account_update_operation& op )const - { - operation_visitor::operator()( op ); - } - - void operator()( const transfer_to_savings_operation& op )const - { - operation_visitor::operator()( op ); - } - - void operator()( const transfer_from_savings_operation& op )const - { - operation_visitor::operator()( op ); - } - - void operator()( const cancel_transfer_from_savings_operation& op )const - { - operation_visitor::operator()( op ); - } - - void operator()( const escrow_transfer_operation& op )const - { - operation_visitor::operator()( op ); - } - - void operator()( const escrow_dispute_operation& op )const - { - operation_visitor::operator()( op ); - } - - void operator()( const escrow_release_operation& op )const - { - operation_visitor::operator()( op ); - } - - void operator()( const escrow_approve_operation& op )const + template< typename T > + void operator()( const T& op )const { - operation_visitor::operator()( op ); + if( _filter.find( fc::get_typename< T >::name() ) != _filter.end() ) + { + if( !_blacklist ) + operation_visitor::operator()( op ); + } + else + { + if( _blacklist ) + operation_visitor::operator()( op ); + } } - - template - void operator()( Op&& op )const{ } }; void account_history_plugin_impl::on_operation( const operation_notification& note ) @@ -205,11 +157,16 @@ void account_history_plugin_impl::on_operation( const operation_notification& no --itr; } - if( !_tracked_accounts.size() || (itr != _tracked_accounts.end() && itr->first <= item && item <= itr->second ) ) { - if( _filter_content ) - note.op.visit( operation_visitor_filter(db, note, new_obj, item) ); + if( !_tracked_accounts.size() || (itr != _tracked_accounts.end() && itr->first <= item && item <= itr->second ) ) + { + if(_filter_content) + { + note.op.visit( operation_visitor_filter( db, note, new_obj, item, _op_list, _blacklist ) ); + } else - note.op.visit( operation_visitor(db, note, new_obj, item) ); + { + note.op.visit( operation_visitor( db, note, new_obj, item ) ); + } } } } @@ -237,8 +194,9 @@ void account_history_plugin::plugin_set_program_options( ) { cli.add_options() - ("track-account-range", boost::program_options::value>()->composing()->multitoken(), "Defines a range of accounts to track as a json pair [\"from\",\"to\"] [from,to] Can be specified multiple times") - ("filter-posting-ops", "Ignore posting operations, only track transfers and account updates") + ("track-account-range", boost::program_options::value< vector< string > >()->composing()->multitoken(), "Defines a range of accounts to track as a json pair [\"from\",\"to\"] [from,to] Can be specified multiple times") + ("history-whitelist-ops", boost::program_options::value< vector< string > >()->composing(), "Defines a list of operations which will be explicitly logged.") + ("history-blacklist-ops", boost::program_options::value< vector< string > >()->composing(), "Defines a list of operations which will be explicitly ignored.") ; cfg.add(cli); } @@ -250,8 +208,43 @@ void account_history_plugin::plugin_initialize(const boost::program_options::var typedef pair pairstring; LOAD_VALUE_SET(options, "track-account-range", my->_tracked_accounts, pairstring); - if( options.count( "filter-posting-ops" ) ) { + + if( options.count( "history-whitelist-ops" ) ) + { my->_filter_content = true; + my->_blacklist = false; + + for( auto& arg : options.at( "history-whitelist-ops" ).as< vector< string > >() ) + { + vector< string > ops; + boost::split( ops, arg, boost::is_any_of( " \t," ) ); + + for( const string& op : ops ) + { + if( op.size() ) + my->_op_list.insert( STEEM_NAMESPACE_PREFIX + op ); + } + } + + ilog( "Account History: whitelisting ops ${o}", ("o", my->_op_list) ); + } + else if( options.count( "history-blacklist-ops" ) ) + { + my->_filter_content = true; + my->_blacklist = true; + for( auto& arg : options.at( "history-blacklist-ops" ).as< vector< string > >() ) + { + vector< string > ops; + boost::split( ops, arg, boost::is_any_of( " \t," ) ); + + for( const string& op : ops ) + { + if( op.size() ) + my->_op_list.insert( STEEM_NAMESPACE_PREFIX + op ); + } + } + + ilog( "Account History: blacklisting ops ${o}", ("o", my->_op_list) ); } } diff --git a/libraries/plugins/account_history/plugin.json b/libraries/plugins/account_history/plugin.json new file mode 100644 index 0000000000..c6c1f83a8e --- /dev/null +++ b/libraries/plugins/account_history/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "account_history", + "plugin_project": "steemit_account_history" +} diff --git a/libraries/plugins/account_statistics/plugin.json b/libraries/plugins/account_statistics/plugin.json new file mode 100644 index 0000000000..9e6e60b27d --- /dev/null +++ b/libraries/plugins/account_statistics/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "account_statistics", + "plugin_project": "steemit_account_statistics" +} diff --git a/libraries/plugins/auth_util/plugin.json b/libraries/plugins/auth_util/plugin.json new file mode 100644 index 0000000000..5e4321d585 --- /dev/null +++ b/libraries/plugins/auth_util/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "auth_util", + "plugin_project": "steemit_auth_util" +} diff --git a/libraries/plugins/block_info/block_info_plugin.cpp b/libraries/plugins/block_info/block_info_plugin.cpp index 51bb576e08..69bc0b7f70 100644 --- a/libraries/plugins/block_info/block_info_plugin.cpp +++ b/libraries/plugins/block_info/block_info_plugin.cpp @@ -47,7 +47,6 @@ void block_info_plugin::on_applied_block( const chain::signed_block& b ) info.block_id = b.id(); info.block_size = fc::raw::pack_size( b ); - info.average_block_size = dgpo.average_block_size; info.aslot = dgpo.current_aslot; info.last_irreversible_block_num = dgpo.last_irreversible_block_num; info.num_pow_witnesses = dgpo.num_pow_witnesses; diff --git a/libraries/plugins/block_info/include/steemit/plugins/block_info/block_info.hpp b/libraries/plugins/block_info/include/steemit/plugins/block_info/block_info.hpp index 8c834f4ea7..6be3ed03f9 100644 --- a/libraries/plugins/block_info/include/steemit/plugins/block_info/block_info.hpp +++ b/libraries/plugins/block_info/include/steemit/plugins/block_info/block_info.hpp @@ -9,7 +9,6 @@ struct block_info { chain::block_id_type block_id; uint32_t block_size = 0; - uint32_t average_block_size = 0; uint64_t aslot = 0; uint32_t last_irreversible_block_num = 0; uint32_t num_pow_witnesses = 0; @@ -26,7 +25,6 @@ struct block_with_info FC_REFLECT( steemit::plugin::block_info::block_info, (block_id) (block_size) - (average_block_size) (aslot) (last_irreversible_block_num) (num_pow_witnesses) diff --git a/libraries/plugins/block_info/plugin.json b/libraries/plugins/block_info/plugin.json new file mode 100644 index 0000000000..9f2ccf29c4 --- /dev/null +++ b/libraries/plugins/block_info/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "block_info", + "plugin_project": "steemit_block_info" +} diff --git a/libraries/plugins/blockchain_statistics/plugin.json b/libraries/plugins/blockchain_statistics/plugin.json new file mode 100644 index 0000000000..a01553e0a1 --- /dev/null +++ b/libraries/plugins/blockchain_statistics/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "blockchain_statistics", + "plugin_project": "steemit_blockchain_statistics" +} diff --git a/libraries/plugins/debug_node/plugin.json b/libraries/plugins/debug_node/plugin.json new file mode 100644 index 0000000000..13c047667e --- /dev/null +++ b/libraries/plugins/debug_node/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "debug_node", + "plugin_project": "steemit_debug_node" +} diff --git a/libraries/plugins/delayed_node/plugin.json b/libraries/plugins/delayed_node/plugin.json new file mode 100644 index 0000000000..d2db4be169 --- /dev/null +++ b/libraries/plugins/delayed_node/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "delayed_node", + "plugin_project": "steemit_delayed_node" +} diff --git a/libraries/plugins/follow/plugin.json b/libraries/plugins/follow/plugin.json new file mode 100644 index 0000000000..887129d31f --- /dev/null +++ b/libraries/plugins/follow/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "follow", + "plugin_project": "steemit_follow" +} diff --git a/libraries/plugins/market_history/plugin.json b/libraries/plugins/market_history/plugin.json new file mode 100644 index 0000000000..046a805856 --- /dev/null +++ b/libraries/plugins/market_history/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "market_history", + "plugin_project": "steemit_market_history" +} diff --git a/libraries/plugins/private_message/plugin.json b/libraries/plugins/private_message/plugin.json new file mode 100644 index 0000000000..9cf317f3a0 --- /dev/null +++ b/libraries/plugins/private_message/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "private_message", + "plugin_project": "steemit_private_message" +} diff --git a/libraries/plugins/raw_block/plugin.json b/libraries/plugins/raw_block/plugin.json new file mode 100644 index 0000000000..2c8d419785 --- /dev/null +++ b/libraries/plugins/raw_block/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "raw_block", + "plugin_project": "steemit_raw_block" +} diff --git a/libraries/plugins/tags/plugin.json b/libraries/plugins/tags/plugin.json new file mode 100644 index 0000000000..e2334e301b --- /dev/null +++ b/libraries/plugins/tags/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "tags", + "plugin_project": "steemit_tags" +} diff --git a/libraries/plugins/witness/include/steemit/witness/witness_objects.hpp b/libraries/plugins/witness/include/steemit/witness/witness_objects.hpp index c56735002b..2193a7ee3b 100644 --- a/libraries/plugins/witness/include/steemit/witness/witness_objects.hpp +++ b/libraries/plugins/witness/include/steemit/witness/witness_objects.hpp @@ -16,7 +16,8 @@ using namespace steemit::chain; enum witness_plugin_object_type { account_bandwidth_object_type = ( WITNESS_SPACE_ID << 8 ), - content_edit_lock_object_type = ( WITNESS_SPACE_ID << 8 ) + 1 + content_edit_lock_object_type = ( WITNESS_SPACE_ID << 8 ) + 1, + reserve_ratio_object_type = ( WITNESS_SPACE_ID << 8 ) + 2 }; enum bandwidth_type @@ -48,6 +49,7 @@ class account_bandwidth_object : public object< account_bandwidth_object_type, a typedef oid< account_bandwidth_object > account_bandwidth_id_type; + class content_edit_lock_object : public object< content_edit_lock_object_type, content_edit_lock_object > { public: @@ -66,6 +68,53 @@ class content_edit_lock_object : public object< content_edit_lock_object_type, c typedef oid< content_edit_lock_object > content_edit_lock_id_type; + +class reserve_ratio_object : public object< reserve_ratio_object_type, reserve_ratio_object > +{ + public: + template< typename Constructor, typename Allocator > + reserve_ratio_object( Constructor&& c, allocator< Allocator > a ) + { + c( *this ); + } + + reserve_ratio_object() {} + + id_type id; + + /** + * Average block size is updated every block to be: + * + * average_block_size = (99 * average_block_size + new_block_size) / 100 + * + * This property is used to update the current_reserve_ratio to maintain approximately + * 50% or less utilization of network capacity. + */ + int32_t average_block_size = 0; + + /** + * Any time average_block_size <= 50% maximum_block_size this value grows by 1 until it + * reaches STEEMIT_MAX_RESERVE_RATIO. Any time average_block_size is greater than + * 50% it falls by 1%. Upward adjustments happen once per round, downward adjustments + * happen every block. + */ + int64_t current_reserve_ratio = 1; + + /** + * The maximum bandwidth the blockchain can support is: + * + * max_bandwidth = maximum_block_size * STEEMIT_BANDWIDTH_AVERAGE_WINDOW_SECONDS / STEEMIT_BLOCK_INTERVAL + * + * The maximum virtual bandwidth is: + * + * max_bandwidth * current_reserve_ratio + */ + uint128_t max_virtual_bandwidth = 0; +}; + +typedef oid< reserve_ratio_object > reserve_ratio_id_type; + + struct by_account_bandwidth_type; typedef multi_index_container < @@ -96,6 +145,15 @@ typedef multi_index_container < allocator< content_edit_lock_object > > content_edit_lock_index; +typedef multi_index_container < + reserve_ratio_object, + indexed_by < + ordered_unique< tag< by_id >, + member< reserve_ratio_object, reserve_ratio_id_type, &reserve_ratio_object::id > > + >, + allocator< reserve_ratio_object > +> reserve_ratio_index; + } } // steemit::witness FC_REFLECT_ENUM( steemit::witness::bandwidth_type, (post)(forum)(market) ) @@ -107,3 +165,7 @@ CHAINBASE_SET_INDEX_TYPE( steemit::witness::account_bandwidth_object, steemit::w FC_REFLECT( steemit::witness::content_edit_lock_object, (id)(account)(lock_time) ) CHAINBASE_SET_INDEX_TYPE( steemit::witness::content_edit_lock_object, steemit::witness::content_edit_lock_index ) + +FC_REFLECT( steemit::witness::reserve_ratio_object, + (id)(average_block_size)(current_reserve_ratio)(max_virtual_bandwidth) ) +CHAINBASE_SET_INDEX_TYPE( steemit::witness::reserve_ratio_object, steemit::witness::reserve_ratio_index ) diff --git a/libraries/plugins/witness/include/steemit/witness/witness_plugin.hpp b/libraries/plugins/witness/include/steemit/witness/witness_plugin.hpp index 522ddfebe6..78f7f40036 100644 --- a/libraries/plugins/witness/include/steemit/witness/witness_plugin.hpp +++ b/libraries/plugins/witness/include/steemit/witness/witness_plugin.hpp @@ -28,6 +28,9 @@ #include +#define RESERVE_RATIO_PRECISION ((int64_t)10000) +#define RESERVE_RATIO_MIN_INCREMENT ((int64_t)5000) + namespace steemit { namespace witness { using std::string; diff --git a/libraries/plugins/witness/plugin.json b/libraries/plugins/witness/plugin.json new file mode 100644 index 0000000000..6e95fa3b55 --- /dev/null +++ b/libraries/plugins/witness/plugin.json @@ -0,0 +1,4 @@ +{ + "plugin_name": "witness", + "plugin_project": "steemit_witness" +} diff --git a/libraries/plugins/witness/witness_plugin.cpp b/libraries/plugins/witness/witness_plugin.cpp index c877699a35..34974f585c 100644 --- a/libraries/plugins/witness/witness_plugin.cpp +++ b/libraries/plugins/witness/witness_plugin.cpp @@ -42,6 +42,10 @@ #include #include + +#define DISTANCE_CALC_PRECISION (10000) + + namespace steemit { namespace witness { namespace bpo = boost::program_options; @@ -70,6 +74,7 @@ namespace detail { using namespace steemit::chain; + class witness_plugin_impl { public: @@ -80,6 +85,7 @@ namespace detail void pre_transaction( const signed_transaction& trx ); void pre_operation( const operation_notification& note ); + void on_block( const signed_block& b ); void update_account_bandwidth( const account_object& a, uint32_t trx_size, const bandwidth_type type ); @@ -113,6 +119,59 @@ namespace detail } }; + void check_memo( const string& memo, const account_object& account, const account_authority_object& auth ) + { + vector< public_key_type > keys; + + try + { + // Check if memo is a private key + keys.push_back( fc::ecc::extended_private_key::from_base58( memo ).get_public_key() ); + } + catch( fc::parse_error_exception& ) {} + catch( fc::assert_exception& ) {} + + // Get possible keys if memo was an account password + string owner_seed = account.name + "owner" + memo; + auto owner_secret = fc::sha256::hash( owner_seed.c_str(), owner_seed.size() ); + keys.push_back( fc::ecc::private_key::regenerate( owner_secret ).get_public_key() ); + + string active_seed = account.name + "active" + memo; + auto active_secret = fc::sha256::hash( active_seed.c_str(), active_seed.size() ); + keys.push_back( fc::ecc::private_key::regenerate( active_secret ).get_public_key() ); + + string posting_seed = account.name + "posting" + memo; + auto posting_secret = fc::sha256::hash( posting_seed.c_str(), posting_seed.size() ); + keys.push_back( fc::ecc::private_key::regenerate( posting_secret ).get_public_key() ); + + // Check keys against public keys in authorites + for( auto& key_weight_pair : auth.owner.key_auths ) + { + for( auto& key : keys ) + STEEMIT_ASSERT( key_weight_pair.first != key, chain::plugin_exception, + "Detected private owner key in memo field. You should change your owner keys." ); + } + + for( auto& key_weight_pair : auth.active.key_auths ) + { + for( auto& key : keys ) + STEEMIT_ASSERT( key_weight_pair.first != key, chain::plugin_exception, + "Detected private active key in memo field. You should change your active keys." ); + } + + for( auto& key_weight_pair : auth.posting.key_auths ) + { + for( auto& key : keys ) + STEEMIT_ASSERT( key_weight_pair.first != key, chain::plugin_exception, + "Detected private posting key in memo field. You should change your posting keys." ); + } + + const auto& memo_key = account.memo_key; + for( auto& key : keys ) + STEEMIT_ASSERT( memo_key != key, chain::plugin_exception, + "Detected private memo key in memo field. You should change your memo key." ); + } + struct operation_visitor { operation_visitor( const chain::database& db ) : _db( db ) {} @@ -159,6 +218,30 @@ namespace detail "The comment is archived" ); } } + + void operator()( const transfer_operation& o )const + { + if( o.memo.length() > 0 ) + check_memo( o.memo, + _db.get< account_object, chain::by_name >( o.from ), + _db.get< account_authority_object, chain::by_account >( o.from ) ); + } + + void operator()( const transfer_to_savings_operation& o )const + { + if( o.memo.length() > 0 ) + check_memo( o.memo, + _db.get< account_object, chain::by_name >( o.from ), + _db.get< account_authority_object, chain::by_account >( o.from ) ); + } + + void operator()( const transfer_from_savings_operation& o )const + { + if( o.memo.length() > 0 ) + check_memo( o.memo, + _db.get< account_object, chain::by_name >( o.from ), + _db.get< account_authority_object, chain::by_account >( o.from ) ); + } }; void witness_plugin_impl::pre_transaction( const signed_transaction& trx ) @@ -195,6 +278,82 @@ namespace detail } } + void witness_plugin_impl::on_block( const signed_block& b ) + { + auto& db = _self.database(); + int64_t max_block_size = db.get_dynamic_global_properties().maximum_block_size; + + auto reserve_ratio_ptr = db.find( reserve_ratio_id_type() ); + + if( BOOST_UNLIKELY( reserve_ratio_ptr == nullptr ) ) + { + db.create< reserve_ratio_object >( [&]( reserve_ratio_object& r ) + { + r.average_block_size = 0; + r.current_reserve_ratio = STEEMIT_MAX_RESERVE_RATIO * RESERVE_RATIO_PRECISION; + r.max_virtual_bandwidth = ( uint128_t( STEEMIT_MAX_BLOCK_SIZE * STEEMIT_MAX_RESERVE_RATIO ) + * STEEMIT_BANDWIDTH_PRECISION * STEEMIT_BANDWIDTH_AVERAGE_WINDOW_SECONDS ) + / STEEMIT_BLOCK_INTERVAL; + }); + } + else + { + db.modify( *reserve_ratio_ptr, [&]( reserve_ratio_object& r ) + { + r.average_block_size = ( 99 * r.average_block_size + fc::raw::pack_size( b ) ) / 100; + + /** + * About once per minute the average network use is consulted and used to + * adjust the reserve ratio. Anything above 25% usage reduces the reserve + * ratio proportional to the distance from 25%. If usage is at 50% then + * the reserve ratio will half. Likewise, if it is at 12% it will increase by 50%. + * + * If the reserve ratio is consistently low, then it is probably time to increase + * the capcacity of the network. + * + * This algorithm is designed to react quickly to observations significantly + * different from past observed behavior and make small adjustments when + * behavior is within expected norms. + */ + if( db.head_block_num() % 20 == 0 ) + { + int64_t distance = ( ( r.average_block_size - ( max_block_size / 4 ) ) * DISTANCE_CALC_PRECISION ) + / ( max_block_size / 4 ); + auto old_reserve_ratio = r.current_reserve_ratio; + + if( distance > 0 ) + { + r.current_reserve_ratio -= ( r.current_reserve_ratio * distance ) / ( distance + DISTANCE_CALC_PRECISION ); + + // We do not want the reserve ratio to drop below 1 + if( r.current_reserve_ratio < RESERVE_RATIO_PRECISION ) + r.current_reserve_ratio = RESERVE_RATIO_PRECISION; + } + else + { + // By default, we should always slowly increase the reserve ratio. + r.current_reserve_ratio += std::max( RESERVE_RATIO_MIN_INCREMENT, ( r.current_reserve_ratio * distance ) / ( distance - DISTANCE_CALC_PRECISION ) ); + + if( r.current_reserve_ratio > STEEMIT_MAX_RESERVE_RATIO * RESERVE_RATIO_PRECISION ) + r.current_reserve_ratio = STEEMIT_MAX_RESERVE_RATIO * RESERVE_RATIO_PRECISION; + } + + if( old_reserve_ratio != r.current_reserve_ratio ) + { + ilog( "Reserve ratio updated from ${old} to ${new}. Block: ${blocknum}", + ("old", old_reserve_ratio) + ("new", r.current_reserve_ratio) + ("blocknum", db.head_block_num()) ); + } + + r.max_virtual_bandwidth = ( uint128_t( max_block_size ) * uint128_t( r.current_reserve_ratio ) + * uint128_t( STEEMIT_BANDWIDTH_PRECISION * STEEMIT_BANDWIDTH_AVERAGE_WINDOW_SECONDS ) ) + / ( STEEMIT_BLOCK_INTERVAL * RESERVE_RATIO_PRECISION ); + } + }); + } + } + void witness_plugin_impl::update_account_bandwidth( const account_object& a, uint32_t trx_size, const bandwidth_type type ) { database& _db = _self.database(); @@ -236,13 +395,13 @@ namespace detail fc::uint128 account_vshares( a.effective_vesting_shares().amount.value ); fc::uint128 total_vshares( props.total_vesting_shares.amount.value ); fc::uint128 account_average_bandwidth( band->average_bandwidth.value ); - fc::uint128 max_virtual_bandwidth( props.max_virtual_bandwidth ); + fc::uint128 max_virtual_bandwidth( _db.get( reserve_ratio_id_type() ).max_virtual_bandwidth ); has_bandwidth = ( account_vshares * max_virtual_bandwidth ) > ( account_average_bandwidth * total_vshares ); if( _db.is_producing() ) STEEMIT_ASSERT( has_bandwidth, chain::plugin_exception, - "Account: ${account} bandwidth limit exeeded. Please wait to transact or power up STEEM.", + "Account: ${account} bandwidth limit exceeded. Please wait to transact or power up STEEM.", ("account", a.name) ("account_vshares", account_vshares) ("account_average_bandwidth", account_average_bandwidth) @@ -316,9 +475,11 @@ void witness_plugin::plugin_initialize(const boost::program_options::variables_m db.on_pre_apply_transaction.connect( [&]( const signed_transaction& tx ){ _my->pre_transaction( tx ); } ); db.pre_apply_operation.connect( [&]( const operation_notification& note ){ _my->pre_operation( note ); } ); + db.applied_block.connect( [&]( const signed_block& b ){ _my->on_block( b ); } ); add_plugin_index< account_bandwidth_index >( db ); add_plugin_index< content_edit_lock_index >( db ); + add_plugin_index< reserve_ratio_index >( db ); } FC_LOG_AND_RETHROW() } void witness_plugin::plugin_startup() diff --git a/libraries/protocol/get_config.cpp b/libraries/protocol/get_config.cpp index faea7688bd..b146811a46 100644 --- a/libraries/protocol/get_config.cpp +++ b/libraries/protocol/get_config.cpp @@ -45,8 +45,6 @@ fc::variant_object get_config() result["STEEMIT_CASHOUT_WINDOW_SECONDS_PRE_HF17"] = STEEMIT_CASHOUT_WINDOW_SECONDS_PRE_HF17; result["STEEMIT_CHAIN_ID"] = STEEMIT_CHAIN_ID; result["STEEMIT_COMMENT_REWARD_FUND_NAME"] = STEEMIT_COMMENT_REWARD_FUND_NAME; - result["STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME"] = STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME; - result["STEEMIT_TEMP_LINEAR_REWARD_FUND_ID"] = STEEMIT_TEMP_LINEAR_REWARD_FUND_ID; result["STEEMIT_CONTENT_APR_PERCENT"] = STEEMIT_CONTENT_APR_PERCENT; result["STEEMIT_CONTENT_CONSTANT_HF0"] = STEEMIT_CONTENT_CONSTANT_HF0; result["STEEMIT_CONTENT_REWARD_PERCENT"] = STEEMIT_CONTENT_REWARD_PERCENT; diff --git a/libraries/protocol/include/steemit/protocol/config.hpp b/libraries/protocol/include/steemit/protocol/config.hpp index 5336f4a922..495cb67198 100644 --- a/libraries/protocol/include/steemit/protocol/config.hpp +++ b/libraries/protocol/include/steemit/protocol/config.hpp @@ -3,7 +3,7 @@ */ #pragma once -#define STEEMIT_BLOCKCHAIN_VERSION ( version(0, 19, 0) ) +#define STEEMIT_BLOCKCHAIN_VERSION ( version(0, 19, 1) ) #define STEEMIT_BLOCKCHAIN_HARDFORK_VERSION ( hardfork_version( STEEMIT_BLOCKCHAIN_VERSION ) ) #ifdef IS_TEST_NET @@ -131,12 +131,12 @@ #define STEEMIT_FREE_TRANSACTIONS_WITH_NEW_ACCOUNT 100 #define STEEMIT_BANDWIDTH_AVERAGE_WINDOW_SECONDS (60*60*24*7) ///< 1 week -#define STEEMIT_BANDWIDTH_PRECISION 1000000ll ///< 1 million +#define STEEMIT_BANDWIDTH_PRECISION (uint64_t(1000000)) ///< 1 million #define STEEMIT_MAX_COMMENT_DEPTH_PRE_HF17 6 #define STEEMIT_MAX_COMMENT_DEPTH 0xffff // 64k #define STEEMIT_SOFT_MAX_COMMENT_DEPTH 0xff // 255 -#define STEEMIT_MAX_RESERVE_RATIO (20000) +#define STEEMIT_MAX_RESERVE_RATIO (20000) #define STEEMIT_CREATE_ACCOUNT_WITH_STEEM_MODIFIER 30 #define STEEMIT_CREATE_ACCOUNT_DELEGATION_RATIO 5 @@ -163,8 +163,6 @@ #define STEEMIT_POST_REWARD_FUND_NAME ("post") #define STEEMIT_COMMENT_REWARD_FUND_NAME ("comment") -#define STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME ("linear") -#define STEEMIT_TEMP_LINEAR_REWARD_FUND_ID (1) #define STEEMIT_RECENT_RSHARES_DECAY_RATE_HF17 (fc::days(30)) #define STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19 (fc::days(15)) #define STEEMIT_CONTENT_CONSTANT_HF0 (uint128_t(uint64_t(2000000000000ll))) diff --git a/libraries/protocol/include/steemit/protocol/operations.hpp b/libraries/protocol/include/steemit/protocol/operations.hpp index 75252faa7f..8c0605454e 100644 --- a/libraries/protocol/include/steemit/protocol/operations.hpp +++ b/libraries/protocol/include/steemit/protocol/operations.hpp @@ -76,7 +76,8 @@ namespace steemit { namespace protocol { hardfork_operation, comment_payout_update_operation, return_vesting_delegation_operation, - comment_benefactor_reward_operation + comment_benefactor_reward_operation, + producer_reward_operation > operation; /*void operation_get_required_authorities( const operation& op, diff --git a/libraries/protocol/include/steemit/protocol/steem_virtual_operations.hpp b/libraries/protocol/include/steemit/protocol/steem_virtual_operations.hpp index fe74801da2..fdb1e8a5ee 100644 --- a/libraries/protocol/include/steemit/protocol/steem_virtual_operations.hpp +++ b/libraries/protocol/include/steemit/protocol/steem_virtual_operations.hpp @@ -166,6 +166,16 @@ namespace steemit { namespace protocol { asset reward; }; + struct producer_reward_operation : public virtual_operation + { + producer_reward_operation(){} + producer_reward_operation( const string& p, const asset& v ) : producer( p ), vesting_shares( v ) {} + + account_name_type producer; + asset vesting_shares; + + }; + } } //steemit::protocol FC_REFLECT( steemit::protocol::author_reward_operation, (author)(permlink)(sbd_payout)(steem_payout)(vesting_payout) ) @@ -182,3 +192,4 @@ FC_REFLECT( steemit::protocol::hardfork_operation, (hardfork_id) ) FC_REFLECT( steemit::protocol::comment_payout_update_operation, (author)(permlink) ) FC_REFLECT( steemit::protocol::return_vesting_delegation_operation, (account)(vesting_shares) ) FC_REFLECT( steemit::protocol::comment_benefactor_reward_operation, (benefactor)(author)(permlink)(reward) ) +FC_REFLECT( steemit::protocol::producer_reward_operation, (producer)(vesting_shares) ) diff --git a/libraries/wallet/include/steemit/wallet/wallet.hpp b/libraries/wallet/include/steemit/wallet/wallet.hpp index d00a809347..349ef528d4 100644 --- a/libraries/wallet/include/steemit/wallet/wallet.hpp +++ b/libraries/wallet/include/steemit/wallet/wallet.hpp @@ -956,6 +956,11 @@ class wallet_api std::shared_ptr my; void encrypt_keys(); + /** + * Checks memos against private keys on account and imported in wallet + */ + void check_memo( const string& memo, const account_api_obj& account )const; + /** * Returns the encrypted memo if memo starts with '#' otherwise returns memo */ diff --git a/libraries/wallet/wallet.cpp b/libraries/wallet/wallet.cpp index aa680c23e1..1b7d13f800 100644 --- a/libraries/wallet/wallet.cpp +++ b/libraries/wallet/wallet.cpp @@ -1763,6 +1763,62 @@ annotated_signed_transaction wallet_api::vote_for_witness(string voting_account, return my->sign_transaction( tx, broadcast ); } FC_CAPTURE_AND_RETHROW( (voting_account)(witness_to_vote_for)(approve)(broadcast) ) } +void wallet_api::check_memo( const string& memo, const account_api_obj& account )const +{ + vector< public_key_type > keys; + + try + { + // Check if memo is a private key + keys.push_back( fc::ecc::extended_private_key::from_base58( memo ).get_public_key() ); + } + catch( fc::parse_error_exception& ) {} + catch( fc::assert_exception& ) {} + + // Get possible keys if memo was an account password + string owner_seed = account.name + "owner" + memo; + auto owner_secret = fc::sha256::hash( owner_seed.c_str(), owner_seed.size() ); + keys.push_back( fc::ecc::private_key::regenerate( owner_secret ).get_public_key() ); + + string active_seed = account.name + "active" + memo; + auto active_secret = fc::sha256::hash( active_seed.c_str(), active_seed.size() ); + keys.push_back( fc::ecc::private_key::regenerate( active_secret ).get_public_key() ); + + string posting_seed = account.name + "posting" + memo; + auto posting_secret = fc::sha256::hash( posting_seed.c_str(), posting_seed.size() ); + keys.push_back( fc::ecc::private_key::regenerate( posting_secret ).get_public_key() ); + + // Check keys against public keys in authorites + for( auto& key_weight_pair : account.owner.key_auths ) + { + for( auto& key : keys ) + FC_ASSERT( key_weight_pair.first != key, "Detected private owner key in memo field. Cancelling transaction." ); + } + + for( auto& key_weight_pair : account.active.key_auths ) + { + for( auto& key : keys ) + FC_ASSERT( key_weight_pair.first != key, "Detected private active key in memo field. Cancelling transaction." ); + } + + for( auto& key_weight_pair : account.posting.key_auths ) + { + for( auto& key : keys ) + FC_ASSERT( key_weight_pair.first != key, "Detected private posting key in memo field. Cancelling transaction." ); + } + + const auto& memo_key = account.memo_key; + for( auto& key : keys ) + FC_ASSERT( memo_key != key, "Detected private memo key in memo field. Cancelling transaction." ); + + // Check against imported keys + for( auto& key_pair : my->_keys ) + { + for( auto& key : keys ) + FC_ASSERT( key != key_pair.first, "Detected imported private key in memo field. Cancelling trasanction." ); + } +} + string wallet_api::get_encrypted_memo( string from, string to, string memo ) { if( memo.size() > 0 && memo[0] == '#' ) { @@ -1794,6 +1850,7 @@ string wallet_api::get_encrypted_memo( string from, string to, string memo ) { annotated_signed_transaction wallet_api::transfer(string from, string to, asset amount, string memo, bool broadcast ) { try { FC_ASSERT( !is_locked() ); + check_memo( memo, get_account( from ) ); transfer_operation op; op.from = from; op.to = to; @@ -1926,6 +1983,7 @@ annotated_signed_transaction wallet_api::escrow_release( annotated_signed_transaction wallet_api::transfer_to_savings( string from, string to, asset amount, string memo, bool broadcast ) { FC_ASSERT( !is_locked() ); + check_memo( memo, get_account( from ) ); transfer_to_savings_operation op; op.from = from; op.to = to; @@ -1945,6 +2003,7 @@ annotated_signed_transaction wallet_api::transfer_to_savings( string from, strin annotated_signed_transaction wallet_api::transfer_from_savings( string from, uint32_t request_id, string to, asset amount, string memo, bool broadcast ) { FC_ASSERT( !is_locked() ); + check_memo( memo, get_account( from ) ); transfer_from_savings_operation op; op.from = from; op.request_id = request_id; diff --git a/programs/steemd/CMakeLists.txt b/programs/steemd/CMakeLists.txt index 018037e278..aa688c6e85 100644 --- a/programs/steemd/CMakeLists.txt +++ b/programs/steemd/CMakeLists.txt @@ -9,8 +9,18 @@ if( GPERFTOOLS_FOUND ) list( APPEND PLATFORM_SPECIFIC_LIBS tcmalloc ) endif() -target_link_libraries( steemd - PRIVATE steemit_external_plugins steemit_internal_plugins steemit_mf_plugins steemit_app steemit_witness steemit_account_history steemit_chain steemit_protocol fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( steemd PRIVATE + steemit_plugins + steemit_mf_plugins + steemit_app + steemit_witness + steemit_account_history + steemit_chain + steemit_protocol + fc + ${CMAKE_DL_LIBS} + ${PLATFORM_SPECIFIC_LIBS} +) install( TARGETS steemd diff --git a/programs/util/newplugin.py b/programs/util/newplugin.py index c778bc441a..3acec4405c 100755 --- a/programs/util/newplugin.py +++ b/programs/util/newplugin.py @@ -1,6 +1,13 @@ #!/usr/bin/env python3 templates = { +"plugin.json" : +"""{{ + "plugin_name": "{plugin_name}", + "plugin_project": "{plugin_provider}_{plugin_name}" +}} +""", + "CMakeLists.txt" : """file(GLOB HEADERS "include/{plugin_provider}/plugins/{plugin_name}/*.hpp") diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp index 5f1258deb6..7168e3ee66 100644 --- a/tests/tests/operation_tests.cpp +++ b/tests/tests/operation_tests.cpp @@ -668,6 +668,7 @@ BOOST_AUTO_TEST_CASE( comment_delete_apply ) tx.clear(); tx.operations.push_back( op ); + tx.set_expiration( db.head_block_time() + STEEMIT_MIN_TRANSACTION_EXPIRATION_LIMIT ); tx.sign( alice_private_key, db.get_chain_id() ); STEEMIT_REQUIRE_THROW( db.push_transaction( tx, 0 ), fc::assert_exception ); @@ -6651,97 +6652,5 @@ BOOST_AUTO_TEST_CASE( comment_beneficiaries_apply ) FC_LOG_AND_RETHROW() } -#warning( "TODO: This test is not needed after HF19" ) -BOOST_AUTO_TEST_CASE( vote_weight_test ) -{ - try - { - BOOST_TEST_MESSAGE( "Test Comment Beneficiaries" ); - ACTORS( (alice)(bob)(sam)(dave) ) - generate_block(); - - set_price_feed( price( ASSET( "1.000 TESTS" ), ASSET( "1.000 TBD" ) ) ); - - comment_operation comment; - vote_operation vote; - signed_transaction tx; - - comment.author = "alice"; - comment.permlink = "test"; - comment.parent_permlink = "test"; - comment.title = "test"; - comment.body = "foobar"; - - tx.operations.push_back( comment ); - tx.set_expiration( db.head_block_time() + STEEMIT_MAX_TIME_UNTIL_EXPIRATION ); - tx.sign( alice_private_key, db.get_chain_id() ); - db.push_transaction( tx, 0 ); - - generate_blocks( 10 ); - - BOOST_TEST_MESSAGE( "--- Testing initial vote" ); - - vote.voter = "alice"; - vote.author = "alice"; - vote.permlink = "test"; - vote.weight = STEEMIT_100_PERCENT; - - tx.clear(); - tx.operations.push_back( vote ); - tx.sign( alice_private_key, db.get_chain_id() ); - db.push_transaction( tx, 0 ); - - { - const auto& alice = db.get_account( "alice" ); - const auto& alice_comment = db.get_comment( comment.author, comment.permlink ); - const auto& alice_vote = db.get< comment_vote_object, by_comment_voter >( boost::make_tuple( alice_comment.id, alice.id ) ); - - BOOST_REQUIRE( alice_vote.weight == alice_vote.sqrt_weight ); - BOOST_REQUIRE( alice_comment.total_vote_weight == alice_comment.total_sqrt_vote_weight ); - } - - generate_blocks( 10 ); - - BOOST_TEST_MESSAGE( "--- Testing second vote" ); - - vote.voter = "bob"; - - tx.clear(); - tx.operations.push_back( vote ); - tx.sign( bob_private_key, db.get_chain_id() ); - db.push_transaction( tx, 0 ); - - { - const auto& bob = db.get_account( "bob" ); - const auto& alice_comment = db.get_comment( comment.author, comment.permlink ); - const auto& bob_vote = db.get< comment_vote_object, by_comment_voter >( boost::make_tuple( alice_comment.id, bob.id ) ); - - BOOST_REQUIRE( bob_vote.weight == bob_vote.sqrt_weight ); - BOOST_REQUIRE( alice_comment.total_vote_weight == alice_comment.total_sqrt_vote_weight ); - } - - generate_blocks( 10 ); - - BOOST_TEST_MESSAGE( "--- Testing removing a vote" ); - - vote.weight = 0; - - tx.clear(); - tx.operations.push_back( vote ); - tx.sign( bob_private_key, db.get_chain_id() ); - db.push_transaction( tx, 0 ); - - { - const auto& bob = db.get_account( "bob" ); - const auto& alice_comment = db.get_comment( comment.author, comment.permlink ); - const auto& bob_vote = db.get< comment_vote_object, by_comment_voter >( boost::make_tuple( alice_comment.id, bob.id ) ); - - BOOST_REQUIRE( bob_vote.weight == bob_vote.sqrt_weight ); - BOOST_REQUIRE( alice_comment.total_vote_weight == alice_comment.total_sqrt_vote_weight ); - } - } - FC_LOG_AND_RETHROW() -} - BOOST_AUTO_TEST_SUITE_END() #endif diff --git a/tests/tests/operation_time_tests.cpp b/tests/tests/operation_time_tests.cpp index f3af8da2b7..4f87f44c1a 100644 --- a/tests/tests/operation_time_tests.cpp +++ b/tests/tests/operation_time_tests.cpp @@ -2073,8 +2073,8 @@ BOOST_AUTO_TEST_CASE( liquidity_rewards ) generate_blocks( db.head_block_time() + fc::seconds( STEEMIT_MIN_LIQUIDITY_REWARD_PERIOD_SEC_HF10.to_seconds() / 2 ), true ); - ops = get_last_operations( 1 ); - fill_order_op = ops[0].get< fill_order_operation >(); + ops = get_last_operations( 3 ); + fill_order_op = ops[2].get< fill_order_operation >(); BOOST_REQUIRE( fill_order_op.open_owner == "alice" ); BOOST_REQUIRE( fill_order_op.open_orderid == 6 ); @@ -2665,7 +2665,7 @@ BOOST_AUTO_TEST_CASE( sbd_stability ) { try { - resize_shared_mem( 1024 * 1024 * 256 ); // Due to number of blocks in the test, it requires a large file. (32 MB) + resize_shared_mem( 1024 * 1024 * 512 ); // Due to number of blocks in the test, it requires a large file. (64 MB) // Using the debug node plugin to manually set account balances to create required market conditions for this test auto db_plugin = app.register_plugin< steemit::plugin::debug_node::debug_node_plugin >();