From e31c998a5833ec907f08f4cc1e256154dfa6d18a Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Thu, 20 Apr 2017 16:36:06 -0400 Subject: [PATCH 01/59] Add HF19 constants #1064 --- libraries/chain/database.cpp | 5 +++++ libraries/chain/hardfork.d/0-preamble.hf | 2 +- libraries/chain/hardfork.d/0_19.hf | 17 +++++++++++++++++ .../include/steemit/protocol/config.hpp | 2 +- 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 libraries/chain/hardfork.d/0_19.hf diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index b775cb5e55..9adfe931b4 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -3519,6 +3519,9 @@ void database::init_hardforks() FC_ASSERT( STEEMIT_HARDFORK_0_18 == 18, "Invalid hardfork configuration" ); _hardfork_times[ STEEMIT_HARDFORK_0_18 ] = fc::time_point_sec( STEEMIT_HARDFORK_0_18_TIME ); _hardfork_versions[ STEEMIT_HARDFORK_0_18 ] = STEEMIT_HARDFORK_0_18_VERSION; + FC_ASSERT( STEEMIT_HARDFORK_0_19 == 19, "Invalid hardfork configuration" ); + _hardfork_times[ STEEMIT_HARDFORK_0_19 ] = fc::time_point_sec( STEEMIT_HARDFORK_0_19_TIME ); + _hardfork_versions[ STEEMIT_HARDFORK_0_19 ] = STEEMIT_HARDFORK_0_19_VERSION; const auto& hardforks = get_hardfork_property_object(); @@ -3805,6 +3808,8 @@ void database::apply_hardfork( uint32_t hardfork ) break; case STEEMIT_HARDFORK_0_18: break; + case STEEMIT_HARDFORK_0_19: + break; default: break; } diff --git a/libraries/chain/hardfork.d/0-preamble.hf b/libraries/chain/hardfork.d/0-preamble.hf index 07cb8787d1..4e118daefc 100644 --- a/libraries/chain/hardfork.d/0-preamble.hf +++ b/libraries/chain/hardfork.d/0-preamble.hf @@ -48,4 +48,4 @@ FC_REFLECT( steemit::chain::hardfork_property_object, (next_hardfork)(next_hardfork_time) ) CHAINBASE_SET_INDEX_TYPE( steemit::chain::hardfork_property_object, steemit::chain::hardfork_property_index ) -#define STEEMIT_NUM_HARDFORKS 18 +#define STEEMIT_NUM_HARDFORKS 19 diff --git a/libraries/chain/hardfork.d/0_19.hf b/libraries/chain/hardfork.d/0_19.hf new file mode 100644 index 0000000000..31e645350c --- /dev/null +++ b/libraries/chain/hardfork.d/0_19.hf @@ -0,0 +1,17 @@ +#ifndef STEEMIT_HARDFORK_0_19 +#define STEEMIT_HARDFORK_0_19 19 + +#define STEEMIT_HARDFORK_0_19_TIME 1590886000 // TBD + +#define STEEMIT_HARDFORK_0_19__822 (STEEMIT_HARDFORK_0_19) +#define STEEMIT_HARDFORK_0_19__876 (STEEMIT_HARDFORK_0_19) +#define STEEMIT_HARDFORK_0_19__971 (STEEMIT_HARDFORK_0_19) +#define STEEMIT_HARDFORK_0_19__977 (STEEMIT_HARDFORK_0_19) +#define STEEMIT_HARDFORK_0_19__987 (STEEMIT_HARDFORK_0_19) +#define STEEMIT_HARDFORK_0_19__997 (STEEMIT_HARDFORK_0_19) +#define STEEMIT_HARDFORK_0_19__1051 (STEEMIT_HARDFORK_0_19) +#define STEEMIT_HARDFORK_0_19__1053 (STEEMIT_HARDFORK_0_19) + +#define STEEMIT_HARDFORK_0_19_VERSION hardfork_version( 0, 19 ) + +#endif diff --git a/libraries/protocol/include/steemit/protocol/config.hpp b/libraries/protocol/include/steemit/protocol/config.hpp index e87ab4cff8..fdb93c5062 100644 --- a/libraries/protocol/include/steemit/protocol/config.hpp +++ b/libraries/protocol/include/steemit/protocol/config.hpp @@ -3,7 +3,7 @@ */ #pragma once -#define STEEMIT_BLOCKCHAIN_VERSION ( version(0, 18, 5) ) +#define STEEMIT_BLOCKCHAIN_VERSION ( version(0, 19, 0) ) #define STEEMIT_BLOCKCHAIN_HARDFORK_VERSION ( hardfork_version( STEEMIT_BLOCKCHAIN_VERSION ) ) #ifdef IS_TEST_NET From 4eadb881df9b010c4e32399f5a2f44898e6e086b Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Thu, 20 Apr 2017 16:53:35 -0400 Subject: [PATCH 02/59] Correctly expire the witness price feed #822 --- libraries/chain/database.cpp | 10 +++++++++- libraries/protocol/get_config.cpp | 2 +- libraries/protocol/include/steemit/protocol/config.hpp | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 9adfe931b4..ad8b059e37 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -2678,7 +2678,15 @@ try { for( int i = 0; i < wso.num_scheduled_witnesses; i++ ) { const auto& wit = get_witness( wso.current_shuffled_witnesses[i] ); - if( wit.last_sbd_exchange_update < now + STEEMIT_MAX_FEED_AGE && + if( has_hardfork( STEEMIT_HARDFORK_0_19__822 ) ) + { + if( now < wit.last_sbd_exchange_update + STEEMIT_MAX_FEED_AGE_SECONDS + && !wit.sbd_exchange_rate.is_null() ) + { + feeds.push_back( wit.sbd_exchange_rate ); + } + } + else if( wit.last_sbd_exchange_update < now + STEEMIT_MAX_FEED_AGE_SECONDS && !wit.sbd_exchange_rate.is_null() ) { feeds.push_back( wit.sbd_exchange_rate ); diff --git a/libraries/protocol/get_config.cpp b/libraries/protocol/get_config.cpp index 86807111e9..a94a35ad47 100644 --- a/libraries/protocol/get_config.cpp +++ b/libraries/protocol/get_config.cpp @@ -88,7 +88,7 @@ fc::variant_object get_config() result["STEEMIT_MAX_CASHOUT_WINDOW_SECONDS"] = STEEMIT_MAX_CASHOUT_WINDOW_SECONDS; result["STEEMIT_MAX_COMMENT_DEPTH"] = STEEMIT_MAX_COMMENT_DEPTH; result["STEEMIT_MAX_COMMENT_DEPTH_PRE_HF17"] = STEEMIT_MAX_COMMENT_DEPTH_PRE_HF17; - result["STEEMIT_MAX_FEED_AGE"] = STEEMIT_MAX_FEED_AGE; + result["STEEMIT_MAX_FEED_AGE_SECONDS"] = STEEMIT_MAX_FEED_AGE_SECONDS; result["STEEMIT_MAX_INSTANCE_ID"] = STEEMIT_MAX_INSTANCE_ID; result["STEEMIT_MAX_MEMO_SIZE"] = STEEMIT_MAX_MEMO_SIZE; result["STEEMIT_MAX_WITNESSES"] = STEEMIT_MAX_WITNESSES; diff --git a/libraries/protocol/include/steemit/protocol/config.hpp b/libraries/protocol/include/steemit/protocol/config.hpp index fdb93c5062..7fd008b73a 100644 --- a/libraries/protocol/include/steemit/protocol/config.hpp +++ b/libraries/protocol/include/steemit/protocol/config.hpp @@ -229,7 +229,7 @@ #define STEEMIT_FEED_INTERVAL_BLOCKS (STEEMIT_BLOCKS_PER_HOUR) #define STEEMIT_FEED_HISTORY_WINDOW_PRE_HF_16 (24*7) /// 7 days * 24 hours per day #define STEEMIT_FEED_HISTORY_WINDOW (12*7) // 3.5 days -#define STEEMIT_MAX_FEED_AGE (fc::days(7)) +#define STEEMIT_MAX_FEED_AGE_SECONDS (60*60*24*7) // 7 days #define STEEMIT_MIN_FEEDS (STEEMIT_MAX_WITNESSES/3) /// protects the network from conversions before price has been established #define STEEMIT_CONVERSION_DELAY_PRE_HF_16 (fc::days(7)) #define STEEMIT_CONVERSION_DELAY (fc::hours(STEEMIT_FEED_HISTORY_WINDOW)) //3.5 day conversion From c773e56e2dc0fc648fc6f91588212b857b02b8e9 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 21 Apr 2017 11:39:46 -0400 Subject: [PATCH 03/59] Correctly enforce the account creation fee #987 --- libraries/chain/steem_evaluator.cpp | 9 ++++++++- tests/tests/operation_tests.cpp | 9 ++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 5b0ea1d6ab..c50e37c5a3 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -115,7 +115,14 @@ void account_create_evaluator::do_apply( const account_create_operation& o ) FC_ASSERT( creator.balance >= o.fee, "Insufficient balance to create account.", ( "creator.balance", creator.balance )( "required", o.fee ) ); - if( _db.has_hardfork( STEEMIT_HARDFORK_0_17__818 ) ) + if( _db.has_hardfork( STEEMIT_HARDFORK_0_19__987) ) + { + const witness_schedule_object& wso = _db.get_witness_schedule_object(); + FC_ASSERT( o.fee >= asset( wso.median_props.account_creation_fee.amount * STEEMIT_CREATE_ACCOUNT_WITH_STEEM_MODIFIER, STEEM_SYMBOL ), "Insufficient Fee: ${f} required, ${p} provided.", + ("f", wso.median_props.account_creation_fee * asset( STEEMIT_CREATE_ACCOUNT_WITH_STEEM_MODIFIER, STEEM_SYMBOL ) ) + ("p", o.fee) ); + } + else if( _db.has_hardfork( STEEMIT_HARDFORK_0_17__818 ) ) { if( _db.is_producing() ) FC_ASSERT( false, "account_create_operation is temporarily disabled. Please use account_create_with_delegation_operation instead" ); diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp index 2d9ac0361c..e10f8c3e9c 100644 --- a/tests/tests/operation_tests.cpp +++ b/tests/tests/operation_tests.cpp @@ -98,8 +98,6 @@ BOOST_AUTO_TEST_CASE( account_create_apply ) tx.operations.push_back( op ); tx.sign( init_account_priv_key, db.get_chain_id() ); tx.validate(); - STEEMIT_REQUIRE_THROW( db.push_transaction( tx, 0 ), fc::assert_exception ); - /* Temporarily commented out while op is disabled db.push_transaction( tx, 0 ); const account_object& acct = db.get_account( "alice" ); @@ -152,15 +150,13 @@ BOOST_AUTO_TEST_CASE( account_create_apply ) STEEMIT_REQUIRE_THROW( db.push_transaction( tx, 0 ), fc::exception ); validate_database(); - idump( (db.get_account( STEEMIT_INIT_MINER_NAME ).balance) ); - BOOST_TEST_MESSAGE( "--- Test failure covering witness fee" ); generate_block(); db_plugin->debug_update( [=]( database& db ) { db.modify( db.get_witness_schedule_object(), [&]( witness_schedule_object& wso ) { - wso.median_props.account_creation_fee = ASSET( "1.000 TESTS" ); + wso.median_props.account_creation_fee = ASSET( "10.000 TESTS" ); }); }); generate_block(); @@ -169,9 +165,8 @@ BOOST_AUTO_TEST_CASE( account_create_apply ) op.fee = ASSET( "1.000 TESTS" ); tx.operations.push_back( op ); tx.sign( init_account_priv_key, db.get_chain_id() ); - STEEMIT_REQUIRE_THROW( db.push_transaction( tx, 0 ), fc::exception ) ); + STEEMIT_REQUIRE_THROW( db.push_transaction( tx, 0 ), fc::exception ); validate_database(); - */ } FC_LOG_AND_RETHROW() } From be045a8d7a5e03ed65dd8ce938f99aa655398fed Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 21 Apr 2017 14:26:11 -0400 Subject: [PATCH 04/59] Prevent deleting a comment after payout and add tests for deleting comments #876 --- libraries/chain/steem_evaluator.cpp | 3 + tests/tests/operation_tests.cpp | 105 ++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 5b0ea1d6ab..7f07ec8736 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -367,6 +367,9 @@ void delete_comment_evaluator::do_apply( const delete_comment_operation& o ) const auto& comment = _db.get_comment( o.author, o.permlink ); FC_ASSERT( comment.children == 0, "Cannot delete a comment with replies." ); + if( _db.has_hardfork( STEEMIT_HARDFORK_0_19__876 ) ) + FC_ASSERT( comment.cashout_time != fc::time_point_sec::maximum() ); + if( _db.is_producing() ) { FC_ASSERT( comment.net_rshares <= 0, "Cannot delete a comment with net positive votes." ); } diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp index 2d9ac0361c..dbded9bb65 100644 --- a/tests/tests/operation_tests.cpp +++ b/tests/tests/operation_tests.cpp @@ -601,6 +601,111 @@ BOOST_AUTO_TEST_CASE( comment_apply ) FC_LOG_AND_RETHROW() } +BOOST_AUTO_TEST_CASE( comment_delete_apply ) +{ + try + { + BOOST_TEST_MESSAGE( "Testing: comment_delete_apply" ); + ACTORS( (alice) ) + generate_block(); + + vest( "alice", ASSET( "1000.000 TESTS" ) ); + + generate_block(); + + set_price_feed( price( ASSET( "1.000 TESTS" ), ASSET( "1.000 TBD" ) ) ); + + signed_transaction tx; + comment_operation comment; + vote_operation vote; + + comment.author = "alice"; + comment.permlink = "test1"; + comment.title = "test"; + comment.body = "foo bar"; + comment.parent_permlink = "test"; + vote.voter = "alice"; + vote.author = "alice"; + vote.permlink = "test1"; + vote.weight = STEEMIT_100_PERCENT; + tx.operations.push_back( comment ); + tx.operations.push_back( vote ); + tx.set_expiration( db.head_block_time() + STEEMIT_MIN_TRANSACTION_EXPIRATION_LIMIT ); + tx.sign( alice_private_key, db.get_chain_id() ); + db.push_transaction( tx, 0 ); + + BOOST_TEST_MESSAGE( "--- Test failue deleting a comment with positive rshares" ); + + delete_comment_operation op; + op.author = "alice"; + op.permlink = "test1"; + tx.clear(); + tx.operations.push_back( op ); + tx.sign( alice_private_key, db.get_chain_id() ); + STEEMIT_REQUIRE_THROW( db.push_transaction( tx, 0 ), fc::assert_exception ); + + + BOOST_TEST_MESSAGE( "--- Test success deleting a comment with negative rshares" ); + + generate_block(); + vote.weight = -1 * STEEMIT_100_PERCENT; + tx.clear(); + tx.operations.push_back( vote ); + tx.operations.push_back( op ); + tx.sign( alice_private_key, db.get_chain_id() ); + db.push_transaction( tx, 0 ); + + auto test_comment = db.find< comment_object, by_permlink >( boost::make_tuple( "alice", string( "test1" ) ) ); + BOOST_REQUIRE( test_comment == nullptr ); + + + BOOST_TEST_MESSAGE( "--- Test failure deleting a comment past cashout" ); + generate_blocks( STEEMIT_MIN_ROOT_COMMENT_INTERVAL.to_seconds() / STEEMIT_BLOCK_INTERVAL ); + + tx.clear(); + tx.operations.push_back( comment ); + tx.set_expiration( db.head_block_time() + STEEMIT_MIN_TRANSACTION_EXPIRATION_LIMIT ); + tx.sign( alice_private_key, db.get_chain_id() ); + db.push_transaction( tx, 0 ); + + generate_blocks( STEEMIT_CASHOUT_WINDOW_SECONDS / STEEMIT_BLOCK_INTERVAL ); + BOOST_REQUIRE( db.get_comment( "alice", string( "test1" ) ).cashout_time == fc::time_point_sec::maximum() ); + + tx.clear(); + tx.operations.push_back( op ); + tx.sign( alice_private_key, db.get_chain_id() ); + STEEMIT_REQUIRE_THROW( db.push_transaction( tx, 0 ), fc::assert_exception ); + + + BOOST_TEST_MESSAGE( "--- Test failure deleting a comment with a reply" ); + + comment.permlink = "test2"; + comment.parent_author = "alice"; + comment.parent_permlink = "test1"; + tx.clear(); + tx.operations.push_back( comment ); + tx.set_expiration( db.head_block_time() + STEEMIT_MIN_TRANSACTION_EXPIRATION_LIMIT ); + tx.sign( alice_private_key, db.get_chain_id() ); + db.push_transaction( tx, 0 ); + + generate_blocks( STEEMIT_MIN_ROOT_COMMENT_INTERVAL.to_seconds() / STEEMIT_BLOCK_INTERVAL ); + comment.permlink = "test3"; + comment.parent_permlink = "test2"; + tx.clear(); + tx.operations.push_back( comment ); + tx.set_expiration( db.head_block_time() + STEEMIT_MIN_TRANSACTION_EXPIRATION_LIMIT ); + tx.sign( alice_private_key, db.get_chain_id() ); + db.push_transaction( tx, 0 ); + + op.permlink = "test2"; + tx.clear(); + tx.operations.push_back( op ); + tx.sign( alice_private_key, db.get_chain_id() ); + STEEMIT_REQUIRE_THROW( db.push_transaction( tx, 0 ), fc::assert_exception ); + } + FC_LOG_AND_RETHROW() +} + BOOST_AUTO_TEST_CASE( vote_validate ) { try From a401f3690b6220cd853fffbb1c1466a9541d25f2 Mon Sep 17 00:00:00 2001 From: Gandalf Date: Tue, 16 May 2017 23:17:37 +0200 Subject: [PATCH 05/59] Removing seed nodes that failed to respond all consecutive attempts --- doc/seednodes.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/seednodes.txt b/doc/seednodes.txt index 5a5383392b..1f3ac54786 100644 --- a/doc/seednodes.txt +++ b/doc/seednodes.txt @@ -1,15 +1,15 @@ -seed.riversteem.com:2001 # riverhead +#####seed.riversteem.com:2001 # riverhead steem-seed1.abit-more.com:2001 # abit 52.74.152.79:2001 # smooth.witness seed.steemd.com:34191 # roadscape steemwitness.matthewniemerg.com:2001 # complexring -steemd.pharesim.me:2001 # pharesim +#####steemd.pharesim.me:2001 # pharesim seed.jesta.us:2001 # jesta seed.liondani.com:2016 # liondani anyx.co:2001 # anyx -seed.xeldal.com:12150 # xeldal +#####seed.xeldal.com:12150 # xeldal seed.steemnodes.com:2001 # wackou -steem.clawmap.com:2001 # steempty +#####steem.clawmap.com:2001 # steempty gtg.steem.house:2001 # gtg 192.99.3.29:2001 # joseph 5.9.18.213:2001 # pfunk @@ -17,20 +17,20 @@ lafonasteem.com:2001 # lafona seed.rossco99.com:2001 # rossco99 212.47.249.84:40696 # ihashfury seed.steemfeeder.com:2001 # au1nethyb1 -52.175.211.168:2001 # aizensou +#####52.175.211.168:2001 # aizensou seed.roelandp.nl:2001 # roelandp 81.89.101.133:2001 # cyrano.witness steem.global:2001 # klye seed.esteem.ws:2001 # good-karma 176.31.126.187:2001 # timcliff -seed.royaltiffany.me:2001 # royaltiffany +#####seed.royaltiffany.me:2001 # royaltiffany seed.thecryptodrive.com:2001 # thecryptodrive steem-id.altexplorer.xyz:2001 # steem-id seed.bitcoiner.me:2001 # bitcoiner 104.199.118.92:2001 # clayop 192.99.4.226:2001 # dele-puppy -89.36.26.82:2001 # dragosroua -104.198.222.18:2001 # bitcoinparadise -138.197.17.188:2001 # chitty +#####89.36.26.82:2001 # dragosroua +#####104.198.222.18:2001 # bitcoinparadise +#####138.197.17.188:2001 # chitty 88.99.33.113:2001 # cervantes seed.bhuz.info:2001 # bhuz From ef8402633fc5cb70862bbd05e5b6df8eb16334d6 Mon Sep 17 00:00:00 2001 From: Gandalf Date: Wed, 17 May 2017 07:27:04 +0200 Subject: [PATCH 06/59] seed node provided by xeldal recovered, pcste added --- doc/seednodes.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/seednodes.txt b/doc/seednodes.txt index 1f3ac54786..f244172b2a 100644 --- a/doc/seednodes.txt +++ b/doc/seednodes.txt @@ -7,7 +7,7 @@ steemwitness.matthewniemerg.com:2001 # complexring seed.jesta.us:2001 # jesta seed.liondani.com:2016 # liondani anyx.co:2001 # anyx -#####seed.xeldal.com:12150 # xeldal +seed.xeldal.com:12150 # xeldal seed.steemnodes.com:2001 # wackou #####steem.clawmap.com:2001 # steempty gtg.steem.house:2001 # gtg @@ -15,7 +15,7 @@ gtg.steem.house:2001 # gtg 5.9.18.213:2001 # pfunk lafonasteem.com:2001 # lafona seed.rossco99.com:2001 # rossco99 -212.47.249.84:40696 # ihashfury +steem-seed.altcap.io:40696 # ihashfury seed.steemfeeder.com:2001 # au1nethyb1 #####52.175.211.168:2001 # aizensou seed.roelandp.nl:2001 # roelandp @@ -34,3 +34,4 @@ seed.bitcoiner.me:2001 # bitcoiner #####138.197.17.188:2001 # chitty 88.99.33.113:2001 # cervantes seed.bhuz.info:2001 # bhuz +78.46.95.157:2001 # pcste From 6559cf09b141bf842aff26ac9f4e32d65eb3c505 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 21 Apr 2017 11:11:28 -0400 Subject: [PATCH 07/59] Allow removal of delegations regardless of minimum delegation #971 --- libraries/chain/steem_evaluator.cpp | 14 +++++- tests/tests/operation_tests.cpp | 66 +++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 5b0ea1d6ab..8ccd1b166f 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -2219,8 +2219,18 @@ void delegate_vesting_shares_evaluator::do_apply( const delegate_vesting_shares_ { auto delta = delegation->vesting_shares - op.vesting_shares; - FC_ASSERT( delta >= min_update, "Steem Power increase is not enough of a different. min_update: ${min}", ("min", min_update) ); - FC_ASSERT( op.vesting_shares >= min_delegation || op.vesting_shares.amount == 0, "Delegation must be removed or leave minimum delegation amount of ${v}", ("v", min_delegation) ); + if( _db.has_hardfork( STEEMIT_HARDFORK_0_19__971 ) ) + { + FC_ASSERT( ( op.vesting_shares.amount == 0 ) + || ( delta >= min_update && op.vesting_shares >= min_delegation ), + "Delegation must be removed or be a significant change and leave a minimum delegation. min_update: ${min_update}, min_delegation: ${min_delegation}", + ("min_update", min_update)("min_delegation", min_delegation) ); + } + else // TODO: Check and remove after HF19 + { + FC_ASSERT( delta >= min_update, "Steem Power increase is not enough of a different. min_update: ${min}", ("min", min_update) ); + FC_ASSERT( op.vesting_shares >= min_delegation || op.vesting_shares.amount == 0, "Delegation must be removed or leave minimum delegation amount of ${v}", ("v", min_delegation) ); + } _db.create< vesting_delegation_expiration_object >( [&]( vesting_delegation_expiration_object& obj ) { diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp index 2d9ac0361c..c655ee257f 100644 --- a/tests/tests/operation_tests.cpp +++ b/tests/tests/operation_tests.cpp @@ -6293,6 +6293,72 @@ BOOST_AUTO_TEST_CASE( delegate_vesting_shares_apply ) FC_LOG_AND_RETHROW() } +BOOST_AUTO_TEST_CASE( issue_971_vesting_removal ) +{ + // This is a regression test specifically for issue #971 + try + { + BOOST_TEST_MESSAGE( "Test Issue 971 Vesting Removal" ); + ACTORS( (alice)(bob) ) + generate_block(); + + vest( "alice", ASSET( "1000.000 TESTS" ) ); + + generate_block(); + + db_plugin->debug_update( [=]( database& db ) + { + db.modify( db.get_witness_schedule_object(), [&]( witness_schedule_object& w ) + { + w.median_props.account_creation_fee = ASSET( "1.000 TESTS" ); + }); + }); + + generate_block(); + + signed_transaction tx; + delegate_vesting_shares_operation op; + op.vesting_shares = ASSET( "10000000.000000 VESTS"); + op.delegator = "alice"; + op.delegatee = "bob"; + + tx.set_expiration( db.head_block_time() + STEEMIT_MAX_TIME_UNTIL_EXPIRATION ); + tx.operations.push_back( op ); + tx.sign( alice_private_key, db.get_chain_id() ); + db.push_transaction( tx, 0 ); + generate_block(); + const account_object& alice_acc = db.get_account( "alice" ); + const account_object& bob_acc = db.get_account( "bob" ); + + BOOST_REQUIRE( alice_acc.delegated_vesting_shares == ASSET( "10000000.000000 VESTS")); + BOOST_REQUIRE( bob_acc.received_vesting_shares == ASSET( "10000000.000000 VESTS")); + + generate_block(); + + db_plugin->debug_update( [=]( database& db ) + { + db.modify( db.get_witness_schedule_object(), [&]( witness_schedule_object& w ) + { + w.median_props.account_creation_fee = ASSET( "100.000 TESTS" ); + }); + }); + + generate_block(); + + op.vesting_shares = ASSET( "0.000000 VESTS" ); + + tx.clear(); + tx.operations.push_back( op ); + tx.sign( alice_private_key, db.get_chain_id() ); + db.push_transaction( tx, 0 ); + generate_block(); + + BOOST_REQUIRE( alice_acc.delegated_vesting_shares == ASSET( "10000000.000000 VESTS")); + BOOST_REQUIRE( bob_acc.received_vesting_shares == ASSET( "0.000000 VESTS")); + } + FC_LOG_AND_RETHROW() +} + BOOST_AUTO_TEST_CASE( comment_beneficiaries_validate ) { try From dbbac03a4a92158065e2051208ecf8aa4afe10dc Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 21 Apr 2017 12:07:47 -0400 Subject: [PATCH 08/59] Requiring a comment has no positive net-rshares is now consensus #977 --- libraries/chain/steem_evaluator.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 7df99edbdf..7b8a2c566e 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -370,7 +370,8 @@ void delete_comment_evaluator::do_apply( const delete_comment_operation& o ) if( _db.has_hardfork( STEEMIT_HARDFORK_0_19__876 ) ) FC_ASSERT( comment.cashout_time != fc::time_point_sec::maximum() ); - if( _db.is_producing() ) { + // TODO: remove is_producing after HF 19. Check if we can remove conditional altogether after HF. + if( _db.is_producing() || _db.has_hardfork( STEEMIT_HARDFORK_0_19__977 ) ) { FC_ASSERT( comment.net_rshares <= 0, "Cannot delete a comment with net positive votes." ); } if( comment.net_rshares > 0 ) return; From af91650b9761c502da75e68f5ec26f641588a83e Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Thu, 18 May 2017 10:06:44 -0400 Subject: [PATCH 09/59] Add build to gitignore #1118 --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 935b72bfae..063217ccf3 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,8 @@ tests/performance_test doxygen +build + wallet.json witness_node_data_dir From f9b92f2c5c04376e0d092c412b0894be84e79aed Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 21 Apr 2017 16:14:17 -0400 Subject: [PATCH 10/59] Rename vote_regeneration_per_day to vote_power_reserve_rate and reduce the reserve rate to increase vote impact #1053 --- libraries/chain/database.cpp | 6 ++++++ .../chain/include/steemit/chain/global_property_object.hpp | 4 ++-- libraries/chain/steem_evaluator.cpp | 2 +- tests/tests/operation_tests.cpp | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index ad8b059e37..c6a238d1f0 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -3817,6 +3817,12 @@ void database::apply_hardfork( uint32_t hardfork ) case STEEMIT_HARDFORK_0_18: break; case STEEMIT_HARDFORK_0_19: + { + modify( get_dynamic_global_properties(), [&]( dynamic_global_property_object& gpo ) + { + gpo.vote_power_reserve_rate = 10; + }); + } break; default: break; diff --git a/libraries/chain/include/steemit/chain/global_property_object.hpp b/libraries/chain/include/steemit/chain/global_property_object.hpp index 53ed6e600f..075feceaca 100644 --- a/libraries/chain/include/steemit/chain/global_property_object.hpp +++ b/libraries/chain/include/steemit/chain/global_property_object.hpp @@ -141,7 +141,7 @@ namespace steemit { namespace chain { * "wasting" voting power through spillover; any user voting faster than this rate will have * their votes reduced. */ - uint32_t vote_regeneration_per_day = 40; + uint32_t vote_power_reserve_rate = 40; }; typedef multi_index_container< @@ -184,6 +184,6 @@ FC_REFLECT( steemit::chain::dynamic_global_property_object, (last_irreversible_block_num) (max_virtual_bandwidth) (current_reserve_ratio) - (vote_regeneration_per_day) + (vote_power_reserve_rate) ) CHAINBASE_SET_INDEX_TYPE( steemit::chain::dynamic_global_property_object, steemit::chain::dynamic_global_property_index ) diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 7df99edbdf..22258baaf1 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -1174,7 +1174,7 @@ void vote_evaluator::do_apply( const vote_operation& o ) // used_power = (current_power * abs_weight / STEEMIT_100_PERCENT) * (reserve / max_vote_denom) // The second multiplication is rounded up as of HF 259 - int64_t max_vote_denom = dgpo.vote_regeneration_per_day * STEEMIT_VOTE_REGENERATION_SECONDS / (60*60*24); + int64_t max_vote_denom = dgpo.vote_power_reserve_rate * STEEMIT_VOTE_REGENERATION_SECONDS / (60*60*24); FC_ASSERT( max_vote_denom > 0 ); if( !_db.has_hardfork( STEEMIT_HARDFORK_0_14__259 ) ) diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp index c6d3c0b1d6..cbc0f4b5f7 100644 --- a/tests/tests/operation_tests.cpp +++ b/tests/tests/operation_tests.cpp @@ -805,7 +805,7 @@ BOOST_AUTO_TEST_CASE( vote_apply ) auto& alice_comment = db.get_comment( "alice", string( "foo" ) ); auto itr = vote_idx.find( std::make_tuple( alice_comment.id, alice.id ) ); - int64_t max_vote_denom = ( db.get_dynamic_global_properties().vote_regeneration_per_day * STEEMIT_VOTE_REGENERATION_SECONDS ) / (60*60*24); + int64_t max_vote_denom = ( db.get_dynamic_global_properties().vote_power_reserve_rate * STEEMIT_VOTE_REGENERATION_SECONDS ) / (60*60*24); BOOST_REQUIRE( alice.voting_power == old_voting_power - ( ( old_voting_power + max_vote_denom - 1 ) / max_vote_denom ) ); BOOST_REQUIRE( alice.last_vote_time == db.head_block_time() ); From 2ca4bf40d636a6bdb496ff4f7538117522cc9182 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Mon, 24 Apr 2017 13:58:26 -0400 Subject: [PATCH 11/59] Refactor rewards calculations to use curve IDs in reward fund objects #1051 #1052 --- libraries/app/database_api.cpp | 5 +- libraries/chain/database.cpp | 14 +++- .../include/steemit/chain/steem_objects.hpp | 14 ++++ .../include/steemit/chain/util/reward.hpp | 12 +-- libraries/chain/steem_evaluator.cpp | 4 +- libraries/chain/util/reward.cpp | 76 +++++++++---------- tests/tests/operation_time_tests.cpp | 8 +- 7 files changed, 77 insertions(+), 56 deletions(-) diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp index ce1456fb3f..50ebe2f5c1 100755 --- a/libraries/app/database_api.cpp +++ b/libraries/app/database_api.cpp @@ -1140,7 +1140,10 @@ void database_api::set_pending_payout( discussion& d )const { uint128_t vshares; if( my->_db.has_hardfork( STEEMIT_HARDFORK_0_17__774 ) ) - vshares = steemit::chain::util::calculate_claims( d.net_rshares.value > 0 ? d.net_rshares.value : 0 , my->_db.get_reward_fund( my->_db.get_comment( d.author, d.permlink ) ) ); + { + const auto& rf = my->_db.get_reward_fund( my->_db.get_comment( d.author, d.permlink ) ); + vshares = d.net_rshares.value > 0 ? steemit::chain::util::calculate_claims( d.net_rshares.value, rf.author_reward_curve, rf.content_constant ) : 0; + } else vshares = steemit::chain::util::calculate_claims( d.net_rshares.value > 0 ? d.net_rshares.value : 0 ); diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index c6a238d1f0..675c9a24f2 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -1482,8 +1482,14 @@ share_type database::cashout_comment_helper( util::comment_reward_context& ctx, { fill_comment_reward_context_local_state( ctx, comment ); - const share_type reward = has_hardfork( STEEMIT_HARDFORK_0_17__774 ) ? - util::get_rshare_reward( ctx, get_reward_fund( comment ) ) : util::get_rshare_reward( ctx ); + if( has_hardfork( STEEMIT_HARDFORK_0_17__774 ) ) + { + const auto rf = get_reward_fund( comment ); + ctx.reward_curve = rf.author_reward_curve; + ctx.content_constant = rf.content_constant; + } + + const share_type reward = util::get_rshare_reward( ctx ); uint128_t reward_tokens = uint128_t( reward.value ); if( reward_tokens > 0 ) @@ -1639,7 +1645,7 @@ void database::process_comment_cashout() if( current->net_rshares > 0 ) { const auto& rf = get_reward_fund( *current ); - funds[ rf.id._id ].recent_claims += util::calculate_claims( current->net_rshares.value, rf ); + funds[ rf.id._id ].recent_claims += util::calculate_claims( current->net_rshares.value, rf.author_reward_curve, rf.content_constant ); } ++current; @@ -3756,6 +3762,8 @@ void database::apply_hardfork( uint32_t hardfork ) #ifndef IS_TEST_NET rfo.recent_claims = STEEMIT_HF_17_RECENT_CLAIMS; #endif + rfo.author_reward_curve = curve_id::quadratic; + rfo.curation_reward_curve = curve_id::quadratic_curation; }); // As a shortcut in payout processing, we use the id as an array index. diff --git a/libraries/chain/include/steemit/chain/steem_objects.hpp b/libraries/chain/include/steemit/chain/steem_objects.hpp index 8e3603365b..f67317222a 100755 --- a/libraries/chain/include/steemit/chain/steem_objects.hpp +++ b/libraries/chain/include/steemit/chain/steem_objects.hpp @@ -249,6 +249,14 @@ namespace steemit { namespace chain { time_point_sec effective_date; }; + enum curve_id + { + quadratic, + quadratic_curation, + linear, + square_root + }; + class reward_fund_object : public object< reward_fund_object_type, reward_fund_object > { public: @@ -268,6 +276,8 @@ namespace steemit { namespace chain { uint128_t content_constant = 0; uint16_t percent_curation_rewards = 0; uint16_t percent_content_rewards = 0; + curve_id author_reward_curve; + curve_id curation_reward_curve; }; struct by_price; @@ -479,6 +489,8 @@ namespace steemit { namespace chain { #include #include +FC_REFLECT_ENUM( steemit::chain::curve_id, + (quadratic)(quadratic_curation)(linear)(square_root)) FC_REFLECT( steemit::chain::limit_order_object, (id)(created)(expiration)(seller)(orderid)(for_sale)(sell_price) ) @@ -524,5 +536,7 @@ FC_REFLECT( steemit::chain::reward_fund_object, (content_constant) (percent_curation_rewards) (percent_content_rewards) + (author_reward_curve) + (curation_reward_curve) ) CHAINBASE_SET_INDEX_TYPE( steemit::chain::reward_fund_object, steemit::chain::reward_fund_index ) diff --git a/libraries/chain/include/steemit/chain/util/reward.hpp b/libraries/chain/include/steemit/chain/util/reward.hpp index 6954e5ba9c..fe9b8f321a 100644 --- a/libraries/chain/include/steemit/chain/util/reward.hpp +++ b/libraries/chain/include/steemit/chain/util/reward.hpp @@ -27,22 +27,20 @@ struct comment_reward_context uint128_t total_reward_shares2; asset total_reward_fund_steem; price current_steem_price; + curve_id reward_curve = quadratic; + uint128_t content_constant = STEEMIT_CONTENT_CONSTANT_HF0; }; uint64_t get_rshare_reward( const comment_reward_context& ctx ); -uint64_t get_rshare_reward( const comment_reward_context& ctx, const reward_fund_object& rf ); - -uint64_t get_vote_weight( uint64_t vote_rshares, const reward_fund_object& rf ); +uint64_t get_vote_weight( uint64_t vote_rshares, const curve_id& curve, const uint128_t& content_constant ); inline uint128_t get_content_constant_s() { return STEEMIT_CONTENT_CONSTANT_HF0; // looking good for posters } -uint128_t calculate_claims( const uint128_t& rshares ); - -uint128_t calculate_claims( const uint128_t& rshares, const reward_fund_object& rf ); +uint128_t calculate_claims( const uint128_t& rshares, const curve_id& curve = quadratic, const uint128_t& content_constant = STEEMIT_CONTENT_CONSTANT_HF0 ); inline bool is_comment_payout_dust( const price& p, uint64_t steem_payout ) { @@ -58,4 +56,6 @@ FC_REFLECT( steemit::chain::util::comment_reward_context, (total_reward_shares2) (total_reward_fund_steem) (current_steem_price) + (reward_curve) + (content_constant) ) diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 6602fa2bb3..db13b120f8 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -1346,8 +1346,8 @@ void vote_evaluator::do_apply( const vote_operation& o ) if( _db.has_hardfork( STEEMIT_HARDFORK_0_17__774 ) ) { const auto& reward_fund = _db.get_reward_fund( comment ); - uint64_t old_weight = util::get_vote_weight( old_vote_rshares.value, reward_fund ); - uint64_t new_weight = util::get_vote_weight( comment.vote_rshares.value, reward_fund ); + uint64_t old_weight = util::get_vote_weight( old_vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ); + uint64_t new_weight = util::get_vote_weight( comment.vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ); cv.weight = new_weight - old_weight; } else if ( _db.has_hardfork( STEEMIT_HARDFORK_0_1 ) ) diff --git a/libraries/chain/util/reward.cpp b/libraries/chain/util/reward.cpp index 4fe9cd3440..1743bee38f 100644 --- a/libraries/chain/util/reward.cpp +++ b/libraries/chain/util/reward.cpp @@ -11,43 +11,12 @@ uint64_t get_rshare_reward( const comment_reward_context& ctx ) FC_ASSERT( ctx.rshares > 0 ); FC_ASSERT( ctx.total_reward_shares2 > 0 ); - u256 rs(ctx.rshares.value); - u256 rf(ctx.total_reward_fund_steem.amount.value); - u256 total_rshares2 = to256( ctx.total_reward_shares2 ); - - //idump( (ctx) ); - - u256 rs2 = to256( calculate_claims( ctx.rshares.value ) ); - rs2 = ( rs2 * ctx.reward_weight ) / STEEMIT_100_PERCENT; - - u256 payout_u256 = ( rf * rs2 ) / total_rshares2; - FC_ASSERT( payout_u256 <= u256( uint64_t( std::numeric_limits::max() ) ) ); - uint64_t payout = static_cast< uint64_t >( payout_u256 ); - - if( is_comment_payout_dust( ctx.current_steem_price, payout ) ) - payout = 0; - - asset max_steem = to_steem( ctx.current_steem_price, ctx.max_sbd ); - - payout = std::min( payout, uint64_t( max_steem.amount.value ) ); - - return payout; - } FC_CAPTURE_AND_RETHROW( (ctx) ) -} - -uint64_t get_rshare_reward( const comment_reward_context& ctx, const reward_fund_object& rf_object ) -{ - try - { - FC_ASSERT( ctx.rshares > 0 ); - FC_ASSERT( ctx.total_reward_shares2 > 0 ); - u256 rf(ctx.total_reward_fund_steem.amount.value); u256 total_claims = to256( ctx.total_reward_shares2 ); //idump( (ctx) ); - u256 claim = to256( calculate_claims( ctx.rshares.value, rf_object ) ); + u256 claim = to256( calculate_claims( ctx.rshares.value, ctx.reward_curve, ctx.content_constant ) ); claim = ( claim * ctx.reward_weight ) / STEEMIT_100_PERCENT; u256 payout_u256 = ( rf * claim ) / total_claims; @@ -81,6 +50,26 @@ uint64_t get_vote_weight( uint64_t vote_rshares, const reward_fund_object& rf ) return result; } +uint64_t get_vote_weight( uint64_t vote_rshares, const curve_id& curve, const uint128_t& content_constant ) +{ + uint64_t result = 0; + + switch( curve ) + { + case quadratic_curation: + { + uint128_t two_alpha = content_constant * 2; + result = ( uint128_t( vote_rshares, 0 ) / ( two_alpha + vote_rshares ) ).to_uint64(); + } + break; + case square_root: + //result = rshares; + break; + } + + return result; +} + uint128_t calculate_claims( const uint128_t& rshares ) { uint128_t s = get_content_constant_s(); @@ -88,21 +77,24 @@ uint128_t calculate_claims( const uint128_t& rshares ) return rshares_plus_s * rshares_plus_s - s * s; } -uint128_t calculate_claims( const uint128_t& rshares, const reward_fund_object& rf ) +uint128_t calculate_claims( const uint128_t& rshares, const curve_id& curve, const uint128_t& content_constant ) { uint128_t result = 0; - if( rf.name == STEEMIT_POST_REWARD_FUND_NAME || rf.name == STEEMIT_COMMENT_REWARD_FUND_NAME ) - { - uint128_t s = rf.content_constant; - uint128_t rshares_plus_s = rshares + s; - result = rshares_plus_s * rshares_plus_s - s * s; - } - else + + switch( curve ) { - wlog( "Unknown reward fund type ${rf}", ("rf",rf.name) ); + case quadratic: + { + uint128_t rshares_plus_s = rshares + content_constant; + result = rshares_plus_s * rshares_plus_s - content_constant * content_constant; + } + break; + case linear: + result = rshares; + break; } return result; } -} } } +} } } // steemit::chain::util diff --git a/tests/tests/operation_time_tests.cpp b/tests/tests/operation_time_tests.cpp index e5b07520e4..fa0b58fcbd 100644 --- a/tests/tests/operation_time_tests.cpp +++ b/tests/tests/operation_time_tests.cpp @@ -318,7 +318,9 @@ BOOST_AUTO_TEST_CASE( recent_claims_decay ) tx.sign( alice_private_key, db.get_chain_id() ); db.push_transaction( tx, 0 ); - auto alice_vshares = util::calculate_claims( db.get_comment( "alice", string( "test" ) ).net_rshares.value, db.get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ) ); + auto alice_vshares = util::calculate_claims( db.get_comment( "alice", string( "test" ) ).net_rshares.value, + db.get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ).author_reward_curve, + db.get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ).content_constant ); generate_blocks( 5 ); @@ -341,7 +343,9 @@ BOOST_AUTO_TEST_CASE( recent_claims_decay ) } auto bob_cashout_time = db.get_comment( "bob", string( "test" ) ).cashout_time; - auto bob_vshares = util::calculate_claims( db.get_comment( "bob", string( "test" ) ).net_rshares.value, db.get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ) ); + auto bob_vshares = util::calculate_claims( db.get_comment( "bob", string( "test" ) ).net_rshares.value, + db.get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ).author_reward_curve, + db.get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ).content_constant ); generate_block(); From a00c012f7f82d30011c22d418e064bc1f22f3d53 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Mon, 24 Apr 2017 16:00:05 -0400 Subject: [PATCH 12/59] Create temporary reward fund for linear rewards #1051 --- libraries/chain/database.cpp | 31 ++++++++++++++++++- libraries/protocol/get_config.cpp | 4 ++- .../include/steemit/protocol/config.hpp | 4 ++- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 675c9a24f2..5795cbfd30 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -1619,7 +1619,11 @@ void database::process_comment_cashout() // Add all reward funds to the local cache and decay their recent rshares modify( *itr, [&]( reward_fund_object& rfo ) { - rfo.recent_claims -= ( rfo.recent_claims * ( head_block_time() - rfo.last_update ).to_seconds() ) / STEEMIT_RECENT_RSHARES_DECAY_RATE.to_seconds(); + if( rfo.name == STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME || has_hardfork( STEEMIT_HARDFORK_0_19__1051 ) ) + rfo.recent_claims -= ( rfo.recent_claims * ( head_block_time() - rfo.last_update ).to_seconds() ) / STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19.to_seconds(); + else + rfo.recent_claims -= ( rfo.recent_claims * ( head_block_time() - rfo.last_update ).to_seconds() ) / STEEMIT_RECENT_RSHARES_DECAY_RATE_HF17.to_seconds(); + rfo.last_update = head_block_time(); }); @@ -1640,12 +1644,15 @@ void database::process_comment_cashout() // add all rshares about to be cashed out to the reward funds. This ensures equal satoshi per rshare payment if( has_hardfork( STEEMIT_HARDFORK_0_17__771 ) ) { + const auto& linear = get< reward_fund_object, by_name >( STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME ); + while( current != cidx.end() && current->cashout_time <= head_block_time() ) { if( current->net_rshares > 0 ) { const auto& rf = get_reward_fund( *current ); funds[ rf.id._id ].recent_claims += util::calculate_claims( current->net_rshares.value, rf.author_reward_curve, rf.content_constant ); + funds[ 1 ].recent_claims += util::calculate_claims( current->net_rshares.value, linear.author_reward_curve, linear.content_constant ); } ++current; @@ -3766,9 +3773,23 @@ void database::apply_hardfork( uint32_t hardfork ) rfo.curation_reward_curve = curve_id::quadratic_curation; }); + auto linear_rf = create< reward_fund_object >( [&]( reward_fund_object& rfo ) + { + rfo.name = STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME; + rfo.last_update = head_block_time(); + rfo.content_constant = 0; + rfo.percent_curation_rewards = STEEMIT_1_PERCENT * 25; + rfo.percent_content_rewards = 0; + rfo.reward_balance = asset( 0, STEEM_SYMBOL ); + rfo.recent_claims = 0; + rfo.author_reward_curve = curve_id::linear; + rfo.curation_reward_curve = curve_id::square_root; + }); + // As a shortcut in payout processing, we use the id as an array index. // The IDs must be assigned this way. The assertion is a dummy check to ensure this happens. FC_ASSERT( post_rf.id._id == 0 ); + FC_ASSERT( linear_rf.id._id == 1 ); modify( gpo, [&]( dynamic_global_property_object& g ) { @@ -3830,6 +3851,14 @@ void database::apply_hardfork( uint32_t hardfork ) { gpo.vote_power_reserve_rate = 10; }); + + const auto& linear = get< reward_fund_object, by_name >( STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME ); + modify( get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ), [&]( reward_fund_object &rfo ) + { + rfo.recent_claims = linear.recent_claims; + rfo.author_reward_curve = curve_id::linear; + rfo.curation_reward_curve = curve_id::square_root; + }); } break; default: diff --git a/libraries/protocol/get_config.cpp b/libraries/protocol/get_config.cpp index a94a35ad47..c690f8d113 100644 --- a/libraries/protocol/get_config.cpp +++ b/libraries/protocol/get_config.cpp @@ -45,6 +45,7 @@ fc::variant_object get_config() result["STEEMIT_CASHOUT_WINDOW_SECONDS_PRE_HF17"] = STEEMIT_CASHOUT_WINDOW_SECONDS_PRE_HF17; result["STEEMIT_CHAIN_ID"] = STEEMIT_CHAIN_ID; result["STEEMIT_COMMENT_REWARD_FUND_NAME"] = STEEMIT_COMMENT_REWARD_FUND_NAME; + result["STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME"] = STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME; result["STEEMIT_CONTENT_APR_PERCENT"] = STEEMIT_CONTENT_APR_PERCENT; result["STEEMIT_CONTENT_CONSTANT_HF0"] = STEEMIT_CONTENT_CONSTANT_HF0; result["STEEMIT_CONTENT_REWARD_PERCENT"] = STEEMIT_CONTENT_REWARD_PERCENT; @@ -151,7 +152,8 @@ fc::variant_object get_config() result["STEEMIT_PROXY_TO_SELF_ACCOUNT"] = STEEMIT_PROXY_TO_SELF_ACCOUNT; result["STEEMIT_SBD_INTEREST_COMPOUND_INTERVAL_SEC"] = STEEMIT_SBD_INTEREST_COMPOUND_INTERVAL_SEC; result["STEEMIT_SECONDS_PER_YEAR"] = STEEMIT_SECONDS_PER_YEAR; - result["STEEMIT_RECENT_RSHARES_DECAY_RATE"] = STEEMIT_RECENT_RSHARES_DECAY_RATE; + result["STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19"] = STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19; + result["STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19"] = STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19; result["STEEMIT_REVERSE_AUCTION_WINDOW_SECONDS"] = STEEMIT_REVERSE_AUCTION_WINDOW_SECONDS; result["STEEMIT_ROOT_POST_PARENT"] = STEEMIT_ROOT_POST_PARENT; result["STEEMIT_SAVINGS_WITHDRAW_REQUEST_LIMIT"] = STEEMIT_SAVINGS_WITHDRAW_REQUEST_LIMIT; diff --git a/libraries/protocol/include/steemit/protocol/config.hpp b/libraries/protocol/include/steemit/protocol/config.hpp index 7fd008b73a..ab40a601e8 100644 --- a/libraries/protocol/include/steemit/protocol/config.hpp +++ b/libraries/protocol/include/steemit/protocol/config.hpp @@ -163,7 +163,9 @@ #define STEEMIT_POST_REWARD_FUND_NAME ("post") #define STEEMIT_COMMENT_REWARD_FUND_NAME ("comment") -#define STEEMIT_RECENT_RSHARES_DECAY_RATE (fc::days(30)) +#define STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME ("linear") +#define STEEMIT_RECENT_RSHARES_DECAY_RATE_HF17 (fc::days(30)) +#define STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19 (fc::days(15)) #define STEEMIT_CONTENT_CONSTANT_HF0 (uint128_t(uint64_t(2000000000000ll))) // note, if redefining these constants make sure calculate_claims doesn't overflow From 8313d977a14a69fa8278ad4a35bc0efc567fba13 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Tue, 25 Apr 2017 09:04:35 -0400 Subject: [PATCH 13/59] Fix unit tests --- tests/tests/operation_time_tests.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests/operation_time_tests.cpp b/tests/tests/operation_time_tests.cpp index fa0b58fcbd..4498ee9541 100644 --- a/tests/tests/operation_time_tests.cpp +++ b/tests/tests/operation_time_tests.cpp @@ -351,7 +351,7 @@ BOOST_AUTO_TEST_CASE( recent_claims_decay ) while( db.head_block_time() < bob_cashout_time ) { - alice_vshares -= ( alice_vshares * STEEMIT_BLOCK_INTERVAL ) / STEEMIT_RECENT_RSHARES_DECAY_RATE.to_seconds(); + alice_vshares -= ( alice_vshares * STEEMIT_BLOCK_INTERVAL ) / STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19.to_seconds(); const auto& post_rf = db.get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ); BOOST_REQUIRE( post_rf.recent_claims == alice_vshares ); @@ -361,7 +361,7 @@ BOOST_AUTO_TEST_CASE( recent_claims_decay ) } { - alice_vshares -= ( alice_vshares * STEEMIT_BLOCK_INTERVAL ) / STEEMIT_RECENT_RSHARES_DECAY_RATE.to_seconds(); + alice_vshares -= ( alice_vshares * STEEMIT_BLOCK_INTERVAL ) / STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19.to_seconds(); const auto& post_rf = db.get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ); BOOST_REQUIRE( post_rf.recent_claims == alice_vshares + bob_vshares ); From 68d38e4341c7d200a417066b69c8d85cfe5f73ed Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Tue, 25 Apr 2017 09:35:03 -0400 Subject: [PATCH 14/59] Fix get_config #1051 --- libraries/protocol/get_config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/protocol/get_config.cpp b/libraries/protocol/get_config.cpp index c690f8d113..bc1a04ddb6 100644 --- a/libraries/protocol/get_config.cpp +++ b/libraries/protocol/get_config.cpp @@ -153,7 +153,7 @@ fc::variant_object get_config() result["STEEMIT_SBD_INTEREST_COMPOUND_INTERVAL_SEC"] = STEEMIT_SBD_INTEREST_COMPOUND_INTERVAL_SEC; result["STEEMIT_SECONDS_PER_YEAR"] = STEEMIT_SECONDS_PER_YEAR; result["STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19"] = STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19; - result["STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19"] = STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19; + result["STEEMIT_RECENT_RSHARES_DECAY_RATE_HF17"] = STEEMIT_RECENT_RSHARES_DECAY_RATE_HF17; result["STEEMIT_REVERSE_AUCTION_WINDOW_SECONDS"] = STEEMIT_REVERSE_AUCTION_WINDOW_SECONDS; result["STEEMIT_ROOT_POST_PARENT"] = STEEMIT_ROOT_POST_PARENT; result["STEEMIT_SAVINGS_WITHDRAW_REQUEST_LIMIT"] = STEEMIT_SAVINGS_WITHDRAW_REQUEST_LIMIT; From 094f819442c8650f9d3783b6a1be2e7ade22a911 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Wed, 26 Apr 2017 09:43:22 -0400 Subject: [PATCH 15/59] Refactor curve calculations in util #1051 --- libraries/app/database_api.cpp | 4 +- libraries/chain/database.cpp | 10 ++-- .../include/steemit/chain/util/reward.hpp | 4 +- libraries/chain/steem_evaluator.cpp | 12 ++-- libraries/chain/util/reward.cpp | 55 ++++--------------- tests/tests/operation_tests.cpp | 2 +- tests/tests/operation_time_tests.cpp | 4 +- 7 files changed, 27 insertions(+), 64 deletions(-) diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp index 50ebe2f5c1..b150e9fb85 100755 --- a/libraries/app/database_api.cpp +++ b/libraries/app/database_api.cpp @@ -1142,10 +1142,10 @@ void database_api::set_pending_payout( discussion& d )const if( my->_db.has_hardfork( STEEMIT_HARDFORK_0_17__774 ) ) { const auto& rf = my->_db.get_reward_fund( my->_db.get_comment( d.author, d.permlink ) ); - vshares = d.net_rshares.value > 0 ? steemit::chain::util::calculate_claims( d.net_rshares.value, rf.author_reward_curve, rf.content_constant ) : 0; + vshares = d.net_rshares.value > 0 ? steemit::chain::util::evaluate_reward_curve( d.net_rshares.value, rf.author_reward_curve, rf.content_constant ) : 0; } else - vshares = steemit::chain::util::calculate_claims( d.net_rshares.value > 0 ? d.net_rshares.value : 0 ); + vshares = steemit::chain::util::evaluate_reward_curve( d.net_rshares.value > 0 ? d.net_rshares.value : 0 ); u256 r2 = to256(vshares); //to256(abs_net_rshares); r2 *= pot.amount.value; diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 5795cbfd30..abc447327e 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -1538,7 +1538,7 @@ share_type database::cashout_comment_helper( util::comment_reward_context& ctx, } if( !has_hardfork( STEEMIT_HARDFORK_0_17__774 ) ) - adjust_rshares2( comment, util::calculate_claims( comment.net_rshares.value ), 0 ); + adjust_rshares2( comment, util::evaluate_reward_curve( comment.net_rshares.value ), 0 ); } modify( comment, [&]( comment_object& c ) @@ -1651,8 +1651,8 @@ void database::process_comment_cashout() if( current->net_rshares > 0 ) { const auto& rf = get_reward_fund( *current ); - funds[ rf.id._id ].recent_claims += util::calculate_claims( current->net_rshares.value, rf.author_reward_curve, rf.content_constant ); - funds[ 1 ].recent_claims += util::calculate_claims( current->net_rshares.value, linear.author_reward_curve, linear.content_constant ); + funds[ rf.id._id ].recent_claims += util::evaluate_reward_curve( current->net_rshares.value, rf.author_reward_curve, rf.content_constant ); + funds[ 1 ].recent_claims += util::evaluate_reward_curve( current->net_rshares.value, linear.author_reward_curve, linear.content_constant ); } ++current; @@ -3986,7 +3986,7 @@ void database::validate_invariants()const { if( itr->net_rshares.value > 0 ) { - auto delta = util::calculate_claims( itr->net_rshares.value ); + auto delta = util::evaluate_reward_curve( itr->net_rshares.value ); total_rshares2 += delta; } } @@ -4057,7 +4057,7 @@ void database::perform_vesting_share_split( uint32_t magnitude ) for( const auto& c : comments ) { if( c.net_rshares.value > 0 ) - adjust_rshares2( c, 0, util::calculate_claims( c.net_rshares.value ) ); + adjust_rshares2( c, 0, util::evaluate_reward_curve( c.net_rshares.value ) ); } } diff --git a/libraries/chain/include/steemit/chain/util/reward.hpp b/libraries/chain/include/steemit/chain/util/reward.hpp index fe9b8f321a..a96e18ef79 100644 --- a/libraries/chain/include/steemit/chain/util/reward.hpp +++ b/libraries/chain/include/steemit/chain/util/reward.hpp @@ -33,14 +33,12 @@ struct comment_reward_context uint64_t get_rshare_reward( const comment_reward_context& ctx ); -uint64_t get_vote_weight( uint64_t vote_rshares, const curve_id& curve, const uint128_t& content_constant ); - inline uint128_t get_content_constant_s() { return STEEMIT_CONTENT_CONSTANT_HF0; // looking good for posters } -uint128_t calculate_claims( const uint128_t& rshares, const curve_id& curve = quadratic, const uint128_t& content_constant = STEEMIT_CONTENT_CONSTANT_HF0 ); +uint128_t evaluate_reward_curve( const uint128_t& rshares, const curve_id& curve = quadratic, const uint128_t& content_constant = STEEMIT_CONTENT_CONSTANT_HF0 ); inline bool is_comment_payout_dust( const price& p, uint64_t steem_payout ) { diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index db13b120f8..179c45e4bd 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -1290,8 +1290,8 @@ void vote_evaluator::do_apply( const vote_operation& o ) fc::uint128_t new_rshares = std::max( comment.net_rshares.value, int64_t(0)); /// calculate rshares2 value - new_rshares = util::calculate_claims( new_rshares ); - old_rshares = util::calculate_claims( old_rshares ); + new_rshares = util::evaluate_reward_curve( new_rshares ); + old_rshares = util::evaluate_reward_curve( old_rshares ); uint64_t max_vote_weight = 0; @@ -1346,8 +1346,8 @@ void vote_evaluator::do_apply( const vote_operation& o ) if( _db.has_hardfork( STEEMIT_HARDFORK_0_17__774 ) ) { const auto& reward_fund = _db.get_reward_fund( comment ); - uint64_t old_weight = util::get_vote_weight( old_vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ); - uint64_t new_weight = util::get_vote_weight( comment.vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ); + uint64_t old_weight = util::evaluate_reward_curve( old_vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ).to_uint64(); + uint64_t new_weight = util::evaluate_reward_curve( comment.vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ).to_uint64(); cv.weight = new_weight - old_weight; } else if ( _db.has_hardfork( STEEMIT_HARDFORK_0_1 ) ) @@ -1479,8 +1479,8 @@ void vote_evaluator::do_apply( const vote_operation& o ) fc::uint128_t new_rshares = std::max( comment.net_rshares.value, int64_t(0)); /// calculate rshares2 value - new_rshares = util::calculate_claims( new_rshares ); - old_rshares = util::calculate_claims( old_rshares ); + new_rshares = util::evaluate_reward_curve( new_rshares ); + old_rshares = util::evaluate_reward_curve( old_rshares ); _db.modify( comment, [&]( comment_object& c ) diff --git a/libraries/chain/util/reward.cpp b/libraries/chain/util/reward.cpp index 1743bee38f..3d7670cd8e 100644 --- a/libraries/chain/util/reward.cpp +++ b/libraries/chain/util/reward.cpp @@ -16,7 +16,7 @@ uint64_t get_rshare_reward( const comment_reward_context& ctx ) //idump( (ctx) ); - u256 claim = to256( calculate_claims( ctx.rshares.value, ctx.reward_curve, ctx.content_constant ) ); + u256 claim = to256( evaluate_reward_curve( ctx.rshares.value, ctx.reward_curve, ctx.content_constant ) ); claim = ( claim * ctx.reward_weight ) / STEEMIT_100_PERCENT; u256 payout_u256 = ( rf * claim ) / total_claims; @@ -34,50 +34,7 @@ uint64_t get_rshare_reward( const comment_reward_context& ctx ) } FC_CAPTURE_AND_RETHROW( (ctx) ) } -uint64_t get_vote_weight( uint64_t vote_rshares, const reward_fund_object& rf ) -{ - uint64_t result = 0; - if( rf.name == STEEMIT_POST_REWARD_FUND_NAME || rf.name == STEEMIT_COMMENT_REWARD_FUND_NAME ) - { - uint128_t two_alpha = rf.content_constant * 2; - result = ( uint128_t( vote_rshares, 0 ) / ( two_alpha + vote_rshares ) ).to_uint64(); - } - else - { - wlog( "Unknown reward fund type ${rf}", ("rf",rf.name) ); - } - - return result; -} - -uint64_t get_vote_weight( uint64_t vote_rshares, const curve_id& curve, const uint128_t& content_constant ) -{ - uint64_t result = 0; - - switch( curve ) - { - case quadratic_curation: - { - uint128_t two_alpha = content_constant * 2; - result = ( uint128_t( vote_rshares, 0 ) / ( two_alpha + vote_rshares ) ).to_uint64(); - } - break; - case square_root: - //result = rshares; - break; - } - - return result; -} - -uint128_t calculate_claims( const uint128_t& rshares ) -{ - uint128_t s = get_content_constant_s(); - uint128_t rshares_plus_s = rshares + s; - return rshares_plus_s * rshares_plus_s - s * s; -} - -uint128_t calculate_claims( const uint128_t& rshares, const curve_id& curve, const uint128_t& content_constant ) +uint128_t evaluate_reward_curve( const uint128_t& rshares, const curve_id& curve, const uint128_t& content_constant ) { uint128_t result = 0; @@ -89,9 +46,17 @@ uint128_t calculate_claims( const uint128_t& rshares, const curve_id& curve, con result = rshares_plus_s * rshares_plus_s - content_constant * content_constant; } break; + case quadratic_curation: + { + uint128_t two_alpha = content_constant * 2; + result = uint128_t( rshares.lo, 0 ) / ( two_alpha + rshares ); + } + break; case linear: result = rshares; break; + case square_root: + break; } return result; diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp index cbc0f4b5f7..5b0b560857 100644 --- a/tests/tests/operation_tests.cpp +++ b/tests/tests/operation_tests.cpp @@ -548,7 +548,7 @@ BOOST_AUTO_TEST_CASE( comment_apply ) db.modify( db.get_dynamic_global_properties(), [&]( dynamic_global_property_object& o) { - o.total_reward_shares2 = steemit::chain::util::calculate_claims( 10 ); + o.total_reward_shares2 = steemit::chain::util::evaluate_reward_curve( 10 ); }); tx.signatures.clear(); diff --git a/tests/tests/operation_time_tests.cpp b/tests/tests/operation_time_tests.cpp index 4498ee9541..3f7b5df69e 100644 --- a/tests/tests/operation_time_tests.cpp +++ b/tests/tests/operation_time_tests.cpp @@ -318,7 +318,7 @@ BOOST_AUTO_TEST_CASE( recent_claims_decay ) tx.sign( alice_private_key, db.get_chain_id() ); db.push_transaction( tx, 0 ); - auto alice_vshares = util::calculate_claims( db.get_comment( "alice", string( "test" ) ).net_rshares.value, + auto alice_vshares = util::evaluate_reward_curve( db.get_comment( "alice", string( "test" ) ).net_rshares.value, db.get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ).author_reward_curve, db.get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ).content_constant ); @@ -343,7 +343,7 @@ BOOST_AUTO_TEST_CASE( recent_claims_decay ) } auto bob_cashout_time = db.get_comment( "bob", string( "test" ) ).cashout_time; - auto bob_vshares = util::calculate_claims( db.get_comment( "bob", string( "test" ) ).net_rshares.value, + auto bob_vshares = util::evaluate_reward_curve( db.get_comment( "bob", string( "test" ) ).net_rshares.value, db.get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ).author_reward_curve, db.get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ).content_constant ); From 5c787c5baa651858658caa2bd47473846b099c0a Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Thu, 27 Apr 2017 14:37:43 -0400 Subject: [PATCH 16/59] Implement approximate integer sqrt function and documentation #1052 --- doc/sqrt.md | 128 +++++++++++++++++++++++++++++++++++ programs/util/CMakeLists.txt | 10 +++ programs/util/test_sqrt.cpp | 71 +++++++++++++++++++ programs/util/test_sqrt.py | 28 ++++++++ 4 files changed, 237 insertions(+) create mode 100644 doc/sqrt.md create mode 100644 programs/util/test_sqrt.cpp create mode 100755 programs/util/test_sqrt.py diff --git a/doc/sqrt.md b/doc/sqrt.md new file mode 100644 index 0000000000..3c09242d07 --- /dev/null +++ b/doc/sqrt.md @@ -0,0 +1,128 @@ + +# Introduction + +In this document we derive the approximate integer square root function used by Steem for the curation curve +[here](https://github.com/steemit/steem/issues/1052). + +# MSB function + +Let function `msb(x)` be defined as follows for `x >= 1`: + +- (Definition 1a) `msb(x)` is the index of the most-significant 1 bit in the binary representation of `x` + +The following definitions are equivalent to Definition (1a): + +- (Definition 1b) `msb(x)` is the length of the binary representation of `x` as a string of bits, minus one +- (Definition 1c) `msb(x)` is the greatest integer such that `2 ^ msb(x) <= x` +- (Definition 1d) `msb(x) = floor(log_2(x))` + +Many CPU's (including Intel CPU's since the Intel 386) can compute the `msb()` function very quickly on +machine-word-size integers with a special instruction directly implemented in hardware. In C++, the +Boost library provides reasonably compiler-independent, hardware-independent access to this +functionality with `boost::multiprecision::detail::find_msb(x)`. + +# Approximate logarithms + +According to Definition 1d, `msb(x)` is already a (somewhat crude) approximate base-2 logarithm. The +bits below the most-significant bit provide the fractional part of the linear interpolation. The +fractional part is called the *mantissa* and the integer part is called the *exponent*; effectively we +have re-invented the floating-point representation. + +Here are some Python functions to convert to/from these approximate logarithms: + +``` +def to_log(x, wordsize=32, ebits=5): + if x <= 1: + return x + # mantissa_bits, mantissa_mask are independent of x + mantissa_bits = wordsize - ebits + mantissa_mask = (1 << mantissa_bits)-1 + + msb = x.bit_length() - 1 + mantissa_shift = mantissa_bits - msb + y = (msb << mantissa_bits) | ((x << mantissa_shift) & mantissa_mask) + return y + +def from_log(y, wordsize=32, ebits=5): + if y <= 1: + return y + # mantissa_bits, leading_1, mantissa_mask are independent of x + mantissa_bits = wordsize - ebits + leading_1 = 1 << mantissa_bits + mantissa_mask = leading_1 - 1 + + msb = y >> mantissa_bits + mantissa_shift = mantissa_bits - msb + y = (leading_1 | (y & mantissa_mask)) >> mantissa_shift + return y +``` + +# Approximate square roots + +To construct an approximate square root algorithm, start from the identity `log(sqrt(x)) = log(x) / 2`. +We can easily obtain `sqrt(x) ~ from_log(to_log(x) >> 1)`. We can proceed by manual inlining the inner +function call: + +``` +def approx_sqrt_v0(x, wordsize=32, ebits=5): + if x <= 1: + return x + # mantissa_bits, leading_1, mantissa_mask are independent of x + mantissa_bits = wordsize - ebits + leading_1 = 1 << mantissa_bits + mantissa_mask = leading_1 - 1 + + msb_x = x.bit_length() - 1 + mantissa_shift_x = mantissa_bits - msb_x + to_log_x = (msb_x << mantissa_bits) | ((x << mantissa_shift_x) & mantissa_mask) + + z = to_log_x >> 1 + + msb_z = z >> mantissa_bits + mantissa_shift_z = mantissa_bits - msb_z + result = (leading_1 | (z & mantissa_mask)) >> mantissa_shift_z + return result +``` + +# Optimized approximate square roots + +First, consider the following simplifications: + +- The exponent part of `z`, denoted here `msb_z`, is simply `msb_x >> 1` +- The MSB of the mantissa part of `z` is the low bit of `msb_x` +- The lower bits of the mantissa part of `z` are simply the bits of the mantissa part of `x` shifted once + +The above simplifications enable a more fundamental improvement: We can compute +the mantissa and exponent of `z` directly from `x` and `msb_x`. Therefore, packing +the intermediate result into `to_log_x` and then immediately unpacking it, effectively +becomes a no-op and can be omitted. This makes the `wordsize` and `ebits` variables fall +out. Making choices for these parameters and allocating extra space at the top of the word +for exponent bits becomes completely unnecessary! + +One subtlety is that the two shift operators result in a net shift of mantissa bits. The +are shifted left by `mantissa_bits - msb_x` and then shifted right by `mantissa_bits - msb_z`. The +net shift is therefore a right-shift of `msb_x - msb_z`. + +The final code looks like this: + +``` +def approx_sqrt_v1(x): + if x <= 1: + return x + # mantissa_bits, leading_1, mantissa_mask are independent of x + msb_x = x.bit_length() - 1 + msb_z = msb_x >> 1 + msb_x_bit = 1 << msb_x + msb_z_bit = 1 << msb_z + mantissa_mask = msb_x_bit-1 + + mantissa_x = x & mantissa_mask + if (msb_x & 1) != 0: + mantissa_z_hi = msb_z_bit + else: + mantissa_z_hi = 0 + mantissa_z_lo = mantissa_x >> (msb_x - msb_z) + mantissa_z = (mantissa_z_hi | mantissa_z_lo) >> 1 + result = msb_z_bit | mantissa_z + return result +``` diff --git a/programs/util/CMakeLists.txt b/programs/util/CMakeLists.txt index 50ec4962a0..58341a3c04 100644 --- a/programs/util/CMakeLists.txt +++ b/programs/util/CMakeLists.txt @@ -90,3 +90,13 @@ install( TARGETS LIBRARY DESTINATION lib ARCHIVE DESTINATION lib ) + +add_executable( test_sqrt test_sqrt.cpp ) +target_link_libraries( test_sqrt PRIVATE fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) +install( TARGETS + test_sqrt + + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) diff --git a/programs/util/test_sqrt.cpp b/programs/util/test_sqrt.cpp new file mode 100644 index 0000000000..c8d3360f7a --- /dev/null +++ b/programs/util/test_sqrt.cpp @@ -0,0 +1,71 @@ + +#include +#include +#include +#include + +#include + +#include + +using fc::uint128_t; + +uint8_t find_msb( const uint128_t& u ) +{ + uint64_t x; + uint8_t places; + x = (u.lo ? u.lo : 1); + places = (u.hi ? 64 : 0); + x = (u.hi ? u.hi : x); + return uint8_t( boost::multiprecision::detail::find_msb(x) + places ); +} + +uint64_t maybe_sqrt( uint64_t x ) +{ + if( x <= 1 ) + return 0; + assert( x <= std::numeric_limits< uint64_t >::max()/2 ); + + uint8_t n = uint8_t( boost::multiprecision::detail::find_msb(x) ); + uint8_t b = n&1; + uint64_t y = x + (uint64_t(1) << (n-(1-b))); + y = y >> ((n >> 1)+b+1); + return y; +} + +uint64_t approx_sqrt( const uint128_t& x ) +{ + if( (x.lo == 0) && (x.hi == 0) ) + return 0; + + uint8_t msb_x = find_msb(x); + uint8_t msb_z = msb_x >> 1; + + uint128_t msb_x_bit = uint128_t(1) << msb_x; + uint64_t msb_z_bit = uint64_t (1) << msb_z; + + uint128_t mantissa_mask = msb_x_bit - 1; + uint128_t mantissa_x = x & mantissa_mask; + uint64_t mantissa_z_hi = (msb_x & 1) ? msb_z_bit : 0; + uint64_t mantissa_z_lo = (mantissa_x >> (msb_x - msb_z)).lo; + uint64_t mantissa_z = (mantissa_z_hi | mantissa_z_lo) >> 1; + uint64_t result = msb_z_bit | mantissa_z; + + return result; +} + +int main( int argc, char** argv, char** envp ) +{ + uint128_t x = 0; + while( true ) + { + uint64_t y = approx_sqrt(x); + std::cout << std::string(x) << " " << y << std::endl; + //uint128_t new_x = x + (x >> 10) + 1; + uint128_t new_x = x + (x / 1000) + 1; + if( new_x < x ) + break; + x = new_x; + } + return 0; +} diff --git a/programs/util/test_sqrt.py b/programs/util/test_sqrt.py new file mode 100755 index 0000000000..ea48846f35 --- /dev/null +++ b/programs/util/test_sqrt.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +def approx_sqrt_v1(x): + if x <= 1: + return x + # mantissa_bits, leading_1, mantissa_mask are independent of x + msb_x = x.bit_length() - 1 + msb_z = msb_x >> 1 + msb_x_bit = 1 << msb_x + msb_z_bit = 1 << msb_z + mantissa_mask = msb_x_bit-1 + + mantissa_x = x & mantissa_mask + if (msb_x & 1) != 0: + mantissa_z_hi = msb_z_bit + else: + mantissa_z_hi = 0 + mantissa_z_lo = mantissa_x >> (msb_x - msb_z) + mantissa_z = (mantissa_z_hi | mantissa_z_lo) >> 1 + result = msb_z_bit | mantissa_z + return result + +x = 0 + +while x < 2**128: + print(x, approx_sqrt_v1(x)) + x += (x // 1000) + 1 + From 2c3ceda85edbe5472f73ef7f9ca795d88365a353 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 28 Apr 2017 16:47:58 -0400 Subject: [PATCH 17/59] Implement square root curation curve #1052 --- libraries/chain/database.cpp | 18 ++++++++++ libraries/chain/hardfork.d/0_19.hf | 1 + .../include/steemit/chain/comment_object.hpp | 6 ++-- libraries/chain/steem_evaluator.cpp | 17 ++++++++++ libraries/chain/util/reward.cpp | 33 ++++++++++++++++++- 5 files changed, 72 insertions(+), 3 deletions(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index abc447327e..2226f3fbc8 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -3859,6 +3859,24 @@ void database::apply_hardfork( uint32_t hardfork ) rfo.author_reward_curve = curve_id::linear; rfo.curation_reward_curve = curve_id::square_root; }); + + const auto& cidx = get_index< comment_index, by_cashout_time >(); + const auto& vidx = get_index< comment_vote_index, by_comment_voter >(); + for( auto c_itr = cidx.begin(); c_itr != cidx.end() && c_itr->cashout_time != fc::time_point_sec::maximum(); ++c_itr ) + { + modify( *c_itr, [&]( comment_object& c ) + { + c.total_vote_weight = c.total_sqrt_vote_weight; + }); + + for( auto v_itr = vidx.lower_bound( c_itr->id ); v_itr != vidx.end() && v_itr->comment == c_itr->id; ++v_itr ) + { + modify( *v_itr, [&]( comment_vote_object& v ) + { + v.weight = v.sqrt_weight; + }); + } + } } break; default: diff --git a/libraries/chain/hardfork.d/0_19.hf b/libraries/chain/hardfork.d/0_19.hf index 31e645350c..74177e1d6a 100644 --- a/libraries/chain/hardfork.d/0_19.hf +++ b/libraries/chain/hardfork.d/0_19.hf @@ -10,6 +10,7 @@ #define STEEMIT_HARDFORK_0_19__987 (STEEMIT_HARDFORK_0_19) #define STEEMIT_HARDFORK_0_19__997 (STEEMIT_HARDFORK_0_19) #define STEEMIT_HARDFORK_0_19__1051 (STEEMIT_HARDFORK_0_19) +#define STEEMIT_HARDFORK_0_19__1052 (STEEMIT_HARDFORK_0_19) #define STEEMIT_HARDFORK_0_19__1053 (STEEMIT_HARDFORK_0_19) #define STEEMIT_HARDFORK_0_19_VERSION hardfork_version( 0, 19 ) diff --git a/libraries/chain/include/steemit/chain/comment_object.hpp b/libraries/chain/include/steemit/chain/comment_object.hpp index 5b3f3c9bf3..02f284741c 100644 --- a/libraries/chain/include/steemit/chain/comment_object.hpp +++ b/libraries/chain/include/steemit/chain/comment_object.hpp @@ -78,6 +78,7 @@ namespace steemit { namespace chain { time_point_sec cashout_time; /// 24 hours from the weighted average of vote time time_point_sec max_cashout_time; uint64_t total_vote_weight = 0; /// the total weight of voting rewards, used to calculate pro-rata share of curation payouts + uint64_t total_sqrt_vote_weight = 0; /// Temporary total weight of votes for post hf19 curation reward curve uint16_t reward_weight = 0; @@ -120,6 +121,7 @@ namespace steemit { namespace chain { account_id_type voter; comment_id_type comment; uint64_t weight = 0; ///< defines the score this vote receives, used by vote payout calc. 0 if a negative vote or changed votes. + uint64_t sqrt_weight = 0; ///< Temporary vote weight for post hf19 curation curve int64_t rshares = 0; ///< The number of rshares this vote is responsible for int16_t vote_percent = 0; ///< The percent weight of the vote time_point_sec last_update; ///< The time of the last update of the vote @@ -250,13 +252,13 @@ FC_REFLECT( steemit::chain::comment_object, (depth)(children) (net_rshares)(abs_rshares)(vote_rshares) (children_abs_rshares)(cashout_time)(max_cashout_time) - (total_vote_weight)(reward_weight)(total_payout_value)(curator_payout_value)(beneficiary_payout_value)(author_rewards)(net_votes)(root_comment) + (total_vote_weight)(total_sqrt_vote_weight)(reward_weight)(total_payout_value)(curator_payout_value)(beneficiary_payout_value)(author_rewards)(net_votes)(root_comment) (max_accepted_payout)(percent_steem_dollars)(allow_replies)(allow_votes)(allow_curation_rewards) (beneficiaries) ) CHAINBASE_SET_INDEX_TYPE( steemit::chain::comment_object, steemit::chain::comment_index ) FC_REFLECT( steemit::chain::comment_vote_object, - (id)(voter)(comment)(weight)(rshares)(vote_percent)(last_update)(num_changes) + (id)(voter)(comment)(weight)(sqrt_weight)(rshares)(vote_percent)(last_update)(num_changes) ) CHAINBASE_SET_INDEX_TYPE( steemit::chain::comment_vote_object, steemit::chain::comment_vote_index ) diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 179c45e4bd..059c78730b 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -1294,6 +1294,7 @@ void vote_evaluator::do_apply( const vote_operation& o ) old_rshares = util::evaluate_reward_curve( old_rshares ); uint64_t max_vote_weight = 0; + uint64_t sqrt_max_vote_weight = 0; /** this verifies uniqueness of voter * @@ -1349,6 +1350,13 @@ void vote_evaluator::do_apply( const vote_operation& o ) uint64_t old_weight = util::evaluate_reward_curve( old_vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ).to_uint64(); uint64_t new_weight = util::evaluate_reward_curve( comment.vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ).to_uint64(); cv.weight = new_weight - old_weight; + if( !_db.has_hardfork( STEEMIT_HARDFORK_0_19__1052 ) ) + { + old_weight = util::evaluate_reward_curve( old_vote_rshares.value, curve_id::square_root ).to_uint64(); + new_weight = util::evaluate_reward_curve( comment.vote_rshares.value, curve_id::square_root ).to_uint64(); + cv.sqrt_weight = new_weight - old_weight; + sqrt_max_vote_weight = cv.sqrt_weight; + } } else if ( _db.has_hardfork( STEEMIT_HARDFORK_0_1 ) ) { @@ -1375,6 +1383,14 @@ void vote_evaluator::do_apply( const vote_operation& o ) w *= delta_t; w /= STEEMIT_REVERSE_AUCTION_WINDOW_SECONDS; cv.weight = w.to_uint64(); + + if( _db.has_hardfork( STEEMIT_HARDFORK_0_17 ) ) + { + uint128_t w(sqrt_max_vote_weight); + w *= delta_t; + w /= STEEMIT_REVERSE_AUCTION_WINDOW_SECONDS; + cv.sqrt_weight = w.to_uint64(); + } } } else @@ -1388,6 +1404,7 @@ void vote_evaluator::do_apply( const vote_operation& o ) _db.modify( comment, [&]( comment_object& c ) { c.total_vote_weight += max_vote_weight; + c.total_sqrt_vote_weight = sqrt_max_vote_weight; }); } if( !_db.has_hardfork( STEEMIT_HARDFORK_0_17__774) ) diff --git a/libraries/chain/util/reward.cpp b/libraries/chain/util/reward.cpp index 3d7670cd8e..750bf9fc81 100644 --- a/libraries/chain/util/reward.cpp +++ b/libraries/chain/util/reward.cpp @@ -4,6 +4,37 @@ namespace steemit { namespace chain { namespace util { +uint8_t find_msb( const uint128_t& u ) +{ + uint64_t x; + uint8_t places; + x = (u.lo ? u.lo : 1); + places = (u.hi ? 64 : 0); + x = (u.hi ? u.hi : x); + return uint8_t( boost::multiprecision::detail::find_msb(x) + places ); +} + +uint64_t approx_sqrt( const uint128_t& x ) +{ + if( (x.lo == 0) && (x.hi == 0) ) + return 0; + + uint8_t msb_x = find_msb(x); + uint8_t msb_z = msb_x >> 1; + + uint128_t msb_x_bit = uint128_t(1) << msb_x; + uint64_t msb_z_bit = uint64_t (1) << msb_z; + + uint128_t mantissa_mask = msb_x_bit - 1; + uint128_t mantissa_x = x & mantissa_mask; + uint64_t mantissa_z_hi = (msb_x & 1) ? msb_z_bit : 0; + uint64_t mantissa_z_lo = (mantissa_x >> (msb_x - msb_z)).lo; + uint64_t mantissa_z = (mantissa_z_hi | mantissa_z_lo) >> 1; + uint64_t result = msb_z_bit | mantissa_z; + + return result; +} + uint64_t get_rshare_reward( const comment_reward_context& ctx ) { try @@ -53,7 +84,7 @@ uint128_t evaluate_reward_curve( const uint128_t& rshares, const curve_id& curve } break; case linear: - result = rshares; + result = approx_sqrt( rshares ); break; case square_root: break; From d6e229dfe3d87cd11bd5d22463c12e86746b0a65 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Thu, 18 May 2017 12:09:18 -0400 Subject: [PATCH 18/59] Clean up reward fund decay and temple linear reward fund calculations #1051 --- libraries/chain/database.cpp | 10 +++++++--- libraries/protocol/get_config.cpp | 1 + libraries/protocol/include/steemit/protocol/config.hpp | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 2226f3fbc8..a65c05a3fd 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -1619,11 +1619,15 @@ void database::process_comment_cashout() // Add all reward funds to the local cache and decay their recent rshares modify( *itr, [&]( reward_fund_object& rfo ) { + fc::microseconds decay_rate; + + // TODO: Remove temp fund after HF 19 if( rfo.name == STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME || has_hardfork( STEEMIT_HARDFORK_0_19__1051 ) ) - rfo.recent_claims -= ( rfo.recent_claims * ( head_block_time() - rfo.last_update ).to_seconds() ) / STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19.to_seconds(); + decay_rate = STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19; else - rfo.recent_claims -= ( rfo.recent_claims * ( head_block_time() - rfo.last_update ).to_seconds() ) / STEEMIT_RECENT_RSHARES_DECAY_RATE_HF17.to_seconds(); + decay_rate = STEEMIT_RECENT_RSHARES_DECAY_RATE_HF17; + rfo.recent_claims -= ( rfo.recent_claims * ( head_block_time() - rfo.last_update ).to_seconds() ) / decay_rate.to_seconds(); rfo.last_update = head_block_time(); }); @@ -1652,7 +1656,7 @@ void database::process_comment_cashout() { const auto& rf = get_reward_fund( *current ); funds[ rf.id._id ].recent_claims += util::evaluate_reward_curve( current->net_rshares.value, rf.author_reward_curve, rf.content_constant ); - funds[ 1 ].recent_claims += util::evaluate_reward_curve( current->net_rshares.value, linear.author_reward_curve, linear.content_constant ); + funds[ STEEMIT_TEMP_LINEAR_REWARD_FUND_ID ].recent_claims += util::evaluate_reward_curve( current->net_rshares.value, linear.author_reward_curve, linear.content_constant ); } ++current; diff --git a/libraries/protocol/get_config.cpp b/libraries/protocol/get_config.cpp index bc1a04ddb6..faea7688bd 100644 --- a/libraries/protocol/get_config.cpp +++ b/libraries/protocol/get_config.cpp @@ -46,6 +46,7 @@ fc::variant_object get_config() result["STEEMIT_CHAIN_ID"] = STEEMIT_CHAIN_ID; result["STEEMIT_COMMENT_REWARD_FUND_NAME"] = STEEMIT_COMMENT_REWARD_FUND_NAME; result["STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME"] = STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME; + result["STEEMIT_TEMP_LINEAR_REWARD_FUND_ID"] = STEEMIT_TEMP_LINEAR_REWARD_FUND_ID; result["STEEMIT_CONTENT_APR_PERCENT"] = STEEMIT_CONTENT_APR_PERCENT; result["STEEMIT_CONTENT_CONSTANT_HF0"] = STEEMIT_CONTENT_CONSTANT_HF0; result["STEEMIT_CONTENT_REWARD_PERCENT"] = STEEMIT_CONTENT_REWARD_PERCENT; diff --git a/libraries/protocol/include/steemit/protocol/config.hpp b/libraries/protocol/include/steemit/protocol/config.hpp index ab40a601e8..5336f4a922 100644 --- a/libraries/protocol/include/steemit/protocol/config.hpp +++ b/libraries/protocol/include/steemit/protocol/config.hpp @@ -164,6 +164,7 @@ #define STEEMIT_POST_REWARD_FUND_NAME ("post") #define STEEMIT_COMMENT_REWARD_FUND_NAME ("comment") #define STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME ("linear") +#define STEEMIT_TEMP_LINEAR_REWARD_FUND_ID (1) #define STEEMIT_RECENT_RSHARES_DECAY_RATE_HF17 (fc::days(30)) #define STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19 (fc::days(15)) #define STEEMIT_CONTENT_CONSTANT_HF0 (uint128_t(uint64_t(2000000000000ll))) From b3421c1aee6a506947b212a2cd3e32237eaf81dc Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 21 Apr 2017 09:19:16 -0400 Subject: [PATCH 19/59] Delete 0 VESTS delegations and prevent their creation in the future. #997 --- libraries/chain/database.cpp | 17 +++++++++++++++++ libraries/chain/steem_evaluator.cpp | 21 ++++++++++++++------- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index a65c05a3fd..151a89fc6d 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -3881,6 +3881,23 @@ void database::apply_hardfork( uint32_t hardfork ) }); } } + + vector< const vesting_delegation_object* > to_remove; + const auto& delegation_idx = get_index< vesting_delegation_index, by_id >(); + auto delegation_itr = delegation_idx.begin(); + + while( delegation_itr != delegation_idx.end() ) + { + if( delegation_itr->vesting_shares.amount == 0 ) + to_remove.push_back( &(*delegation_itr) ); + + ++delegation_itr; + } + + for( const vesting_delegation_object* delegation_ptr: to_remove ) + { + remove( *delegation_ptr ); + } } break; default: diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 4f02074667..4d7be93ac5 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -271,13 +271,20 @@ void account_create_with_delegation_evaluator::do_apply( const account_create_wi auth.last_owner_update = fc::time_point_sec::min(); }); - _db.create< vesting_delegation_object >( [&]( vesting_delegation_object& vdo ) - { - vdo.delegator = o.creator; - vdo.delegatee = o.new_account_name; - vdo.vesting_shares = o.delegation; - vdo.min_delegation_time = _db.head_block_time() + STEEMIT_CREATE_ACCOUNT_DELEGATION_TIME; - }); + /* TODO: Check if not creating 0 delegation objects in HF19 passes consensus + * If it does we can remove the apply_hardfork logic for deleting 0 delegation objects + */ + if( ( _db.has_hardfork( STEEMIT_HARDFORK_0_19__997 ) && o.delegation.amount > 0 ) + || !_db.has_hardfork( STEEMIT_HARDFORK_0_19__997 ) ) + { + _db.create< vesting_delegation_object >( [&]( vesting_delegation_object& vdo ) + { + vdo.delegator = o.creator; + vdo.delegatee = o.new_account_name; + vdo.vesting_shares = o.delegation; + vdo.min_delegation_time = _db.head_block_time() + STEEMIT_CREATE_ACCOUNT_DELEGATION_TIME; + }); + } if( o.fee.amount > 0 ) _db.create_vesting( new_account, o.fee ); From 906aaa0bc4ad8633fcf36b606b7ac4a5ffd022f6 Mon Sep 17 00:00:00 2001 From: Gandalf Date: Sun, 21 May 2017 14:14:55 +0200 Subject: [PATCH 20/59] removed entries for obsolete nodes (off for > 1 week) --- doc/seednodes.txt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/doc/seednodes.txt b/doc/seednodes.txt index f244172b2a..8f9866e34f 100644 --- a/doc/seednodes.txt +++ b/doc/seednodes.txt @@ -1,15 +1,12 @@ -#####seed.riversteem.com:2001 # riverhead steem-seed1.abit-more.com:2001 # abit 52.74.152.79:2001 # smooth.witness seed.steemd.com:34191 # roadscape steemwitness.matthewniemerg.com:2001 # complexring -#####steemd.pharesim.me:2001 # pharesim seed.jesta.us:2001 # jesta seed.liondani.com:2016 # liondani anyx.co:2001 # anyx seed.xeldal.com:12150 # xeldal seed.steemnodes.com:2001 # wackou -#####steem.clawmap.com:2001 # steempty gtg.steem.house:2001 # gtg 192.99.3.29:2001 # joseph 5.9.18.213:2001 # pfunk @@ -17,21 +14,16 @@ lafonasteem.com:2001 # lafona seed.rossco99.com:2001 # rossco99 steem-seed.altcap.io:40696 # ihashfury seed.steemfeeder.com:2001 # au1nethyb1 -#####52.175.211.168:2001 # aizensou seed.roelandp.nl:2001 # roelandp 81.89.101.133:2001 # cyrano.witness steem.global:2001 # klye seed.esteem.ws:2001 # good-karma 176.31.126.187:2001 # timcliff -#####seed.royaltiffany.me:2001 # royaltiffany seed.thecryptodrive.com:2001 # thecryptodrive steem-id.altexplorer.xyz:2001 # steem-id seed.bitcoiner.me:2001 # bitcoiner 104.199.118.92:2001 # clayop 192.99.4.226:2001 # dele-puppy -#####89.36.26.82:2001 # dragosroua -#####104.198.222.18:2001 # bitcoinparadise -#####138.197.17.188:2001 # chitty 88.99.33.113:2001 # cervantes seed.bhuz.info:2001 # bhuz 78.46.95.157:2001 # pcste From d21d39b55c88d5cd87e0caa3a9ab8a44d737c5ed Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Mon, 22 May 2017 10:18:33 -0400 Subject: [PATCH 21/59] Calculate proper curves #1124 --- libraries/chain/util/reward.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/chain/util/reward.cpp b/libraries/chain/util/reward.cpp index 750bf9fc81..8ca508f856 100644 --- a/libraries/chain/util/reward.cpp +++ b/libraries/chain/util/reward.cpp @@ -84,9 +84,10 @@ uint128_t evaluate_reward_curve( const uint128_t& rshares, const curve_id& curve } break; case linear: - result = approx_sqrt( rshares ); + result = rshares; break; case square_root: + result = approx_sqrt( rshares ); break; } From eb0d9b27d4b005267f10b0e59f0aa07521bbbfef Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Mon, 22 May 2017 13:52:09 -0400 Subject: [PATCH 22/59] Check invariants when database is opened #1125 --- libraries/chain/database.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 151a89fc6d..8407040807 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -123,6 +123,7 @@ void database::open( const fc::path& data_dir, const fc::path& shared_mem_dir, u undo_all(); FC_ASSERT( revision() == head_block_num(), "Chainbase revision does not match head block num", ("rev", revision())("head_block", head_block_num()) ); + validate_invariants(); }); if( head_block_num() ) From f4c40541e3c4b8fafa7b8d2c6711a53d4fb2ec58 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Mon, 22 May 2017 14:15:09 -0400 Subject: [PATCH 23/59] Refactor complex conditionals and error messages for readability #1126 --- libraries/chain/steem_evaluator.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 4d7be93ac5..90e17e8bc7 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -2231,7 +2231,7 @@ void delegate_vesting_shares_evaluator::do_apply( const delegate_vesting_shares_ { auto delta = op.vesting_shares - delegation->vesting_shares; - FC_ASSERT( delta >= min_update, "Steem Power increase is not enough of a different. min_update: ${min}", ("min", min_update) ); + FC_ASSERT( delta >= min_update, "Steem Power increase is not enough of a difference. min_update: ${min}", ("min", min_update) ); FC_ASSERT( available_shares >= op.vesting_shares - delegation->vesting_shares, "Account does not have enough vesting shares to delegate." ); _db.modify( delegator, [&]( account_object& a ) @@ -2256,15 +2256,19 @@ void delegate_vesting_shares_evaluator::do_apply( const delegate_vesting_shares_ if( _db.has_hardfork( STEEMIT_HARDFORK_0_19__971 ) ) { - FC_ASSERT( ( op.vesting_shares.amount == 0 ) - || ( delta >= min_update && op.vesting_shares >= min_delegation ), - "Delegation must be removed or be a significant change and leave a minimum delegation. min_update: ${min_update}, min_delegation: ${min_delegation}", - ("min_update", min_update)("min_delegation", min_delegation) ); + if( op.vesting_shares.amount > 0 ) + { + FC_ASSERT( delta >= min_update, "Steem Power decrease is not enough of a difference. min_update: ${min}", ("min", min_update) ); + FC_ASSERT( op.vesting_shares >= min_delegation, "Delegation must be removed or leave minimum delegation amount of ${v}", ("v", min_delegation) ); + } } else // TODO: Check and remove after HF19 { - FC_ASSERT( delta >= min_update, "Steem Power increase is not enough of a different. min_update: ${min}", ("min", min_update) ); - FC_ASSERT( op.vesting_shares >= min_delegation || op.vesting_shares.amount == 0, "Delegation must be removed or leave minimum delegation amount of ${v}", ("v", min_delegation) ); + FC_ASSERT( delta >= min_update, "Steem Power decrease is not enough of a difference. min_update: ${min}", ("min", min_update) ); + if( op.vesting_shares.amount > 0 ) + { + FC_ASSERT( op.vesting_shares >= min_delegation, "Delegation must be removed or leave minimum delegation amount of ${v}", ("v", min_delegation) ); + } } _db.create< vesting_delegation_expiration_object >( [&]( vesting_delegation_expiration_object& obj ) From 56ad7752f79ae7aa4a4b0f60fda4aa58ab0199f1 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Mon, 22 May 2017 14:37:04 -0400 Subject: [PATCH 24/59] Fix incorrect assertion for witness vote invariant #1125 --- libraries/chain/database.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 8407040807..a50ad794a6 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -3946,7 +3946,7 @@ void database::validate_invariants()const /// verify no witness has too many votes const auto& witness_idx = get_index< witness_index >().indices(); for( auto itr = witness_idx.begin(); itr != witness_idx.end(); ++itr ) - FC_ASSERT( itr->votes < gpo.total_vesting_shares.amount, "", ("itr",*itr) ); + FC_ASSERT( itr->votes <= gpo.total_vesting_shares.amount, "", ("itr",*itr) ); for( auto itr = account_idx.begin(); itr != account_idx.end(); ++itr ) { From 30a81324948f8914094bbeb0b68f34d1034101fe Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Tue, 23 May 2017 14:30:44 -0400 Subject: [PATCH 25/59] Remove immutable_chain_parameters.hpp #1131 --- .../chain/immutable_chain_parameters.hpp | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 libraries/chain/include/steemit/chain/immutable_chain_parameters.hpp diff --git a/libraries/chain/include/steemit/chain/immutable_chain_parameters.hpp b/libraries/chain/include/steemit/chain/immutable_chain_parameters.hpp deleted file mode 100644 index d3f2f68644..0000000000 --- a/libraries/chain/include/steemit/chain/immutable_chain_parameters.hpp +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include - -#include - -#include - -namespace steemit { namespace chain { - -struct immutable_chain_parameters -{ - uint16_t min_committee_member_count = STEEMIT_DEFAULT_MIN_COMMITTEE_MEMBER_COUNT; - uint16_t min_witness_count = STEEMIT_DEFAULT_MIN_WITNESS_COUNT; - uint32_t num_special_accounts = 0; - uint32_t num_special_assets = 0; -}; - -} } // steemit::chain - -FC_REFLECT( steemit::chain::immutable_chain_parameters, - (min_committee_member_count) - (min_witness_count) - (num_special_accounts) - (num_special_assets) -) From da72f7eb80854ba797b2ff4deaa102b2f7d15256 Mon Sep 17 00:00:00 2001 From: JustinW Date: Wed, 24 May 2017 09:05:29 -0400 Subject: [PATCH 26/59] adding steemdsync changes to master --- contrib/sync-sv-run.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/contrib/sync-sv-run.sh b/contrib/sync-sv-run.sh index fe5948f49b..b6b0c11f51 100644 --- a/contrib/sync-sv-run.sh +++ b/contrib/sync-sv-run.sh @@ -38,9 +38,13 @@ if [[ ! -z "$BLOCKCHAIN_TIME" ]]; then echo steemdsync: waiting for steemd to exit cleanly while [ -e /proc/$STEEMD_PID ]; do sleep 0.1; done echo steemdsync: starting a new blockchainstate upload operation - cd $HOME + cd ${COMPRESSPATH:-$HOME} echo steemdsync: compressing blockchainstate... - tar cf blockchain.tar.bz2 --use-compress-prog=pbzip2 blockchain + tar cf blockchain.tar.bz2 --use-compress-prog=pbzip2 -C $HOME blockchain + if [[ ! $? -eq 0 ]]; then + echo NOTIFYALERT! steemdsync was unable to compress shared memory file, check the logs. + exit 1 + fi FILE_NAME=blockchain-$VERSION-`date '+%Y%m%d-%H%M%S'`.tar.bz2 echo steemdsync: uploading $FILE_NAME to $S3_BUCKET aws s3 cp blockchain.tar.bz2 s3://$S3_BUCKET/$FILE_NAME @@ -56,6 +60,7 @@ if [[ ! -z "$BLOCKCHAIN_TIME" ]]; then exit 1 fi # upload a current block_log + cd $HOME aws s3 cp blockchain/block_log s3://$S3_BUCKET/block_log-intransit aws s3 cp s3://$S3_BUCKET/block_log-intransit s3://$S3_BUCKET/block_log-latest # kill the container starting the process over again From 10c2b1a917c81ea2d4acbc9c2655e9bf7b1bb1c1 Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 30 May 2017 11:44:06 -0400 Subject: [PATCH 27/59] increase default shared file size to 48gb --- libraries/app/application.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index e44f19387c..1f01f31772 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -970,7 +970,7 @@ void application::set_program_options(boost::program_options::options_descriptio ("seed-node,s", bpo::value>()->composing(), "P2P nodes to connect to on startup (may specify multiple times)") ("checkpoint,c", bpo::value>()->composing(), "Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints.") ("shared-file-dir", bpo::value(), "Location of the shared memory file. Defaults to data_dir/blockchain") - ("shared-file-size", bpo::value()->default_value("32G"), "Size of the shared memory file. Default: 32G") + ("shared-file-size", bpo::value()->default_value("48G"), "Size of the shared memory file. Default: 48G") ("rpc-endpoint", bpo::value()->implicit_value("127.0.0.1:8090"), "Endpoint for websocket RPC to listen on") ("rpc-tls-endpoint", bpo::value()->implicit_value("127.0.0.1:8089"), "Endpoint for TLS websocket RPC to listen on") ("read-forward-rpc", bpo::value(), "Endpoint to forward write API calls to for a read node" ) From edd36bf38f5ad12649c94098e5133b148354f3c3 Mon Sep 17 00:00:00 2001 From: Gandalf Date: Tue, 30 May 2017 21:31:59 +0200 Subject: [PATCH 28/59] Added preliminary quickstart checklist to help getting steemd up and running while avoiding most common issues for certain use cases --- doc/quickstart.md | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 doc/quickstart.md diff --git a/doc/quickstart.md b/doc/quickstart.md new file mode 100644 index 0000000000..f7dba20985 --- /dev/null +++ b/doc/quickstart.md @@ -0,0 +1,52 @@ +Quickstart +---------- + +### Get current steemd +#### Docker +Preferred way is to use docker. +#### Building +If you can't or don't want to use Docker and you want to build `steemd` and `cli_wallet` yourself follow [build instructions](building.md). +Those was made for `Ubuntu 16.04 LTS` and `macOS`. +If you have troubles building, please use Docker. + +#### Low memory node? +Unless you run full api node, use +`LOW_MEMORY_NODE=ON` +Suitable for: +- seed nodes +- witness nodes +- exchanges, etc. +For full api node use `LOW_MEMORY_NODE=OFF` + +### Configure for your use case +#### Full API node +Do not use low memory node for this case: +`LOW_MEMORY_NODE=OFF` +Use `contrib/fullnode.config.ini` as a base for your `config.ini` file. + +#### Exchanges +Use low memory node: +`LOW_MEMORY_NODE=ON` +Also make sure that your `config.ini` contains: +``` +enable-plugin = account_history +public-api = database_api login_api +track-account-range = ["yourexchangeid", "yourexchangeid"] +``` +Do not add other APIs or plugins unless you know what you are doing. + +### Resources usage + +Please make sure that you have enough resources available. +Provided values are expected to grow significantly over time. + +Blockchain data takes over **12GB** of storage space. + +#### Full node +Shared memory file for full node uses over **45GB** + +#### Seed node +Shared memory file for seed node uses over **3GB** + +#### Other use cases +Shared memory file size varies, depends on your specific configuration but it is expected to be somewhere between "seed node" and "full node" usage. From fdbc4c326fd30ff846a25dd2a625aa374462e9e9 Mon Sep 17 00:00:00 2001 From: Gandalf Date: Tue, 30 May 2017 22:43:33 +0200 Subject: [PATCH 29/59] showing instead of telling people looking how to build by themselves are not the target of this doc --- doc/quickstart.md | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/doc/quickstart.md b/doc/quickstart.md index f7dba20985..24b16ef623 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -2,31 +2,33 @@ Quickstart ---------- ### Get current steemd -#### Docker -Preferred way is to use docker. -#### Building -If you can't or don't want to use Docker and you want to build `steemd` and `cli_wallet` yourself follow [build instructions](building.md). -Those was made for `Ubuntu 16.04 LTS` and `macOS`. -If you have troubles building, please use Docker. - +Use docker: +``` +docker run \ + -d -p 2001:2001 -p 8090:8090 --name steemd-default \ + steemit/steem +``` #### Low memory node? -Unless you run full api node, use -`LOW_MEMORY_NODE=ON` -Suitable for: +Above runs low memory node, which is suitable for: - seed nodes - witness nodes - exchanges, etc. -For full api node use `LOW_MEMORY_NODE=OFF` +For full api node use: +``` +docker run \ + --env USE_WAY_TOO_MUCH_RAM=1 --env USE_FULL_WEB_NODE=1 \ + -d -p 2001:2001 -p 8090:8090 --name steemd-full \ + steemit/steem +``` ### Configure for your use case #### Full API node -Do not use low memory node for this case: -`LOW_MEMORY_NODE=OFF` -Use `contrib/fullnode.config.ini` as a base for your `config.ini` file. +You need to use `USE_WAY_TOO_MUCH_RAM=1` and `USE_FULL_WEB_NODE=1` as stated above. +You can Use `contrib/fullnode.config.ini` as a base for your `config.ini` file. #### Exchanges -Use low memory node: -`LOW_MEMORY_NODE=ON` +Use low memory node. + Also make sure that your `config.ini` contains: ``` enable-plugin = account_history From 8878c80afde449af2674d3ba0aac634c2020f8b7 Mon Sep 17 00:00:00 2001 From: Gandalf Date: Thu, 1 Jun 2017 09:54:12 +0200 Subject: [PATCH 30/59] note about shared-file-size (common issue) --- doc/quickstart.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/quickstart.md b/doc/quickstart.md index 24b16ef623..2931ef497d 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -40,6 +40,9 @@ Do not add other APIs or plugins unless you know what you are doing. ### Resources usage Please make sure that you have enough resources available. +Check `shared-file-size =` in your `config.ini` to reflect your needs. +Set it to at least 20% more than current size. + Provided values are expected to grow significantly over time. Blockchain data takes over **12GB** of storage space. From 55a0e21f39d72ec684892c11dfab1f80699ffc33 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 2 Jun 2017 11:46:35 -0400 Subject: [PATCH 31/59] Misc cleanup of 0.19.0 codebase #1149 --- libraries/app/database_api.cpp | 2 +- libraries/chain/database.cpp | 24 +++++++++++++------ .../include/steemit/chain/comment_object.hpp | 2 ++ libraries/chain/steem_evaluator.cpp | 15 +++++------- libraries/chain/witness_schedule.cpp | 1 - 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp index b150e9fb85..753ffea8e7 100755 --- a/libraries/app/database_api.cpp +++ b/libraries/app/database_api.cpp @@ -1145,7 +1145,7 @@ void database_api::set_pending_payout( discussion& d )const vshares = d.net_rshares.value > 0 ? steemit::chain::util::evaluate_reward_curve( d.net_rshares.value, rf.author_reward_curve, rf.content_constant ) : 0; } else - vshares = steemit::chain::util::evaluate_reward_curve( d.net_rshares.value > 0 ? d.net_rshares.value : 0 ); + vshares = d.net_rshares.value > 0 ? steemit::chain::util::evaluate_reward_curve( d.net_rshares.value ) : 0; u256 r2 = to256(vshares); //to256(abs_net_rshares); r2 *= pot.amount.value; diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index a50ad794a6..c48a327bae 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -1151,7 +1151,8 @@ void database::clear_witness_votes( const account_object& a ) remove(current); } - if( has_hardfork( STEEMIT_HARDFORK_0_6__104 ) ) // TODO: this check can be removed after hard fork + #warning( "TODO: Remove this check after HF 19" ) + if( has_hardfork( STEEMIT_HARDFORK_0_6__104 ) ) modify( a, [&](account_object& acc ) { acc.witnesses_voted_for = 0; @@ -1622,7 +1623,7 @@ void database::process_comment_cashout() { fc::microseconds decay_rate; - // TODO: Remove temp fund after HF 19 + #warning( "TODO: Remove temp reward fund after HF 19" ) if( rfo.name == STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME || has_hardfork( STEEMIT_HARDFORK_0_19__1051 ) ) decay_rate = STEEMIT_RECENT_RSHARES_DECAY_RATE_HF19; else @@ -3087,7 +3088,7 @@ void database::update_last_irreversible_block() witness_time_pairs.push_back( std::make_pair( b->data.witness, b->data.timestamp ) ); } - ilog( "Encountered a block num collision due to a fork. Walking the current fork to determine the correct block. block_num:${n}", ("n", log_head_num + 1) ); // TODO: Delete when we know this code works as intended + ilog( "Encountered a block num collision due to a fork. Walking the current fork to determine the correct block. block_num:${n}", ("n", log_head_num + 1) ); ilog( "Colliding blocks produced by witnesses at times: ${w}", ("w", witness_time_pairs) ); auto next = _fork_db.head(); @@ -3748,8 +3749,8 @@ void database::apply_hardfork( uint32_t hardfork ) case STEEMIT_HARDFORK_0_17: { static_assert( - STEEMIT_MAX_VOTED_WITNESSES_HF0 + STEEMIT_MAX_MINER_WITNESSES_HF0 + STEEMIT_MAX_RUNNER_WITNESSES_HF0 == STEEMIT_MAX_WITNESSES, - "HF0 witness counts must add up to STEEMIT_MAX_WITNESSES" ); + STEEMIT_MAX_VOTED_WITNESSES_HF0 + STEEMIT_MAX_MINER_WITNESSES_HF0 + STEEMIT_MAX_RUNNER_WITNESSES_HF0 == STEEMIT_MAX_WITNESSES, + "HF0 witness counts must add up to STEEMIT_MAX_WITNESSES" ); static_assert( STEEMIT_MAX_VOTED_WITNESSES_HF17 + STEEMIT_MAX_MINER_WITNESSES_HF17 + STEEMIT_MAX_RUNNER_WITNESSES_HF17 == STEEMIT_MAX_WITNESSES, "HF17 witness counts must add up to STEEMIT_MAX_WITNESSES" ); @@ -3794,7 +3795,7 @@ void database::apply_hardfork( uint32_t hardfork ) // As a shortcut in payout processing, we use the id as an array index. // The IDs must be assigned this way. The assertion is a dummy check to ensure this happens. FC_ASSERT( post_rf.id._id == 0 ); - FC_ASSERT( linear_rf.id._id == 1 ); + FC_ASSERT( linear_rf.id._id == STEEMIT_TEMP_LINEAR_REWARD_FUND_ID ); modify( gpo, [&]( dynamic_global_property_object& g ) { @@ -3860,14 +3861,21 @@ void database::apply_hardfork( uint32_t hardfork ) const auto& linear = get< reward_fund_object, by_name >( STEEMIT_TEMP_LINEAR_REWARD_FUND_NAME ); modify( get< reward_fund_object, by_name >( STEEMIT_POST_REWARD_FUND_NAME ), [&]( reward_fund_object &rfo ) { + #warning( "TODO: Replace with constant after HF 19" ) rfo.recent_claims = linear.recent_claims; rfo.author_reward_curve = curve_id::linear; rfo.curation_reward_curve = curve_id::square_root; }); + #warning( "TODO: Remove weight conversion after HF 19" ) const auto& cidx = get_index< comment_index, by_cashout_time >(); const auto& vidx = get_index< comment_vote_index, by_comment_voter >(); - for( auto c_itr = cidx.begin(); c_itr != cidx.end() && c_itr->cashout_time != fc::time_point_sec::maximum(); ++c_itr ) + + /* + * Iterator through all comments that have not yet been paid, setting their total vote weight to the sqrt weight + * and update their votes as well. + */ + for( auto c_itr = cidx.begin(); c_itr != cidx.end() && c_itr->cashout_time < fc::time_point_sec::maximum(); ++c_itr ) { modify( *c_itr, [&]( comment_object& c ) { @@ -3883,6 +3891,8 @@ void database::apply_hardfork( uint32_t hardfork ) } } + #warning( "TODO: Remove if 0 delegation opjects are not created in pre HF19 consensus" ) + /* Remove all 0 delegation objects */ vector< const vesting_delegation_object* > to_remove; const auto& delegation_idx = get_index< vesting_delegation_index, by_id >(); auto delegation_itr = delegation_idx.begin(); diff --git a/libraries/chain/include/steemit/chain/comment_object.hpp b/libraries/chain/include/steemit/chain/comment_object.hpp index 02f284741c..7ae09db327 100644 --- a/libraries/chain/include/steemit/chain/comment_object.hpp +++ b/libraries/chain/include/steemit/chain/comment_object.hpp @@ -78,6 +78,7 @@ namespace steemit { namespace chain { time_point_sec cashout_time; /// 24 hours from the weighted average of vote time time_point_sec max_cashout_time; uint64_t total_vote_weight = 0; /// the total weight of voting rewards, used to calculate pro-rata share of curation payouts + #warning( "Remove total_sqrt_vote_weight after HF 19" ) uint64_t total_sqrt_vote_weight = 0; /// Temporary total weight of votes for post hf19 curation reward curve uint16_t reward_weight = 0; @@ -121,6 +122,7 @@ namespace steemit { namespace chain { account_id_type voter; comment_id_type comment; uint64_t weight = 0; ///< defines the score this vote receives, used by vote payout calc. 0 if a negative vote or changed votes. + #warning( "Remove sqrt_weight after HF 19. It is a temporary field" ) uint64_t sqrt_weight = 0; ///< Temporary vote weight for post hf19 curation curve int64_t rshares = 0; ///< The number of rshares this vote is responsible for int16_t vote_percent = 0; ///< The percent weight of the vote diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 4d7be93ac5..7f34a7f3df 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -124,6 +124,7 @@ void account_create_evaluator::do_apply( const account_create_operation& o ) } else if( _db.has_hardfork( STEEMIT_HARDFORK_0_17__818 ) ) { + #warning( "TODO: Remove after HF19. Entire conditional might be able to be removed." ) if( _db.is_producing() ) FC_ASSERT( false, "account_create_operation is temporarily disabled. Please use account_create_with_delegation_operation instead" ); const witness_schedule_object& wso = _db.get_witness_schedule_object(); @@ -271,9 +272,7 @@ void account_create_with_delegation_evaluator::do_apply( const account_create_wi auth.last_owner_update = fc::time_point_sec::min(); }); - /* TODO: Check if not creating 0 delegation objects in HF19 passes consensus - * If it does we can remove the apply_hardfork logic for deleting 0 delegation objects - */ + #warning( "TODO: Check if not creating 0 delegation objects in HF19 passes consensus. Expectation is it should work." ) if( ( _db.has_hardfork( STEEMIT_HARDFORK_0_19__997 ) && o.delegation.amount > 0 ) || !_db.has_hardfork( STEEMIT_HARDFORK_0_19__997 ) ) { @@ -384,7 +383,7 @@ void delete_comment_evaluator::do_apply( const delete_comment_operation& o ) if( _db.has_hardfork( STEEMIT_HARDFORK_0_19__876 ) ) FC_ASSERT( comment.cashout_time != fc::time_point_sec::maximum() ); - // TODO: remove is_producing after HF 19. Check if we can remove conditional altogether after HF. + #warning( "TODO: Remove is_producing check after HF19. Conditional may be removed entirely after HF." ) if( _db.is_producing() || _db.has_hardfork( STEEMIT_HARDFORK_0_19__977 ) ) { FC_ASSERT( comment.net_rshares <= 0, "Cannot delete a comment with net positive votes." ); } @@ -460,8 +459,6 @@ void comment_options_evaluator::do_apply( const comment_options_operation& o ) if( !o.allow_curation_rewards || !o.allow_votes || o.max_accepted_payout < comment.max_accepted_payout ) FC_ASSERT( comment.abs_rshares == 0, "One of the included comment options requires the comment to have no rshares allocated to it." ); - if( !_db.has_hardfork( STEEMIT_HARDFORK_0_17__773) ) // TODO: Remove after hardfork 17 - FC_ASSERT( o.extensions.size() == 0, "Operation extensions for the comment_options_operation are not currently supported." ); FC_ASSERT( comment.allow_curation_rewards >= o.allow_curation_rewards, "Curation rewards cannot be re-enabled." ); FC_ASSERT( comment.allow_votes >= o.allow_votes, "Voting cannot be re-enabled." ); FC_ASSERT( comment.max_accepted_payout >= o.max_accepted_payout, "A comment cannot accept a greater payout." ); @@ -1364,6 +1361,7 @@ void vote_evaluator::do_apply( const vote_operation& o ) uint64_t old_weight = util::evaluate_reward_curve( old_vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ).to_uint64(); uint64_t new_weight = util::evaluate_reward_curve( comment.vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ).to_uint64(); cv.weight = new_weight - old_weight; + #warning( "TODO: After HF19 caclulate which comment was the first to be paid on sqrt curation and activate the logic based on time" ) if( !_db.has_hardfork( STEEMIT_HARDFORK_0_19__1052 ) ) { old_weight = util::evaluate_reward_curve( old_vote_rshares.value, curve_id::square_root ).to_uint64(); @@ -2189,8 +2187,6 @@ void claim_reward_balance_evaluator::do_apply( const claim_reward_balance_operat void delegate_vesting_shares_evaluator::do_apply( const delegate_vesting_shares_operation& op ) { - FC_ASSERT( _db.has_hardfork( STEEMIT_HARDFORK_0_17__818 ), "delegate_vesting_shares_operation is not enabled until HF 17" ); //TODO: Delete after hardfork - const auto& delegator = _db.get_account( op.delegator ); const auto& delegatee = _db.get_account( op.delegatee ); auto delegation = _db.find< vesting_delegation_object, by_delegation >( boost::make_tuple( op.delegator, op.delegatee ) ); @@ -2261,7 +2257,8 @@ void delegate_vesting_shares_evaluator::do_apply( const delegate_vesting_shares_ "Delegation must be removed or be a significant change and leave a minimum delegation. min_update: ${min_update}, min_delegation: ${min_delegation}", ("min_update", min_update)("min_delegation", min_delegation) ); } - else // TODO: Check and remove after HF19 + #warning( "TODO: Check and remove conditional after HF19" ) + else { FC_ASSERT( delta >= min_update, "Steem Power increase is not enough of a different. min_update: ${min}", ("min", min_update) ); FC_ASSERT( op.vesting_shares >= min_delegation || op.vesting_shares.amount == 0, "Delegation must be removed or leave minimum delegation amount of ${v}", ("v", min_delegation) ); diff --git a/libraries/chain/witness_schedule.cpp b/libraries/chain/witness_schedule.cpp index e2cd2c7718..52ff45ba0c 100644 --- a/libraries/chain/witness_schedule.cpp +++ b/libraries/chain/witness_schedule.cpp @@ -259,7 +259,6 @@ void update_witness_schedule4( database& db ) db.modify( wso, [&]( witness_schedule_object& _wso ) { - // active witnesses has exactly STEEMIT_MAX_WITNESSES elements, asserted above for( size_t i = 0; i < active_witnesses.size(); i++ ) { _wso.current_shuffled_witnesses[i] = active_witnesses[i]; From 2b15cfa35f4723423aa84ca31d13a222314e8b9e Mon Sep 17 00:00:00 2001 From: JustinW Date: Fri, 2 Jun 2017 12:13:53 -0400 Subject: [PATCH 32/59] update shared memory size, proxy buffering off --- contrib/fullnode.config.ini | 2 +- contrib/healthcheck.conf.template | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/fullnode.config.ini b/contrib/fullnode.config.ini index fa6182cac4..9a126801a3 100644 --- a/contrib/fullnode.config.ini +++ b/contrib/fullnode.config.ini @@ -1,5 +1,5 @@ # Set larger shared-file-size than default -shared-file-size = 48G +shared-file-size = 54G # Set an API to be publicly available, may be specified multiple times public-api = database_api login_api account_by_key_api network_broadcast_api tag_api follow_api market_history_api raw_block_api diff --git a/contrib/healthcheck.conf.template b/contrib/healthcheck.conf.template index 9df3d404ae..3b690f4cb6 100644 --- a/contrib/healthcheck.conf.template +++ b/contrib/healthcheck.conf.template @@ -13,6 +13,7 @@ proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_http_version 1.1; + proxy_buffering off; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; add_header 'Access-Control-Allow-Origin' '*'; From 7c4d213d231afc9c7e1ed7fb461db618ca51e8db Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 2 Jun 2017 12:24:21 -0400 Subject: [PATCH 33/59] Update to 56G --- libraries/app/application.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index 1f01f31772..d744670682 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -970,7 +970,7 @@ void application::set_program_options(boost::program_options::options_descriptio ("seed-node,s", bpo::value>()->composing(), "P2P nodes to connect to on startup (may specify multiple times)") ("checkpoint,c", bpo::value>()->composing(), "Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints.") ("shared-file-dir", bpo::value(), "Location of the shared memory file. Defaults to data_dir/blockchain") - ("shared-file-size", bpo::value()->default_value("48G"), "Size of the shared memory file. Default: 48G") + ("shared-file-size", bpo::value()->default_value("54G"), "Size of the shared memory file. Default: 54G") ("rpc-endpoint", bpo::value()->implicit_value("127.0.0.1:8090"), "Endpoint for websocket RPC to listen on") ("rpc-tls-endpoint", bpo::value()->implicit_value("127.0.0.1:8089"), "Endpoint for TLS websocket RPC to listen on") ("read-forward-rpc", bpo::value(), "Endpoint to forward write API calls to for a read node" ) From 06c34f3cc5ec62edb8e289264c0ab4a39f5c24dd Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 2 Jun 2017 11:49:20 -0400 Subject: [PATCH 34/59] Fix miscalculation for vote sort vote weight #1148 --- libraries/chain/steem_evaluator.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 7f34a7f3df..9ffe3f1460 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -1416,7 +1416,7 @@ void vote_evaluator::do_apply( const vote_operation& o ) _db.modify( comment, [&]( comment_object& c ) { c.total_vote_weight += max_vote_weight; - c.total_sqrt_vote_weight = sqrt_max_vote_weight; + c.total_sqrt_vote_weight += sqrt_max_vote_weight; }); } if( !_db.has_hardfork( STEEMIT_HARDFORK_0_17__774) ) @@ -1515,6 +1515,7 @@ void vote_evaluator::do_apply( const vote_operation& o ) _db.modify( comment, [&]( comment_object& c ) { c.total_vote_weight -= itr->weight; + c.total_sqrt_vote_weight -= itr->sqrt_weight; }); _db.modify( *itr, [&]( comment_vote_object& cv ) @@ -1523,6 +1524,7 @@ void vote_evaluator::do_apply( const vote_operation& o ) cv.vote_percent = o.weight; cv.last_update = _db.head_block_time(); cv.weight = 0; + cv.sqrt_weight = 0; cv.num_changes += 1; }); From 7a417d2186a275e2c189dc7c27656d89a6451377 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 2 Jun 2017 15:39:52 -0400 Subject: [PATCH 35/59] Test patch #1148 --- libraries/chain/hardfork.d/0_6.hf | 4 ++ libraries/chain/steem_evaluator.cpp | 10 +++- tests/tests/operation_tests.cpp | 92 +++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 1 deletion(-) diff --git a/libraries/chain/hardfork.d/0_6.hf b/libraries/chain/hardfork.d/0_6.hf index aa86b802b5..31962cbd71 100644 --- a/libraries/chain/hardfork.d/0_6.hf +++ b/libraries/chain/hardfork.d/0_6.hf @@ -16,6 +16,10 @@ // Fri Jun 24 14:00:00 UTC 2016 // Fri Jun 24 10:00:00 EDT 2016 +#ifdef IS_TEST_NET +#define STEEMIT_HARDFORK_0_6_REVERSE_AUCTION_TIME (0) +#else #define STEEMIT_HARDFORK_0_6_REVERSE_AUCTION_TIME (1467295200-(60*60*24*6)) +#endif #define STEEMIT_HARDFORK_0_6_VERSION hardfork_version( 0, 6 ) #endif diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 9ffe3f1460..a099bbcb0c 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -1362,7 +1362,14 @@ void vote_evaluator::do_apply( const vote_operation& o ) uint64_t new_weight = util::evaluate_reward_curve( comment.vote_rshares.value, reward_fund.curation_reward_curve, reward_fund.content_constant ).to_uint64(); cv.weight = new_weight - old_weight; #warning( "TODO: After HF19 caclulate which comment was the first to be paid on sqrt curation and activate the logic based on time" ) + +#ifndef IS_TEST_NET +/* + * Disabling this check so we can test the precalculation logic. + * It is not needed on live because the precalculation is only needed pre HF19 + */ if( !_db.has_hardfork( STEEMIT_HARDFORK_0_19__1052 ) ) +#endif { old_weight = util::evaluate_reward_curve( old_vote_rshares.value, curve_id::square_root ).to_uint64(); new_weight = util::evaluate_reward_curve( comment.vote_rshares.value, curve_id::square_root ).to_uint64(); @@ -1395,8 +1402,9 @@ void vote_evaluator::do_apply( const vote_operation& o ) w *= delta_t; w /= STEEMIT_REVERSE_AUCTION_WINDOW_SECONDS; cv.weight = w.to_uint64(); - +#ifndef IS_TEST_NET if( _db.has_hardfork( STEEMIT_HARDFORK_0_17 ) ) +#endif { uint128_t w(sqrt_max_vote_weight); w *= delta_t; diff --git a/tests/tests/operation_tests.cpp b/tests/tests/operation_tests.cpp index 0494dee515..5f1258deb6 100644 --- a/tests/tests/operation_tests.cpp +++ b/tests/tests/operation_tests.cpp @@ -6651,5 +6651,97 @@ BOOST_AUTO_TEST_CASE( comment_beneficiaries_apply ) FC_LOG_AND_RETHROW() } +#warning( "TODO: This test is not needed after HF19" ) +BOOST_AUTO_TEST_CASE( vote_weight_test ) +{ + try + { + BOOST_TEST_MESSAGE( "Test Comment Beneficiaries" ); + ACTORS( (alice)(bob)(sam)(dave) ) + generate_block(); + + set_price_feed( price( ASSET( "1.000 TESTS" ), ASSET( "1.000 TBD" ) ) ); + + comment_operation comment; + vote_operation vote; + signed_transaction tx; + + comment.author = "alice"; + comment.permlink = "test"; + comment.parent_permlink = "test"; + comment.title = "test"; + comment.body = "foobar"; + + tx.operations.push_back( comment ); + tx.set_expiration( db.head_block_time() + STEEMIT_MAX_TIME_UNTIL_EXPIRATION ); + tx.sign( alice_private_key, db.get_chain_id() ); + db.push_transaction( tx, 0 ); + + generate_blocks( 10 ); + + BOOST_TEST_MESSAGE( "--- Testing initial vote" ); + + vote.voter = "alice"; + vote.author = "alice"; + vote.permlink = "test"; + vote.weight = STEEMIT_100_PERCENT; + + tx.clear(); + tx.operations.push_back( vote ); + tx.sign( alice_private_key, db.get_chain_id() ); + db.push_transaction( tx, 0 ); + + { + const auto& alice = db.get_account( "alice" ); + const auto& alice_comment = db.get_comment( comment.author, comment.permlink ); + const auto& alice_vote = db.get< comment_vote_object, by_comment_voter >( boost::make_tuple( alice_comment.id, alice.id ) ); + + BOOST_REQUIRE( alice_vote.weight == alice_vote.sqrt_weight ); + BOOST_REQUIRE( alice_comment.total_vote_weight == alice_comment.total_sqrt_vote_weight ); + } + + generate_blocks( 10 ); + + BOOST_TEST_MESSAGE( "--- Testing second vote" ); + + vote.voter = "bob"; + + tx.clear(); + tx.operations.push_back( vote ); + tx.sign( bob_private_key, db.get_chain_id() ); + db.push_transaction( tx, 0 ); + + { + const auto& bob = db.get_account( "bob" ); + const auto& alice_comment = db.get_comment( comment.author, comment.permlink ); + const auto& bob_vote = db.get< comment_vote_object, by_comment_voter >( boost::make_tuple( alice_comment.id, bob.id ) ); + + BOOST_REQUIRE( bob_vote.weight == bob_vote.sqrt_weight ); + BOOST_REQUIRE( alice_comment.total_vote_weight == alice_comment.total_sqrt_vote_weight ); + } + + generate_blocks( 10 ); + + BOOST_TEST_MESSAGE( "--- Testing removing a vote" ); + + vote.weight = 0; + + tx.clear(); + tx.operations.push_back( vote ); + tx.sign( bob_private_key, db.get_chain_id() ); + db.push_transaction( tx, 0 ); + + { + const auto& bob = db.get_account( "bob" ); + const auto& alice_comment = db.get_comment( comment.author, comment.permlink ); + const auto& bob_vote = db.get< comment_vote_object, by_comment_voter >( boost::make_tuple( alice_comment.id, bob.id ) ); + + BOOST_REQUIRE( bob_vote.weight == bob_vote.sqrt_weight ); + BOOST_REQUIRE( alice_comment.total_vote_weight == alice_comment.total_sqrt_vote_weight ); + } + } + FC_LOG_AND_RETHROW() +} + BOOST_AUTO_TEST_SUITE_END() #endif From aeb2703443ff9aea4b9b340fe405f9c48d500f7a Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Fri, 2 Jun 2017 15:48:04 -0400 Subject: [PATCH 36/59] Forbid no-op zero delegation #1126 --- libraries/chain/steem_evaluator.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libraries/chain/steem_evaluator.cpp b/libraries/chain/steem_evaluator.cpp index 90e17e8bc7..63a1ec3fa2 100644 --- a/libraries/chain/steem_evaluator.cpp +++ b/libraries/chain/steem_evaluator.cpp @@ -2261,6 +2261,10 @@ void delegate_vesting_shares_evaluator::do_apply( const delegate_vesting_shares_ FC_ASSERT( delta >= min_update, "Steem Power decrease is not enough of a difference. min_update: ${min}", ("min", min_update) ); FC_ASSERT( op.vesting_shares >= min_delegation, "Delegation must be removed or leave minimum delegation amount of ${v}", ("v", min_delegation) ); } + else + { + FC_ASSERT( delegation->vesting_shares.amount > 0, "Delegation would set vesting_shares to zero, but it is already zero"); + } } else // TODO: Check and remove after HF19 { From 16ff1524d9defcb3ec4a763379597d6d8ef14c51 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Fri, 2 Jun 2017 16:15:28 -0400 Subject: [PATCH 37/59] Fix test that broke as an unrelated consequence of the testing infrastructure #1148 --- tests/tests/operation_time_tests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests/operation_time_tests.cpp b/tests/tests/operation_time_tests.cpp index 3f7b5df69e..f3af8da2b7 100644 --- a/tests/tests/operation_time_tests.cpp +++ b/tests/tests/operation_time_tests.cpp @@ -145,7 +145,7 @@ BOOST_AUTO_TEST_CASE( comment_payout_equalize ) const account_object& bob_account = db.get_account("bob"); const account_object& dave_account = db.get_account("dave"); - BOOST_CHECK( alice_account.reward_sbd_balance == ASSET( "10720.000 TBD" ) ); + BOOST_CHECK( alice_account.reward_sbd_balance == ASSET( "14288.000 TBD" ) ); BOOST_CHECK( bob_account.reward_sbd_balance == ASSET( "0.000 TBD" ) ); BOOST_CHECK( dave_account.reward_sbd_balance == alice_account.reward_sbd_balance ); } From 7d5b8aedf99953a3bb662aae33f16af1396c96a8 Mon Sep 17 00:00:00 2001 From: JustinW Date: Sat, 3 Jun 2017 13:57:08 -0400 Subject: [PATCH 38/59] drop shared memory to 52G, max for current instance size --- contrib/fullnode.config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fullnode.config.ini b/contrib/fullnode.config.ini index 9a126801a3..1727a0c021 100644 --- a/contrib/fullnode.config.ini +++ b/contrib/fullnode.config.ini @@ -1,5 +1,5 @@ # Set larger shared-file-size than default -shared-file-size = 54G +shared-file-size = 52G # Set an API to be publicly available, may be specified multiple times public-api = database_api login_api account_by_key_api network_broadcast_api tag_api follow_api market_history_api raw_block_api From 979203186616c4f635ef5bed1dcf7b2ef3608cf7 Mon Sep 17 00:00:00 2001 From: "J. Chitty" Date: Sun, 4 Jun 2017 23:32:35 -0300 Subject: [PATCH 39/59] adding @chitty node Chitty's node changed servers, just added it. --- doc/seednodes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/seednodes.txt b/doc/seednodes.txt index 8f9866e34f..a6cfefebe5 100644 --- a/doc/seednodes.txt +++ b/doc/seednodes.txt @@ -27,3 +27,4 @@ seed.bitcoiner.me:2001 # bitcoiner 88.99.33.113:2001 # cervantes seed.bhuz.info:2001 # bhuz 78.46.95.157:2001 # pcste +45.76.13.167:2001 # chitty From 70975df292fe30394f90cf3b215333d24fefae0c Mon Sep 17 00:00:00 2001 From: ausbitbank Date: Wed, 7 Jun 2017 01:03:53 +1000 Subject: [PATCH 40/59] Added new node seed.steemviz.com --- doc/seednodes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/seednodes.txt b/doc/seednodes.txt index 8f9866e34f..090eb85ac2 100644 --- a/doc/seednodes.txt +++ b/doc/seednodes.txt @@ -27,3 +27,4 @@ seed.bitcoiner.me:2001 # bitcoiner 88.99.33.113:2001 # cervantes seed.bhuz.info:2001 # bhuz 78.46.95.157:2001 # pcste +seed.steemviz.com:2001 # ausbitbank From 48002bcf3f5523d2d75f555ca754397e546616f3 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Tue, 6 Jun 2017 14:59:45 -0400 Subject: [PATCH 41/59] Make by_transaction_id an optional index of the operation_object #1160 --- CMakeLists.txt | 7 +++++++ libraries/app/database_api.cpp | 4 ++++ libraries/chain/include/steemit/chain/history_object.hpp | 5 ++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b5ad095b81..22d938b9a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,13 @@ if( CLEAR_VOTES ) SET( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DCLEAR_VOTES" ) endif() +OPTION( SKIP_BY_TX_ID "Skip ordering operation history by transaction id (ON or OFF)" OFF ) +MESSAGE( STATUS "SKIP_BY_TX_ID: ${SKIP_BY_TX_ID}" ) +if( SKIP_BY_TX_ID ) + SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSKIP_BY_TX_ID" ) + SET( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSKIP_BY_TX_ID" ) +endif() + IF( WIN32 ) SET(BOOST_ROOT $ENV{BOOST_ROOT}) set(Boost_USE_MULTITHREADED ON) diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp index 753ffea8e7..59ffb4b78a 100755 --- a/libraries/app/database_api.cpp +++ b/libraries/app/database_api.cpp @@ -2348,6 +2348,9 @@ state database_api::get_state( string path )const annotated_signed_transaction database_api::get_transaction( transaction_id_type id )const { +#ifdef SKIP_BY_TX_ID + return annotated_signed_transaction(); +#else return my->_db.with_read_lock( [&](){ const auto& idx = my->_db.get_index().indices().get(); auto itr = idx.lower_bound( id ); @@ -2362,6 +2365,7 @@ annotated_signed_transaction database_api::get_transaction( transaction_id_type } FC_ASSERT( false, "Unknown Transaction ${t}", ("t",id)); }); +#endif } diff --git a/libraries/chain/include/steemit/chain/history_object.hpp b/libraries/chain/include/steemit/chain/history_object.hpp index 654532c996..ee67223250 100644 --- a/libraries/chain/include/steemit/chain/history_object.hpp +++ b/libraries/chain/include/steemit/chain/history_object.hpp @@ -49,13 +49,16 @@ namespace steemit { namespace chain { member< operation_object, uint64_t, &operation_object::virtual_op>, member< operation_object, operation_id_type, &operation_object::id> > - >, + > +#ifndef SKIP_BY_TX_ID + , ordered_unique< tag< by_transaction_id >, composite_key< operation_object, member< operation_object, transaction_id_type, &operation_object::trx_id>, member< operation_object, operation_id_type, &operation_object::id> > > +#endif >, allocator< operation_object > > operation_index; From 86a4cd57c22d26b31a7b8f46fe053d5542d3e9e9 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Tue, 6 Jun 2017 15:00:55 -0400 Subject: [PATCH 42/59] Optimize update_tags to not deserialize json metadata on every call #1161 --- .../tags/include/steemit/tags/tags_plugin.hpp | 8 +- libraries/plugins/tags/tags_plugin.cpp | 77 +++++++++++++------ 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/libraries/plugins/tags/include/steemit/tags/tags_plugin.hpp b/libraries/plugins/tags/include/steemit/tags/tags_plugin.hpp index 80bd29e020..ab3e8fe7af 100644 --- a/libraries/plugins/tags/include/steemit/tags/tags_plugin.hpp +++ b/libraries/plugins/tags/include/steemit/tags/tags_plugin.hpp @@ -118,7 +118,13 @@ typedef multi_index_container< tag_object, indexed_by< ordered_unique< tag< by_id >, member< tag_object, tag_id_type, &tag_object::id > >, - ordered_non_unique< tag< by_comment >, member< tag_object, comment_id_type, &tag_object::comment > >, + ordered_unique< tag< by_comment >, + composite_key< tag_object, + member< tag_object, comment_id_type, &tag_object::comment >, + member< tag_object, tag_id_type, &tag_object::id > + >, + composite_key_compare< std::less< comment_id_type >, std::less< tag_id_type > > + >, ordered_unique< tag< by_author_comment >, composite_key< tag_object, member< tag_object, account_id_type, &tag_object::author >, diff --git a/libraries/plugins/tags/tags_plugin.cpp b/libraries/plugins/tags/tags_plugin.cpp index 4a9bbf755e..f815b942fe 100644 --- a/libraries/plugins/tags/tags_plugin.cpp +++ b/libraries/plugins/tags/tags_plugin.cpp @@ -234,45 +234,72 @@ struct operation_visitor { } /** finds tags that have been added or removed or updated */ - void update_tags( const comment_object& c )const { + void update_tags( const comment_object& c, bool parse_tags = false )const + { try { auto hot = calculate_hot( c.net_rshares, c.created ); auto trending = calculate_trending( c.net_rshares, c.created ); - auto meta = filter_tags( c ); + const auto& comment_idx = _db.get_index< tag_index >().indices().get< by_comment >(); - auto citr = comment_idx.lower_bound( c.id ); - - map< string, const tag_object* > existing_tags; - vector< const tag_object* > remove_queue; - while( citr != comment_idx.end() && citr->comment == c.id ) { - const tag_object* tag = &*citr; - ++citr; - if( meta.tags.find( tag->tag ) == meta.tags.end() ) { - remove_queue.push_back(tag); - } else { - existing_tags[tag->tag] = tag; + + if( parse_tags ) + { + auto meta = filter_tags( c ); + auto citr = comment_idx.lower_bound( c.id ); + + map< string, const tag_object* > existing_tags; + vector< const tag_object* > remove_queue; + + while( citr != comment_idx.end() && citr->comment == c.id ) + { + const tag_object* tag = &*citr; + ++citr; + + if( meta.tags.find( tag->tag ) == meta.tags.end() ) + { + remove_queue.push_back(tag); + } + else + { + existing_tags[tag->tag] = tag; + } } - } - for( const auto& tag : meta.tags ) { - auto existing = existing_tags.find(tag); - if( existing == existing_tags.end() ) { - create_tag( tag, c, hot, trending ); - } else { - update_tag( *existing->second, c, hot, trending ); + for( const auto& tag : meta.tags ) + { + auto existing = existing_tags.find(tag); + + if( existing == existing_tags.end() ) + { + create_tag( tag, c, hot, trending ); + } + else + { + update_tag( *existing->second, c, hot, trending ); + } } + + for( const auto& item : remove_queue ) + remove_tag(*item); } + else + { + auto citr = comment_idx.lower_bound( c.id ); - for( const auto& item : remove_queue ) - remove_tag(*item); + while( citr != comment_idx.end() && citr->comment == c.id ) + { + update_tag( *citr, c, hot, trending ); + ++citr; + } + } if( c.parent_author.size() ) { update_tags( _db.get_comment( c.parent_author, c.parent_permlink ) ); } - } FC_CAPTURE_LOG_AND_RETHROW( (c) ) - } + } FC_CAPTURE_LOG_AND_RETHROW( (c) ) +} const peer_stats_object& get_or_create_peer_stats( account_id_type voter, account_id_type peer )const { const auto& peeridx = _db.get_index().indices().get(); @@ -321,7 +348,7 @@ struct operation_visitor { } void operator()( const comment_operation& op )const { - update_tags( _db.get_comment( op.author, op.permlink ) ); + update_tags( _db.get_comment( op.author, op.permlink ), true ); } void operator()( const transfer_operation& op )const { From 9eef62c8e7be98d779b96bcbab7863ea52e66d46 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Tue, 6 Jun 2017 15:07:04 -0400 Subject: [PATCH 43/59] Allow the follow plugin to delay calculating feeds to a later date. #1162 --- contrib/startpaassteemd.sh | 8 +- contrib/steemd.run | 6 ++ .../plugins/follow/follow_evaluators.cpp | 77 ++++++++++--------- libraries/plugins/follow/follow_plugin.cpp | 55 +++++++------ .../include/steemit/follow/follow_plugin.hpp | 1 + 5 files changed, 86 insertions(+), 61 deletions(-) diff --git a/contrib/startpaassteemd.sh b/contrib/startpaassteemd.sh index e0855f8417..29f33be232 100644 --- a/contrib/startpaassteemd.sh +++ b/contrib/startpaassteemd.sh @@ -31,6 +31,12 @@ if [[ ! -z "$STEEMD_SEED_NODES" ]]; then done fi +NOW=`date +%s` +STEEMD_FEED_START_TIME=`expr $NOW - 1209600` +STEEMD_FEED_START_TIMESTAMP=`date --date=@$STEEMD_FEED_START_TIME +%Y-%m-%d:%H:%M:%S` + +ARGS+=" --follow-start-feeds=\"$STEEMD_FEED_START_TIMESTAMP\"" + # overwrite local config with image one cp /etc/steemd/fullnode.config.ini $HOME/config.ini @@ -49,7 +55,7 @@ if [[ "$USE_RAMDISK" ]]; then ARGS+=" --shared-file-dir=/mnt/ramdisk/blockchain" s3cmd get s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x --wildcards 'blockchain/block*' -C /mnt/ramdisk 'blockchain/shared*' chown -R steemd:steemd /mnt/ramdisk/blockchain -else +else s3cmd get s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x fi if [[ $? -ne 0 ]]; then diff --git a/contrib/steemd.run b/contrib/steemd.run index 27c1ac0340..428cc64994 100644 --- a/contrib/steemd.run +++ b/contrib/steemd.run @@ -45,6 +45,12 @@ if [[ ! -z "$STEEMD_PRIVATE_KEY" ]]; then ARGS+=" --private-key=$STEEMD_PRIVATE_KEY" fi +NOW=`date +%s` +STEEMD_FEED_START_TIME=`expr $NOW - 1209600` +STEEMD_FEED_START_TIMESTAMP=`date --date=@$STEEMD_FEED_START_TIME +%Y-%m-%d:%H:%M:%S` + +ARGS+=" --follow-start-feeds=\"$STEEMD_FEED_START_TIMESTAMP\"" + # overwrite local config with image one if [[ "$USE_FULL_WEB_NODE" ]]; then cp /etc/steemd/fullnode.config.ini $HOME/config.ini diff --git a/libraries/plugins/follow/follow_evaluators.cpp b/libraries/plugins/follow/follow_evaluators.cpp index 0580cfd380..d31d01f33a 100644 --- a/libraries/plugins/follow/follow_evaluators.cpp +++ b/libraries/plugins/follow/follow_evaluators.cpp @@ -165,54 +165,57 @@ void reblog_evaluator::do_apply( const reblog_operation& o ) const auto& idx = db.get_index< follow_index >().indices().get< by_following_follower >(); auto itr = idx.find( o.account ); - while( itr != idx.end() && itr->following == o.account ) + if( db.head_block_time() >= _plugin->start_feeds ) { - - if( itr->what & ( 1 << blog ) ) + while( itr != idx.end() && itr->following == o.account ) { - uint32_t next_id = 0; - auto last_feed = feed_idx.lower_bound( itr->follower ); - if( last_feed != feed_idx.end() && last_feed->account == itr->follower ) + if( itr->what & ( 1 << blog ) ) { - next_id = last_feed->account_feed_id + 1; - } + uint32_t next_id = 0; + auto last_feed = feed_idx.lower_bound( itr->follower ); - auto feed_itr = comment_idx.find( boost::make_tuple( c.id, itr->follower ) ); + if( last_feed != feed_idx.end() && last_feed->account == itr->follower ) + { + next_id = last_feed->account_feed_id + 1; + } - if( feed_itr == comment_idx.end() ) - { - db.create< feed_object >( [&]( feed_object& f ) + auto feed_itr = comment_idx.find( boost::make_tuple( c.id, itr->follower ) ); + + if( feed_itr == comment_idx.end() ) { - f.account = itr->follower; - f.reblogged_by.push_back( o.account ); - f.first_reblogged_by = o.account; - f.first_reblogged_on = db.head_block_time(); - f.comment = c.id; - f.reblogs = 1; - f.account_feed_id = next_id; - }); - } - else - { - db.modify( *feed_itr, [&]( feed_object& f ) + db.create< feed_object >( [&]( feed_object& f ) + { + f.account = itr->follower; + f.reblogged_by.push_back( o.account ); + f.first_reblogged_by = o.account; + f.first_reblogged_on = db.head_block_time(); + f.comment = c.id; + f.reblogs = 1; + f.account_feed_id = next_id; + }); + } + else { - f.reblogged_by.push_back( o.account ); - f.reblogs++; - }); - } + db.modify( *feed_itr, [&]( feed_object& f ) + { + f.reblogged_by.push_back( o.account ); + f.reblogs++; + }); + } - const auto& old_feed_idx = db.get_index< feed_index >().indices().get< by_old_feed >(); - auto old_feed = old_feed_idx.lower_bound( itr->follower ); + const auto& old_feed_idx = db.get_index< feed_index >().indices().get< by_old_feed >(); + auto old_feed = old_feed_idx.lower_bound( itr->follower ); - while( old_feed->account == itr->follower && next_id - old_feed->account_feed_id > _plugin->max_feed_size ) - { - db.remove( *old_feed ); - old_feed = old_feed_idx.lower_bound( itr->follower ); - }; - } + while( old_feed->account == itr->follower && next_id - old_feed->account_feed_id > _plugin->max_feed_size ) + { + db.remove( *old_feed ); + old_feed = old_feed_idx.lower_bound( itr->follower ); + }; + } - ++itr; + ++itr; + } } } FC_CAPTURE_AND_RETHROW( (o) ) diff --git a/libraries/plugins/follow/follow_plugin.cpp b/libraries/plugins/follow/follow_plugin.cpp index 98f1460876..e8ab44c930 100644 --- a/libraries/plugins/follow/follow_plugin.cpp +++ b/libraries/plugins/follow/follow_plugin.cpp @@ -194,39 +194,42 @@ struct post_operation_visitor const auto& feed_idx = db.get_index< feed_index >().indices().get< by_feed >(); - while( itr != idx.end() && itr->following == op.author ) + if( db.head_block_time() >= _plugin.start_feeds ) { - if( itr->what & ( 1 << blog ) ) + while( itr != idx.end() && itr->following == op.author ) { - uint32_t next_id = 0; - auto last_feed = feed_idx.lower_bound( itr->follower ); - - if( last_feed != feed_idx.end() && last_feed->account == itr->follower ) + if( itr->what & ( 1 << blog ) ) { - next_id = last_feed->account_feed_id + 1; - } + uint32_t next_id = 0; + auto last_feed = feed_idx.lower_bound( itr->follower ); - if( comment_idx.find( boost::make_tuple( c.id, itr->follower ) ) == comment_idx.end() ) - { - db.create< feed_object >( [&]( feed_object& f ) + if( last_feed != feed_idx.end() && last_feed->account == itr->follower ) { - f.account = itr->follower; - f.comment = c.id; - f.account_feed_id = next_id; - }); - - const auto& old_feed_idx = db.get_index< feed_index >().indices().get< by_old_feed >(); - auto old_feed = old_feed_idx.lower_bound( itr->follower ); + next_id = last_feed->account_feed_id + 1; + } - while( old_feed->account == itr->follower && next_id - old_feed->account_feed_id > _plugin.max_feed_size ) + if( comment_idx.find( boost::make_tuple( c.id, itr->follower ) ) == comment_idx.end() ) { - db.remove( *old_feed ); - old_feed = old_feed_idx.lower_bound( itr->follower ); + db.create< feed_object >( [&]( feed_object& f ) + { + f.account = itr->follower; + f.comment = c.id; + f.account_feed_id = next_id; + }); + + const auto& old_feed_idx = db.get_index< feed_index >().indices().get< by_old_feed >(); + auto old_feed = old_feed_idx.lower_bound( itr->follower ); + + while( old_feed->account == itr->follower && next_id - old_feed->account_feed_id > _plugin.max_feed_size ) + { + db.remove( *old_feed ); + old_feed = old_feed_idx.lower_bound( itr->follower ); + } } } - } - ++itr; + ++itr; + } } const auto& blog_idx = db.get_index< blog_index >().indices().get< by_blog >(); @@ -345,6 +348,7 @@ void follow_plugin::plugin_set_program_options( { cli.add_options() ("follow-max-feed-size", boost::program_options::value< uint32_t >()->default_value( 500 ), "Set the maximum size of cached feed for an account" ) + ("follow-start-feeds", boost::program_options::value< uint32_t >()->default_value( 0 ), "Block time (in epoch seconds) when to start calculating feeds" ) ; cfg.add( cli ); } @@ -371,6 +375,11 @@ void follow_plugin::plugin_initialize( const boost::program_options::variables_m uint32_t feed_size = options[ "follow-max-feed-size" ].as< uint32_t >(); max_feed_size = feed_size; } + + if( options.count( "follow-start-feeds" ) ) + { + start_feeds = fc::time_point_sec( options[ "follow-start-feeds" ].as< uint32_t >() ); + } } FC_CAPTURE_AND_RETHROW() } diff --git a/libraries/plugins/follow/include/steemit/follow/follow_plugin.hpp b/libraries/plugins/follow/include/steemit/follow/follow_plugin.hpp index ae91de0746..3f3c4ff35b 100644 --- a/libraries/plugins/follow/include/steemit/follow/follow_plugin.hpp +++ b/libraries/plugins/follow/include/steemit/follow/follow_plugin.hpp @@ -28,6 +28,7 @@ class follow_plugin : public steemit::app::plugin friend class detail::follow_plugin_impl; std::unique_ptr my; uint32_t max_feed_size = 500; + fc::time_point_sec start_feeds; }; } } //steemit::follow From 87a55c6e7dc695d3440c8e938b8d81e1b0dad7bd Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Tue, 6 Jun 2017 15:12:42 -0400 Subject: [PATCH 44/59] Build with SKIP_BY_TX_ID in Dockerfile and add documentation on cmake option #1160 --- Dockerfile | 4 ++++ doc/building.md | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/Dockerfile b/Dockerfile index e9b19298b9..e1ea159d29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,7 @@ RUN \ -DBUILD_STEEM_TESTNET=ON \ -DLOW_MEMORY_NODE=OFF \ -DCLEAR_VOTES=ON \ + -DSKIP_BY_TX_ID=ON \ .. && \ make -j$(nproc) chain_test test_fixed_string && \ ./tests/chain_test && \ @@ -70,6 +71,7 @@ RUN \ -DBUILD_STEEM_TESTNET=ON \ -DLOW_MEMORY_NODE=OFF \ -DCLEAR_VOTES=ON \ + -DSKIP_BY_TX_ID=ON \ -DCHAINBASE_CHECK_LOCKING=OFF \ .. && \ make -j$(nproc) chain_test && \ @@ -89,6 +91,7 @@ RUN \ -DCMAKE_BUILD_TYPE=Release \ -DLOW_MEMORY_NODE=ON \ -DCLEAR_VOTES=ON \ + -DSKIP_BY_TX_ID=ON \ -DBUILD_STEEM_TESTNET=OFF \ .. \ && \ @@ -110,6 +113,7 @@ RUN \ -DCMAKE_BUILD_TYPE=Release \ -DLOW_MEMORY_NODE=OFF \ -DCLEAR_VOTES=OFF \ + -DSKIP_BY_TX_ID=ON \ -DBUILD_STEEM_TESTNET=OFF \ .. \ && \ diff --git a/doc/building.md b/doc/building.md index 4fd41653ae..7cd8ac57c6 100644 --- a/doc/building.md +++ b/doc/building.md @@ -22,6 +22,12 @@ Clears old votes from memory that are no longer required for consensus. Builds steem for use in a private testnet. Also required for building unit tests. +### SKIP_BY_TX_ID=[OFF/ON] + +The account history plugin does not allow querying of operations by transaction id. +This requires around 65% of CPU time when reindexing so disabling the index is a +huge gain if you do not need this functionality. + ## Building under Docker We ship a Dockerfile. This builds both common node type binaries. From f62d92f3f46e58706e242ce3317ddc5a40bf8021 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Wed, 7 Jun 2017 15:43:38 -0400 Subject: [PATCH 45/59] Iterate by reference #1161 --- libraries/plugins/tags/tags_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/plugins/tags/tags_plugin.cpp b/libraries/plugins/tags/tags_plugin.cpp index f815b942fe..c2019d1321 100644 --- a/libraries/plugins/tags/tags_plugin.cpp +++ b/libraries/plugins/tags/tags_plugin.cpp @@ -409,7 +409,7 @@ struct operation_visitor { auto meta = filter_tags( c ); - for( auto tag : meta.tags ) + for( const string& tag : meta.tags ) { _db.modify( get_stats( tag ), [&]( tag_stats_object& ts ) { From 9f7213be373c0cb657c94a0407ae77a5f294389f Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Wed, 7 Jun 2017 15:54:13 -0400 Subject: [PATCH 46/59] Add some clarification #1161 --- libraries/plugins/tags/tags_plugin.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/plugins/tags/tags_plugin.cpp b/libraries/plugins/tags/tags_plugin.cpp index c2019d1321..32ba066af1 100644 --- a/libraries/plugins/tags/tags_plugin.cpp +++ b/libraries/plugins/tags/tags_plugin.cpp @@ -116,11 +116,14 @@ struct operation_visitor { } } + // We need to write the transformed tags into a temporary + // local container because we can't modify meta.tags concurrently + // as we iterate over it. set< string > lower_tags; uint8_t tag_limit = 5; uint8_t count = 0; - for( const auto& tag : meta.tags ) + for( const string& tag : meta.tags ) { ++count; if( count > tag_limit || lower_tags.size() > tag_limit ) @@ -407,7 +410,7 @@ struct operation_visitor { const auto& c = _db.get_comment( op.author, op.permlink ); update_tags( c ); - auto meta = filter_tags( c ); + comment_metadata meta = filter_tags( c ); for( const string& tag : meta.tags ) { From 915c0610f72ef4240dd3ead60c2bf7a40df2a77a Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Wed, 7 Jun 2017 15:36:28 -0400 Subject: [PATCH 47/59] Fix whitespace in tags_plugin.cpp #1161 --- libraries/plugins/tags/tags_plugin.cpp | 167 ++++++++++++++++--------- 1 file changed, 108 insertions(+), 59 deletions(-) diff --git a/libraries/plugins/tags/tags_plugin.cpp b/libraries/plugins/tags/tags_plugin.cpp index 32ba066af1..cb9deef087 100644 --- a/libraries/plugins/tags/tags_plugin.cpp +++ b/libraries/plugins/tags/tags_plugin.cpp @@ -48,28 +48,40 @@ tags_plugin_impl::~tags_plugin_impl() return; } -struct operation_visitor { +struct operation_visitor +{ operation_visitor( database& db ):_db(db){}; typedef void result_type; database& _db; - void remove_stats( const tag_object& tag, const tag_stats_object& stats )const { - _db.modify( stats, [&]( tag_stats_object& s ) { - if( tag.parent == comment_id_type() ) { + void remove_stats( const tag_object& tag, const tag_stats_object& stats )const + { + _db.modify( stats, [&]( tag_stats_object& s ) + { + if( tag.parent == comment_id_type() ) + { s.top_posts--; - } else { + } + else + { s.comments--; } s.total_trending -= static_cast(tag.trending); s.net_votes -= tag.net_votes; }); } - void add_stats( const tag_object& tag, const tag_stats_object& stats )const { - _db.modify( stats, [&]( tag_stats_object& s ) { - if( tag.parent == comment_id_type() ) { + + void add_stats( const tag_object& tag, const tag_stats_object& stats )const + { + _db.modify( stats, [&]( tag_stats_object& s ) + { + if( tag.parent == comment_id_type() ) + { s.top_posts++; - } else { + } + else + { s.comments++; } s.total_trending += static_cast(tag.trending); @@ -77,27 +89,33 @@ struct operation_visitor { }); } - void remove_tag( const tag_object& tag )const { + void remove_tag( const tag_object& tag )const + { /// TODO: update tag stats object _db.remove(tag); const auto& idx = _db.get_index().indices().get(); auto itr = idx.lower_bound( boost::make_tuple(tag.author,tag.tag) ); - if( itr != idx.end() && itr->author == tag.author && itr->tag == tag.tag ) { - _db.modify( *itr, [&]( author_tag_stats_object& stats ) { + if( itr != idx.end() && itr->author == tag.author && itr->tag == tag.tag ) + { + _db.modify( *itr, [&]( author_tag_stats_object& stats ) + { stats.total_posts--; }); } } - const tag_stats_object& get_stats( const string& tag )const { + const tag_stats_object& get_stats( const string& tag )const + { const auto& stats_idx = _db.get_index().indices().get(); auto itr = stats_idx.find( tag ); - if( itr != stats_idx.end() ) return *itr; + if( itr != stats_idx.end() ) + return *itr; - return _db.create( [&]( tag_stats_object& stats ) { - stats.tag = tag; - }); + return _db.create( [&]( tag_stats_object& stats ) + { + stats.tag = tag; + }); } comment_metadata filter_tags( const comment_object& c ) const @@ -168,14 +186,15 @@ struct operation_visitor { } void create_tag( const string& tag, const comment_object& comment, double hot, double trending )const -{ + { comment_id_type parent; account_id_type author = _db.get_account( comment.author ).id; if( comment.parent_author.size() ) parent = _db.get_comment( comment.parent_author, comment.parent_permlink ).id; - const auto& tag_obj = _db.create( [&]( tag_object& obj ) { + const auto& tag_obj = _db.create( [&]( tag_object& obj ) + { obj.tag = tag; obj.comment = comment.id; obj.parent = parent; @@ -194,12 +213,17 @@ struct operation_visitor { const auto& idx = _db.get_index().indices().get(); auto itr = idx.lower_bound( boost::make_tuple(author,tag) ); - if( itr != idx.end() && itr->author == author && itr->tag == tag ) { - _db.modify( *itr, [&]( author_tag_stats_object& stats ) { + if( itr != idx.end() && itr->author == author && itr->tag == tag ) + { + _db.modify( *itr, [&]( author_tag_stats_object& stats ) + { stats.total_posts++; }); - } else { - _db.create( [&]( author_tag_stats_object& stats ) { + } + else + { + _db.create( [&]( author_tag_stats_object& stats ) + { stats.author = author; stats.tag = tag; stats.total_posts = 1; @@ -301,13 +325,15 @@ struct operation_visitor { { update_tags( _db.get_comment( c.parent_author, c.parent_permlink ) ); } - } FC_CAPTURE_LOG_AND_RETHROW( (c) ) -} + } FC_CAPTURE_LOG_AND_RETHROW( (c) ) + } - const peer_stats_object& get_or_create_peer_stats( account_id_type voter, account_id_type peer )const { + const peer_stats_object& get_or_create_peer_stats( account_id_type voter, account_id_type peer )const + { const auto& peeridx = _db.get_index().indices().get(); auto itr = peeridx.find( boost::make_tuple( voter, peer ) ); - if( itr == peeridx.end() ) { + if( itr == peeridx.end() ) + { return _db.create( [&]( peer_stats_object& obj ) { obj.voter = voter; obj.peer = peer; @@ -315,73 +341,91 @@ struct operation_visitor { } return *itr; } - void update_indirect_vote( account_id_type a, account_id_type b, int positive )const { - if( a == b ) return; + + void update_indirect_vote( account_id_type a, account_id_type b, int positive )const + { + if( a == b ) + return; const auto& ab = get_or_create_peer_stats( a, b ); const auto& ba = get_or_create_peer_stats( b, a ); - _db.modify( ab, [&]( peer_stats_object& o ) { - o.indirect_positive_votes += positive; - o.indirect_votes++; - o.update_rank(); - }); - _db.modify( ba, [&]( peer_stats_object& o ) { - o.indirect_positive_votes += positive; - o.indirect_votes++; - o.update_rank(); - }); + _db.modify( ab, [&]( peer_stats_object& o ) + { + o.indirect_positive_votes += positive; + o.indirect_votes++; + o.update_rank(); + }); + _db.modify( ba, [&]( peer_stats_object& o ) + { + o.indirect_positive_votes += positive; + o.indirect_votes++; + o.update_rank(); + }); } - void update_peer_stats( const account_object& voter, const account_object& author, const comment_object& c, int vote )const { + void update_peer_stats( const account_object& voter, const account_object& author, const comment_object& c, int vote )const + { if( voter.id == author.id ) return; /// ignore votes for yourself if( c.parent_author.size() ) return; /// only count top level posts const auto& stat = get_or_create_peer_stats( voter.id, author.id ); - _db.modify( stat, [&]( peer_stats_object& obj ) { - obj.direct_votes++; - obj.direct_positive_votes += vote > 0; - obj.update_rank(); + _db.modify( stat, [&]( peer_stats_object& obj ) + { + obj.direct_votes++; + obj.direct_positive_votes += vote > 0; + obj.update_rank(); }); const auto& voteidx = _db.get_index().indices().get(); auto itr = voteidx.lower_bound( boost::make_tuple( comment_id_type(c.id), account_id_type() ) ); - while( itr != voteidx.end() && itr->comment == c.id ) { + while( itr != voteidx.end() && itr->comment == c.id ) + { update_indirect_vote( voter.id, itr->voter, (itr->vote_percent > 0) == (vote > 0) ); ++itr; } } - void operator()( const comment_operation& op )const { + void operator()( const comment_operation& op )const + { update_tags( _db.get_comment( op.author, op.permlink ), true ); } - void operator()( const transfer_operation& op )const { - if( op.to == STEEMIT_NULL_ACCOUNT && op.amount.symbol == SBD_SYMBOL ) { + void operator()( const transfer_operation& op )const + { + if( op.to == STEEMIT_NULL_ACCOUNT && op.amount.symbol == SBD_SYMBOL ) + { vector part; part.reserve(4); auto path = op.memo; boost::split( part, path, boost::is_any_of("/") ); - if( part[0].size() && part[0][0] == '@' ) { + if( part[0].size() && part[0][0] == '@' ) + { auto acnt = part[0].substr(1); auto perm = part[1]; auto c = _db.find_comment( acnt, perm ); - if( c && c->parent_author.size() == 0 ) { + if( c && c->parent_author.size() == 0 ) + { const auto& comment_idx = _db.get_index().indices().get(); auto citr = comment_idx.lower_bound( c->id ); - while( citr != comment_idx.end() && citr->comment == c->id ) { - _db.modify( *citr, [&]( tag_object& t ) { + while( citr != comment_idx.end() && citr->comment == c->id ) + { + _db.modify( *citr, [&]( tag_object& t ) + { if( t.cashout != fc::time_point_sec::maximum() ) t.promoted_balance += op.amount.amount; }); ++citr; } - } else { + } + else + { ilog( "unable to find body" ); } } } } - void operator()( const vote_operation& op )const { + void operator()( const vote_operation& op )const + { update_tags( _db.get_comment( op.author, op.permlink ) ); /* update_peer_stats( _db.get_account(op.voter), @@ -391,22 +435,26 @@ struct operation_visitor { */ } - void operator()( const delete_comment_operation& op )const { + void operator()( const delete_comment_operation& op )const + { const auto& idx = _db.get_index().indices().get(); const auto& auth = _db.get_account(op.author); auto itr = idx.lower_bound( boost::make_tuple( auth.id ) ); - while( itr != idx.end() && itr->author == auth.id ) { + while( itr != idx.end() && itr->author == auth.id ) + { const auto& tobj = *itr; const auto* obj = _db.find< comment_object >( itr->comment ); ++itr; - if( !obj ) { + if( !obj ) + { _db.remove( tobj ); } } } - void operator()( const comment_reward_operation& op )const { + void operator()( const comment_reward_operation& op )const + { const auto& c = _db.get_comment( op.author, op.permlink ); update_tags( c ); @@ -421,7 +469,8 @@ struct operation_visitor { } } - void operator()( const comment_payout_update_operation& op )const { + void operator()( const comment_payout_update_operation& op )const + { const auto& c = _db.get_comment( op.author, op.permlink ); update_tags( c ); } From 2705fcd2fad92d5eac02c0b72dd2446886a65697 Mon Sep 17 00:00:00 2001 From: JustinW Date: Tue, 6 Jun 2017 16:47:21 -0400 Subject: [PATCH 48/59] Enable single account history tracking on consensus nodes --- contrib/config-for-docker.ini | 2 +- contrib/steemd.run | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/config-for-docker.ini b/contrib/config-for-docker.ini index 3b2602636f..462f8efb98 100644 --- a/contrib/config-for-docker.ini +++ b/contrib/config-for-docker.ini @@ -38,7 +38,7 @@ enable-plugin = witness account_history bcd-trigger = [[0,10],[85,300]] # Defines a range of accounts to track as a json pair ["from","to"] [from,to] -# track-account-range = +track-account-range = ["",""] # Ignore posting operations, only track transfers and account updates # filter-posting-ops = diff --git a/contrib/steemd.run b/contrib/steemd.run index 428cc64994..5a22516d13 100644 --- a/contrib/steemd.run +++ b/contrib/steemd.run @@ -45,6 +45,10 @@ if [[ ! -z "$STEEMD_PRIVATE_KEY" ]]; then ARGS+=" --private-key=$STEEMD_PRIVATE_KEY" fi +if [[ ! -z "$TRACK_ACCOUNT" ]]; then + ARGS+=" --track-account-range=[\"$TRACK_ACCOUNT\",\"$TRACK_ACCOUNT\"]" +fi + NOW=`date +%s` STEEMD_FEED_START_TIME=`expr $NOW - 1209600` STEEMD_FEED_START_TIMESTAMP=`date --date=@$STEEMD_FEED_START_TIME +%Y-%m-%d:%H:%M:%S` From 2e0057a41946adb4e0321043dfc07aa255e57bd0 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Wed, 7 Jun 2017 16:23:46 -0400 Subject: [PATCH 49/59] Optimize plugin loading for consensus only node --- contrib/config-for-docker.ini | 4 ++-- contrib/steemd.run | 7 +++++-- doc/quickstart.md | 6 ++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/contrib/config-for-docker.ini b/contrib/config-for-docker.ini index 462f8efb98..ff2eb8dba7 100644 --- a/contrib/config-for-docker.ini +++ b/contrib/config-for-docker.ini @@ -32,13 +32,13 @@ public-api = database_api login_api # Plugin(s) to enable, may be specified multiple times -enable-plugin = witness account_history +enable-plugin = witness # JSON list of [nblocks,nseconds] pairs, see doc/bcd-trigger.md bcd-trigger = [[0,10],[85,300]] # Defines a range of accounts to track as a json pair ["from","to"] [from,to] -track-account-range = ["",""] +# track-account-range = # Ignore posting operations, only track transfers and account updates # filter-posting-ops = diff --git a/contrib/steemd.run b/contrib/steemd.run index 5a22516d13..22be7ffb3b 100644 --- a/contrib/steemd.run +++ b/contrib/steemd.run @@ -26,7 +26,7 @@ fi # if user did pass in desired seed nodes, use # the ones the user specified: -if [[ ! -z "$STEEMD_SEED_NODES" ]]; then +if [[ ! -z "$STEEMD_SEED_NODES" ]]; then for NODE in $STEEMD_SEED_NODES ; do ARGS+=" --seed-node=$NODE" done @@ -46,7 +46,10 @@ if [[ ! -z "$STEEMD_PRIVATE_KEY" ]]; then fi if [[ ! -z "$TRACK_ACCOUNT" ]]; then - ARGS+=" --track-account-range=[\"$TRACK_ACCOUNT\",\"$TRACK_ACCOUNT\"]" + if [[ ! "$USE_WAY_TOO_MUCH_RAM" ]]; then + ARGS+=" --enable-plugin=witness account_history" + fi + ARGS+=" --track-account-range=[\"$TRACK_ACCOUNT\",\"$TRACK_ACCOUNT\"]" fi NOW=`date +%s` diff --git a/doc/quickstart.md b/doc/quickstart.md index 2931ef497d..62dfeff51c 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -37,6 +37,12 @@ track-account-range = ["yourexchangeid", "yourexchangeid"] ``` Do not add other APIs or plugins unless you know what you are doing. +This configuration exists in Docker with the following command + +``` +docker run -d --env TRACK_ACCOUNT="yourexchangeid" steemit/steem +``` + ### Resources usage Please make sure that you have enough resources available. From d9eb91a94800d402856808278670a31d3549d4ed Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Thu, 8 Jun 2017 09:35:57 -0400 Subject: [PATCH 50/59] Update Docker scripts to use epoch time rather than timestamp #1162 --- contrib/startpaassteemd.sh | 3 +-- contrib/steemd.run | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/contrib/startpaassteemd.sh b/contrib/startpaassteemd.sh index 29f33be232..b898c0a623 100644 --- a/contrib/startpaassteemd.sh +++ b/contrib/startpaassteemd.sh @@ -33,9 +33,8 @@ fi NOW=`date +%s` STEEMD_FEED_START_TIME=`expr $NOW - 1209600` -STEEMD_FEED_START_TIMESTAMP=`date --date=@$STEEMD_FEED_START_TIME +%Y-%m-%d:%H:%M:%S` -ARGS+=" --follow-start-feeds=\"$STEEMD_FEED_START_TIMESTAMP\"" +ARGS+=" --follow-start-feeds=$STEEMD_FEED_START_TIME" # overwrite local config with image one cp /etc/steemd/fullnode.config.ini $HOME/config.ini diff --git a/contrib/steemd.run b/contrib/steemd.run index 22be7ffb3b..d2ebfd37ea 100644 --- a/contrib/steemd.run +++ b/contrib/steemd.run @@ -54,9 +54,8 @@ fi NOW=`date +%s` STEEMD_FEED_START_TIME=`expr $NOW - 1209600` -STEEMD_FEED_START_TIMESTAMP=`date --date=@$STEEMD_FEED_START_TIME +%Y-%m-%d:%H:%M:%S` -ARGS+=" --follow-start-feeds=\"$STEEMD_FEED_START_TIMESTAMP\"" +ARGS+=" --follow-start-feeds=$STEEMD_FEED_START_TIME" # overwrite local config with image one if [[ "$USE_FULL_WEB_NODE" ]]; then From 0ecfcd6bcbf5770844737ebfe7707d327a136e99 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Thu, 8 Jun 2017 11:53:29 -0400 Subject: [PATCH 51/59] Add "--disable-get-block" option #1172 --- contrib/startpaassteemd.sh | 2 ++ libraries/app/application.cpp | 4 ++++ libraries/app/database_api.cpp | 8 ++++++++ libraries/app/include/steemit/app/application.hpp | 1 + 4 files changed, 15 insertions(+) diff --git a/contrib/startpaassteemd.sh b/contrib/startpaassteemd.sh index 29f33be232..413284616c 100644 --- a/contrib/startpaassteemd.sh +++ b/contrib/startpaassteemd.sh @@ -37,6 +37,8 @@ STEEMD_FEED_START_TIMESTAMP=`date --date=@$STEEMD_FEED_START_TIME +%Y-%m-%d:%H:% ARGS+=" --follow-start-feeds=\"$STEEMD_FEED_START_TIMESTAMP\"" +ARGS+=" --disable-get-block" + # overwrite local config with image one cp /etc/steemd/fullnode.config.ini $HOME/config.ini diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index d744670682..b457f44a82 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -265,6 +265,9 @@ namespace detail { else _shared_dir = _data_dir / "blockchain"; + if( _options->count( "disable_get_block" ) ) + _self->_disable_get_block = true; + if( !read_only ) { _self->_read_only = false; @@ -989,6 +992,7 @@ void application::set_program_options(boost::program_options::options_descriptio ("force-validate", "Force validation of all transactions") ("read-only", "Node will not connect to p2p network and can only read from the chain state" ) ("check-locks", "Check correctness of chainbase locking") + ("disable-get-block", "Disable get_block API call" ) ; command_line_options.add(_cli_options); configuration_file_options.add(_cfg_options); diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp index 59ffb4b78a..027bc69238 100755 --- a/libraries/app/database_api.cpp +++ b/libraries/app/database_api.cpp @@ -86,6 +86,8 @@ class database_api_impl : public std::enable_shared_from_this std::shared_ptr< steemit::follow::follow_api > _follow_api; boost::signals2::scoped_connection _block_applied_connection; + + bool _disable_get_block = false; }; applied_operation::applied_operation() {} @@ -202,6 +204,8 @@ database_api_impl::database_api_impl( const steemit::app::api_context& ctx ) { wlog("creating database api ${x}", ("x",int64_t(this)) ); + _disable_get_block = ctx.app._disable_get_block; + try { ctx.app.get_plugin< follow::follow_plugin >( FOLLOW_PLUGIN_NAME ); @@ -225,6 +229,8 @@ void database_api::on_api_startup() {} optional database_api::get_block_header(uint32_t block_num)const { + FC_ASSERT( !my->_disable_get_block, "get_block_header is disabled on this node." ); + return my->_db.with_read_lock( [&]() { return my->get_block_header( block_num ); @@ -241,6 +247,8 @@ optional database_api_impl::get_block_header(uint32_t block_num) c optional database_api::get_block(uint32_t block_num)const { + FC_ASSERT( !my->_disable_get_block, "get_block is disabled on this node." ); + return my->_db.with_read_lock( [&]() { return my->get_block( block_num ); diff --git a/libraries/app/include/steemit/app/application.hpp b/libraries/app/include/steemit/app/application.hpp index 10c8b46084..1c8b3a29fd 100644 --- a/libraries/app/include/steemit/app/application.hpp +++ b/libraries/app/include/steemit/app/application.hpp @@ -124,6 +124,7 @@ namespace steemit { namespace app { void connect_to_write_node(); bool _read_only = true; + bool _disable_get_block = false; fc::optional< string > _remote_endpoint; fc::optional< fc::api< network_broadcast_api > > _remote_net_api; fc::optional< fc::api< login_api > > _remote_login; From 5c95f1daaaa62f9e024164e20febc7c41477ce27 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Thu, 8 Jun 2017 14:37:44 -0400 Subject: [PATCH 52/59] Refactor walk_main_branch_to_num() into separate method #1175 --- libraries/chain/database.cpp | 29 ++----------------- libraries/chain/fork_database.cpp | 21 ++++++++++++++ .../include/steemit/chain/fork_database.hpp | 3 ++ 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index c48a327bae..808fd65711 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -3075,32 +3075,9 @@ void database::update_last_irreversible_block() { while( log_head_num < dpo.last_irreversible_block_num ) { - signed_block* block_ptr; - auto blocks = _fork_db.fetch_block_by_number( log_head_num + 1 ); - - if( blocks.size() == 1 ) - block_ptr = &( blocks[0]->data ); - else - { - vector< std::pair< account_name_type, fc::time_point_sec > > witness_time_pairs; - for( const auto& b : blocks ) - { - witness_time_pairs.push_back( std::make_pair( b->data.witness, b->data.timestamp ) ); - } - - ilog( "Encountered a block num collision due to a fork. Walking the current fork to determine the correct block. block_num:${n}", ("n", log_head_num + 1) ); - ilog( "Colliding blocks produced by witnesses at times: ${w}", ("w", witness_time_pairs) ); - - auto next = _fork_db.head(); - while( next.get() != nullptr && next->num > log_head_num + 1 ) - next = next->prev.lock(); - - FC_ASSERT( next.get() != nullptr, "Current fork in the fork database does not contain the last_irreversible_block" ); - - block_ptr = &( next->data ); - } - - _block_log.append( *block_ptr ); + shared_ptr< fork_item > block = _fork_db.fetch_block_on_main_branch_by_number( log_head_num+1 ); + FC_ASSERT( block, "Current fork in the fork database does not contain the last_irreversible_block" ); + _block_log.append( block->data ); log_head_num++; } diff --git a/libraries/chain/fork_database.cpp b/libraries/chain/fork_database.cpp index 3a5d9b1b07..96cf016456 100644 --- a/libraries/chain/fork_database.cpp +++ b/libraries/chain/fork_database.cpp @@ -209,6 +209,27 @@ pair return result; } FC_CAPTURE_AND_RETHROW( (first)(second) ) } +shared_ptr fork_database::walk_main_branch_to_num( uint32_t block_num )const +{ + shared_ptr next = head(); + if( block_num > next->num ) + return shared_ptr(); + + while( next.get() != nullptr && next->num > block_num ) + next = next->prev.lock(); + return next; +} + +shared_ptr fork_database::fetch_block_on_main_branch_by_number( uint32_t block_num )const +{ + vector blocks = fetch_block_by_number(block_num); + if( blocks.size() == 1 ) + return blocks[0]; + if( blocks.size() == 0 ) + return shared_ptr(); + return walk_main_branch_to_num(block_num); +} + void fork_database::set_head(shared_ptr h) { _head = h; diff --git a/libraries/chain/include/steemit/chain/fork_database.hpp b/libraries/chain/include/steemit/chain/fork_database.hpp index 96683a1d0b..3552b367ed 100644 --- a/libraries/chain/include/steemit/chain/fork_database.hpp +++ b/libraries/chain/include/steemit/chain/fork_database.hpp @@ -75,6 +75,8 @@ namespace steemit { namespace chain { */ pair< branch_type, branch_type > fetch_branch_from(block_id_type first, block_id_type second)const; + shared_ptr walk_main_branch_to_num( uint32_t block_num )const; + shared_ptr fetch_block_on_main_branch_by_number( uint32_t block_num )const; struct block_id; struct block_num; @@ -101,4 +103,5 @@ namespace steemit { namespace chain { fork_multi_index_type _index; shared_ptr _head; }; + } } // steemit::chain From 1f80f5f6516c5229998877040e3a7dadced7c2b9 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Thu, 8 Jun 2017 15:03:56 -0400 Subject: [PATCH 53/59] Re-implement multiple production warning #1175 --- libraries/chain/database.cpp | 18 ++++++++++++++++++ .../chain/include/steemit/chain/database.hpp | 1 + 2 files changed, 19 insertions(+) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 808fd65711..03f5be2227 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -500,6 +500,22 @@ bool database::push_block(const signed_block& new_block, uint32_t skip) return result; } +void database::_maybe_warn_multiple_production( const database& db, uint32_t height )const +{ + auto blocks = _fork_db.fetch_block_by_number( height ); + if( blocks.size() > 1 ) + { + vector< std::pair< account_name_type, fc::time_point_sec > > witness_time_pairs; + for( const auto& b : blocks ) + { + witness_time_pairs.push_back( std::make_pair( b->data.witness, b->data.timestamp ) ); + } + + ilog( "Encountered block num collision at block ${n} due to a fork, witnesses are:", ("n", height)("w", witness_time_pairs) ); + } + return; +} + bool database::_push_block(const signed_block& new_block) { try { uint32_t skip = get_node_properties().skip_flags; @@ -508,6 +524,8 @@ bool database::_push_block(const signed_block& new_block) if( !(skip&skip_fork_db) ) { shared_ptr new_head = _fork_db.push_block(new_block); + _maybe_warn_multiple_production( *this, new_head->num ); + //If the head block from the longest chain does not build off of the current head, we need to switch forks. if( new_head->data.previous != head_block_id() ) { diff --git a/libraries/chain/include/steemit/chain/database.hpp b/libraries/chain/include/steemit/chain/database.hpp index e7f06eb738..0118bf502d 100644 --- a/libraries/chain/include/steemit/chain/database.hpp +++ b/libraries/chain/include/steemit/chain/database.hpp @@ -164,6 +164,7 @@ namespace steemit { namespace chain { bool push_block( const signed_block& b, uint32_t skip = skip_nothing ); void push_transaction( const signed_transaction& trx, uint32_t skip = skip_nothing ); + void _maybe_warn_multiple_production( const database& db, uint32_t height )const; bool _push_block( const signed_block& b ); void _push_transaction( const signed_transaction& trx ); From 33ecf12120ee77540e6338c43b9c3f9006b21b72 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Thu, 8 Jun 2017 18:03:30 -0400 Subject: [PATCH 54/59] Fix _maybe_warn_multiple_production args #1175 --- libraries/chain/database.cpp | 4 ++-- libraries/chain/include/steemit/chain/database.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index 03f5be2227..e2cd42b29e 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -500,7 +500,7 @@ bool database::push_block(const signed_block& new_block, uint32_t skip) return result; } -void database::_maybe_warn_multiple_production( const database& db, uint32_t height )const +void database::_maybe_warn_multiple_production( uint32_t height )const { auto blocks = _fork_db.fetch_block_by_number( height ); if( blocks.size() > 1 ) @@ -524,7 +524,7 @@ bool database::_push_block(const signed_block& new_block) if( !(skip&skip_fork_db) ) { shared_ptr new_head = _fork_db.push_block(new_block); - _maybe_warn_multiple_production( *this, new_head->num ); + _maybe_warn_multiple_production( new_head->num ); //If the head block from the longest chain does not build off of the current head, we need to switch forks. if( new_head->data.previous != head_block_id() ) diff --git a/libraries/chain/include/steemit/chain/database.hpp b/libraries/chain/include/steemit/chain/database.hpp index 0118bf502d..1602c7d474 100644 --- a/libraries/chain/include/steemit/chain/database.hpp +++ b/libraries/chain/include/steemit/chain/database.hpp @@ -164,7 +164,7 @@ namespace steemit { namespace chain { bool push_block( const signed_block& b, uint32_t skip = skip_nothing ); void push_transaction( const signed_transaction& trx, uint32_t skip = skip_nothing ); - void _maybe_warn_multiple_production( const database& db, uint32_t height )const; + void _maybe_warn_multiple_production( uint32_t height )const; bool _push_block( const signed_block& b ); void _push_transaction( const signed_transaction& trx ); From 3ab2b2247f77e67784f106ff10564b2ae41054a2 Mon Sep 17 00:00:00 2001 From: theoreticalbts Date: Thu, 8 Jun 2017 15:04:45 -0400 Subject: [PATCH 55/59] Fix get_blockchain_synopsis() implementation #1174 --- libraries/chain/database.cpp | 32 ++++++++++++++++--- .../chain/include/steemit/chain/database.hpp | 1 + 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index e2cd42b29e..46a2396466 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -246,22 +246,46 @@ bool database::is_known_transaction( const transaction_id_type& id )const return trx_idx.find( id ) != trx_idx.end(); } FC_CAPTURE_AND_RETHROW() } -block_id_type database::get_block_id_for_num( uint32_t block_num )const +block_id_type database::find_block_id_for_num( uint32_t block_num )const { try { + if( block_num == 0 ) + return block_id_type(); + + // Reversible blocks are *usually* in the TAPOS buffer. Since this + // is the fastest check, we do it first. + block_summary_id_type bsid = block_num & 0xFFFF; + const block_summary_object* bs = find< block_summary_object, by_id >( bsid ); + if( bs != nullptr ) + { + if( protocol::block_header::num_from_id(bs->block_id) == block_num ) + return bs->block_id; + } + + // Next we query the block log. Irreversible blocks are here. auto b = _block_log.read_block_by_num( block_num ); if( b.valid() ) return b->id(); - auto results = _fork_db.fetch_block_by_number( block_num ); - FC_ASSERT( results.size() == 1 ); - return results[0]->data.id(); + // Finally we query the fork DB. + shared_ptr< fork_item > fitem = _fork_db.fetch_block_on_main_branch_by_number( block_num ); + if( fitem ) + return fitem->id; + return block_id_type(); } FC_CAPTURE_AND_RETHROW( (block_num) ) } +block_id_type database::get_block_id_for_num( uint32_t block_num )const +{ + block_id_type bid = find_block_id_for_num( block_num ); + FC_ASSERT( bid != block_id_type() ); + return bid; +} + + optional database::fetch_block_by_id( const block_id_type& id )const { try { auto b = _fork_db.fetch_block( id ); diff --git a/libraries/chain/include/steemit/chain/database.hpp b/libraries/chain/include/steemit/chain/database.hpp index 1602c7d474..02240e1f22 100644 --- a/libraries/chain/include/steemit/chain/database.hpp +++ b/libraries/chain/include/steemit/chain/database.hpp @@ -106,6 +106,7 @@ namespace steemit { namespace chain { bool is_known_transaction( const transaction_id_type& id )const; fc::sha256 get_pow_target()const; uint32_t get_pow_summary_target()const; + block_id_type find_block_id_for_num( uint32_t block_num )const; block_id_type get_block_id_for_num( uint32_t block_num )const; optional fetch_block_by_id( const block_id_type& id )const; optional fetch_block_by_number( uint32_t num )const; From 03d4a8f3bc27b418fb02bfee730cd44d235000b7 Mon Sep 17 00:00:00 2001 From: JustinW Date: Thu, 8 Jun 2017 18:50:23 -0400 Subject: [PATCH 56/59] reduce shared memory to 50G for more breathing room --- contrib/fullnode.config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fullnode.config.ini b/contrib/fullnode.config.ini index 1727a0c021..3fd5eff6e6 100644 --- a/contrib/fullnode.config.ini +++ b/contrib/fullnode.config.ini @@ -1,5 +1,5 @@ # Set larger shared-file-size than default -shared-file-size = 52G +shared-file-size = 50G # Set an API to be publicly available, may be specified multiple times public-api = database_api login_api account_by_key_api network_broadcast_api tag_api follow_api market_history_api raw_block_api From 0f45e5c8cdb11c32df21766d69e0c0f9b23cc9fd Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Mon, 12 Jun 2017 14:18:25 -0400 Subject: [PATCH 57/59] Fix timed_lock to use universal time #1186 --- libraries/chainbase/include/chainbase/chainbase.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libraries/chainbase/include/chainbase/chainbase.hpp b/libraries/chainbase/include/chainbase/chainbase.hpp index 5a8cbac408..890a31d3b2 100644 --- a/libraries/chainbase/include/chainbase/chainbase.hpp +++ b/libraries/chainbase/include/chainbase/chainbase.hpp @@ -936,8 +936,7 @@ namespace chainbase { } else { - - if( !lock.timed_lock( boost::posix_time::microsec_clock::local_time() + boost::posix_time::microseconds( wait_micro ) ) ) + if( !lock.timed_lock( boost::posix_time::microsec_clock::universal_time() + boost::posix_time::microseconds( wait_micro ) ) ) BOOST_THROW_EXCEPTION( std::runtime_error( "unable to acquire lock" ) ); } @@ -962,7 +961,7 @@ namespace chainbase { } else { - while( !lock.timed_lock( boost::posix_time::microsec_clock::local_time() + boost::posix_time::microseconds( wait_micro ) ) ) + while( !lock.timed_lock( boost::posix_time::microsec_clock::universal_time() + boost::posix_time::microseconds( wait_micro ) ) ) { _rw_manager->next_lock(); std::cerr << "Lock timeout, moving to lock " << _rw_manager->current_lock_num() << std::endl; From 98891203cb7f358826a3bbf03f5ece9cbb2f9e1c Mon Sep 17 00:00:00 2001 From: JustinW Date: Mon, 12 Jun 2017 15:00:15 -0400 Subject: [PATCH 58/59] add ARGS to readers so shared-file-dir is included for ramdisk deployments --- contrib/startpaassteemd.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/startpaassteemd.sh b/contrib/startpaassteemd.sh index fe6f90d0ae..95ffb075f8 100644 --- a/contrib/startpaassteemd.sh +++ b/contrib/startpaassteemd.sh @@ -115,6 +115,7 @@ if [[ "$USE_MULTICORE_READONLY" ]]; then $STEEMD \ --rpc-endpoint=127.0.0.1:$PORT_NUM \ --data-dir=$HOME \ + $ARGS \ --read-forward-rpc=127.0.0.1:8091 \ --read-only \ 2>&1 & From 24b871648bc513f28a19622cbfdb26a90b99dfe9 Mon Sep 17 00:00:00 2001 From: Michael Vandeberg Date: Tue, 13 Jun 2017 16:46:15 -0400 Subject: [PATCH 59/59] Set hardfork 19 date to 11:00 AM EDT 06/20/17 --- libraries/chain/hardfork.d/0_19.hf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/hardfork.d/0_19.hf b/libraries/chain/hardfork.d/0_19.hf index 74177e1d6a..2b78f1689f 100644 --- a/libraries/chain/hardfork.d/0_19.hf +++ b/libraries/chain/hardfork.d/0_19.hf @@ -1,7 +1,7 @@ #ifndef STEEMIT_HARDFORK_0_19 #define STEEMIT_HARDFORK_0_19 19 -#define STEEMIT_HARDFORK_0_19_TIME 1590886000 // TBD +#define STEEMIT_HARDFORK_0_19_TIME 1497970800 // Tue, 20 Jun 2017 15:00:00 UTC (11:00:00 EDT) #define STEEMIT_HARDFORK_0_19__822 (STEEMIT_HARDFORK_0_19) #define STEEMIT_HARDFORK_0_19__876 (STEEMIT_HARDFORK_0_19)