Obligatory incremental commit

libraries/chain/database.cpp

@@ -1314,6 +1314,8 @@ void database::adjust_total_payout( const comment_object& cur, const asset& sbd_
  *  @recursively pays out parent posts
  */
 void database::cashout_comment_helper( const comment_object& cur, const comment_object& origin, asset vesting_steem_reward, asset sbd_reward ) {
+   try
+   {
    const auto& author = get_account( cur.author );
    if( cur.parent_author.size() ) {
       auto parent_vesting_steem_reward = vesting_steem_reward;
@@ -1358,6 +1360,7 @@ void database::cashout_comment_helper( const comment_object& cur, const comment_
       push_applied_operation( comment_reward_operation( cur.author, cur.permlink, origin.author, origin.permlink, sbd_created, vest_created ) );
       adjust_total_payout( cur, sbd_created + to_sbd( vesting_steem_reward ) );
    }
+   } FC_CAPTURE_LOG_AND_RETHROW( (cur) );
 }
 
 /**
@@ -1376,6 +1379,8 @@ share_type database::pay_curators( const comment_object& c, share_type max_rewar
    auto end = cvidx.lower_bound( boost::make_tuple( c.id, uint64_t(0), account_id_type() ) );
    while( itr != end ) {
       // TODO: Add minimum curation pay limit
+      try
+      {
       u256 weight( itr->weight );
       auto claim = static_cast<uint64_t>((max_rewards.value * weight) / total_weight);
       if( claim > 1 ) // min_amt is non-zero satoshis
@@ -1384,6 +1389,7 @@ share_type database::pay_curators( const comment_object& c, share_type max_rewar
          auto reward = create_vesting( itr->voter(*this), asset( claim, STEEM_SYMBOL ) );
          push_applied_operation( curate_reward_operation( itr->voter(*this).name, reward, c.author, c.permlink ) );
       }
+      } FC_CAPTURE_LOG_AND_RETHROW( (*itr) )
       ++itr;
    }
    if( max_rewards.value - unclaimed_rewards.value )
@@ -1408,6 +1414,8 @@ void database::process_comment_cashout() {
    auto current = cidx.begin();
    //auto end = cidx.lower_bound( head_block_time() );
    while( current != cidx.end() && current->cashout_time <= head_block_time() ) {
+      try
+      {
       const auto& cur = *current; ++current;
       asset sbd_created(0,SBD_SYMBOL);
       asset vest_created(0,VESTS_SYMBOL);
@@ -1461,6 +1469,7 @@ void database::process_comment_cashout() {
          ++vote_itr;
          remove(cur_vote);
       }
+   } FC_CAPTURE_LOG_AND_RETHROW( (*current) );
    }
 }
 
@@ -1633,6 +1642,8 @@ asset database::to_steem( const asset& sbd )const {
  *  redeemed.
  */
 share_type database::claim_rshare_reward( share_type rshares ) {
+   try
+   {
    FC_ASSERT( rshares > 0 );
 
    const auto& props = get_dynamic_global_properties();
@@ -1654,7 +1665,7 @@ share_type database::claim_rshare_reward( share_type rshares ) {
    modify( props, [&]( dynamic_global_property_object& p ){
      p.total_reward_fund_steem.amount -= payout;
    });
-
+   } FC_CAPTURE_LOG_AND_RETHROW( (rshares) )
    return payout;
 }
 
@@ -1872,7 +1883,7 @@ void database::apply_block( const signed_block& next_block, uint32_t skip )
    } );
 
    /// check invariants
-   //if( !( skip & skip_validate_invariants ) )
+   if( !( skip & skip_validate_invariants ) )
       validate_invariants();
 }
 

libraries/chain/steem_evaluator.cpp

@@ -9,6 +9,8 @@
 #include <fc/uint128.hpp>
 #include <fc/utf8.hpp>
 
+#include <cmath>
+
 namespace steemit { namespace chain {
    using fc::uint128_t;
 
@@ -196,7 +198,7 @@ void comment_evaluator::do_apply( const comment_operation& o )
 
       const auto& new_comment_stats = db().create<comment_stats_object>( [&]( comment_stats_object& cso ){
           cso.comment_id  = new_comment.id;
-          if( parent ) 
+          if( parent )
             cso.parent_comment_id = parent->id;
           cso.category_id = cat->id;
           cso.author_id   = auth.get_id();
@@ -537,7 +539,6 @@ void vote_evaluator::do_apply( const vote_operation& o )
 
    FC_ASSERT( abs_rshares > 0 );
 
-
    db().modify( comment, [&]( comment_object& c ){
       c.net_rshares += rshares;
       c.abs_rshares += abs_rshares;

libraries/plugins/debug_node/debug_node_api.cpp

@@ -23,7 +23,7 @@ class debug_node_api_impl
    public:
       debug_node_api_impl( steemit::app::application& _app );
 
-      uint32_t debug_push_blocks( const std::string& src_filename, uint32_t count );
+      uint32_t debug_push_blocks( const std::string& src_filename, uint32_t count, bool skip_validate_invariants );
       uint32_t debug_generate_blocks( const std::string& debug_key, uint32_t count );
       uint32_t debug_generate_blocks_until( const std::string& debug_key, const fc::time_point_sec& head_block_time, bool generate_sparsely );
       void debug_update_object( const fc::variant_object& update );
@@ -40,7 +40,7 @@ debug_node_api_impl::debug_node_api_impl( steemit::app::application& _app ) : ap
 {}
 
 
-uint32_t debug_node_api_impl::debug_push_blocks( const std::string& src_filename, uint32_t count )
+uint32_t debug_node_api_impl::debug_push_blocks( const std::string& src_filename, uint32_t count, bool skip_validate_invariants )
 {
    if( count == 0 )
       return 0;
@@ -50,9 +50,16 @@ uint32_t debug_node_api_impl::debug_push_blocks( const std::string& src_filename
    if( fc::is_directory( src_path ) )
    {
       ilog( "Loading ${n} from block_database ${fn}", ("n", count)("fn", src_filename) );
+      idump( (src_filename)(count)(skip_validate_invariants) );
       steemit::chain::block_database bdb;
       bdb.open( src_path );
       uint32_t first_block = db->head_block_num()+1;
+      uint32_t skip_flags = steemit::chain::database::skip_nothing;
+      if( skip_validate_invariants )
+      {
+         skip_flags = skip_flags | steemit::chain::database::skip_validate_invariants;
+         ilog( "Skipping invariante check" );
+      }
       for( uint32_t i=0; i<count; i++ )
       {
          fc::optional< steemit::chain::signed_block > block = bdb.fetch_by_number( first_block+i );
@@ -63,7 +70,7 @@ uint32_t debug_node_api_impl::debug_push_blocks( const std::string& src_filename
          }
          try
          {
-            db->push_block( *block );
+            db->push_block( *block, skip_flags );
          }
          catch( const fc::exception& e )
          {
@@ -146,6 +153,8 @@ uint32_t debug_node_api_impl::debug_generate_blocks_until( const std::string& de
 
          db->generate_block( scheduled_time, scheduled_witness_name, *debug_private_key, steemit::chain::database::skip_nothing );
          new_blocks++;
+
+         FC_ASSERT( head_block_time.sec_since_epoch() - db->head_block_time().sec_since_epoch() < STEEMIT_BLOCK_INTERVAL, "", ("desired_time", head_block_time)("db->head_block_time()",db->head_block_time()) );
       }
    }
    else
@@ -199,9 +208,9 @@ debug_node_api::debug_node_api( steemit::app::application& app )
 
 void debug_node_api::on_api_startup() {}
 
-uint32_t debug_node_api::debug_push_blocks( std::string source_filename, uint32_t count )
+uint32_t debug_node_api::debug_push_blocks( std::string source_filename, uint32_t count, bool skip_validate_invariants )
 {
-   return my->debug_push_blocks( source_filename, count );
+   return my->debug_push_blocks( source_filename, count, skip_validate_invariants );
 }
 
 uint32_t debug_node_api::debug_generate_blocks( std::string debug_key, uint32_t count )

libraries/plugins/debug_node/include/steemit/plugins/debug_node/debug_node_api.hpp

@@ -27,7 +27,7 @@ class debug_node_api
       /**
        * Push blocks from existing database.
        */
-      uint32_t debug_push_blocks( std::string src_filename, uint32_t count );
+      uint32_t debug_push_blocks( std::string src_filename, uint32_t count, bool skip_validate_invariants = false );
 
       /**
        * Generate blocks locally.

python_scripts/steemdebugnode/debugnode.py

@@ -1,5 +1,7 @@
-import datetime, json, logging
+import json, logging
 
+from datetime import datetime
+from datetime import timezone
 from os import devnull
 from pathlib import Path
 from signal import SIGINT, SIGTERM
@@ -13,7 +15,7 @@
 class DebugNode( object ):
    """ Wraps the steemd debug node plugin for easier automated testing of the Steem Network"""
 
-   def __init__( self, steemd, data_dir, plugins=[], apis=[], steemd_stdout=None, steemd_stderr=None ):
+   def __init__( self, steemd, data_dir, plugins=[], apis=[], steemd_out=None, steemd_err=None ):
       """ Creates a steemd debug node.
 
       It can be ran by using 'with debug_node:'
@@ -56,15 +58,15 @@ def __init__( self, steemd, data_dir, plugins=[], apis=[], steemd_stdout=None, s
       self.apis = apis
 
       self._FNULL = open( devnull, 'w' )
-      if( steemd_stdout != None ):
-         self.steemd_stdout = steemd_stdout
+      if( steemd_out != None ):
+         self.steemd_out = steemd_out
       else:
-         self.steemd_stdout = self._FNULL
+         self.steemd_out = self._FNULL
 
-      if( steemd_stderr != None ):
-         self.steemd_stderr = steemd_stderr
+      if( steemd_err != None ):
+         self.steemd_err = steemd_err
       else:
-         self.steemd_stderr = self._FNULL
+         self.steemd_err = self._FNULL
 
       self._debug_key = '5JHNbFNDg834SFj8CMArV6YW7td4zrPzXveqTfaShmYVuYNeK69'
       self._steemd_lock = Lock()
@@ -79,7 +81,7 @@ def __enter__( self ):
       config.touch()
       config.write_text( self._get_config() )
 
-      self._steemd_process = Popen( [ str( self._steemd_bin ), '--data-dir="' + str( self._temp_data_dir.name ) + '"' ], stdout=self._FNULL, stderr=self._FNULL )
+      self._steemd_process = Popen( [ str( self._steemd_bin ), '--data-dir="' + str( self._temp_data_dir.name ) + '"' ], stdout=self.steemd_out, stderr=self.steemd_err )
       self._steemd_process.poll()
       sleep( 5 )
       if( not self._steemd_process.returncode ):
@@ -122,7 +124,7 @@ def _get_config( self ):
           + "public-api = database_api login_api debug_node_api " + " ".join( self.apis ) + "\n"
 
 
-   def debug_push_blocks( self, count=0 ):
+   def debug_push_blocks( self, count=0, skip_validate_invariants=False ):
       """
       Push count blocks from an existing chain.
       There is no guarantee pushing blocks will work depending on set hardforks, or generated blocks
@@ -136,13 +138,17 @@ def debug_push_blocks( self, count=0 ):
          int: The number of blocks actually pushed.
       """
       num_blocks = 0
+      skip_validate_invariants_str = "false"
+      if( skip_validate_invariants ):
+         skip_validate_invariants_str = "true"
+
       if( count == 0 ):
          ret = 10000
          while( ret == 10000 ):
-            ret = self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_push_blocks",["' + str( self._block_dir ) + '", 10000]], "id": 1}' ) )
+            ret = self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_push_blocks",["' + str( self._block_dir ) + '", 10000,"' + skip_validate_invariants_str + '"]], "id": 1}' ) )
             num_blocks += ret
       else:
-         ret = self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_push_blocks",["' + str( self._block_dir ) + '",' + str( count ) + ']], "id": 1}' ) )
+         ret = self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_push_blocks",["' + str( self._block_dir ) + '",' + str( count ) + ',"' + skip_validate_invariants_str + '"]], "id": 1}' ) )
          num_blocks += ret
 
       return num_blocks
@@ -167,7 +173,7 @@ def debug_generate_blocks( self, count ):
       return self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_generate_blocks",["' + self._debug_key + '",' + str( count ) + ']], "id": 1}' ) )
 
 
-   def debug_generate_blocks_until( self, time, generate_sparsely=True ):
+   def debug_generate_blocks_until( self, timestamp, generate_sparsely=True ):
       """
       Generate block up until a head block time rather than a specific number of blocks. As with
       `debug_generate_blocks` all blocks will be empty unless there were pending transactions.
@@ -177,7 +183,7 @@ def debug_generate_blocks_until( self, time, generate_sparsely=True ):
       `get_dev_key steem debug`. Do not use this key on the live chain for any reason.
 
       args:
-         time -- The desired new head block time. This is a UTC Timestmap.
+         time -- The desired new head block time. This is a POSIX Timestmap.
          generate_sparsely -- True if you wish to skip all intermediate blocks between the current
             head block time and the desired head block time. This is useful to trigger events, such
             as payouts and bandwidth updates, without generating blocks. However, many automatic chain
@@ -188,9 +194,19 @@ def debug_generate_blocks_until( self, time, generate_sparsely=True ):
          (time, int): A tuple including the new head block time and the number of blocks that were
             generated.
       """
-      if( not instance( time, datetime ) ):
-         raise ValueError( "Time must be a datetime object" )
-      self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_generate_blocks_until",["' + self._debug_key + '",' + str( time ) + ']], "id": 1}' ) )
+      if( not isinstance( timestamp, int ) ):
+         raise ValueError( "Time must be a int" )
+      generate_sparsely_str = "true"
+      if( not generate_sparsely ):
+         generate_sparsely_str = "false"
+
+      iso_string = datetime.fromtimestamp( timestamp, timezone.utc ).isoformat().split( '+' )[0].split( '-' )
+      if( len( iso_string ) == 4 ):
+         iso_string = iso_string[:-1]
+      iso_string = '-'.join( iso_string )
+
+      print( iso_string )
+      return self._rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [2,"debug_generate_blocks_until",["' + self._debug_key + '","' + iso_string + '","' + generate_sparsely_str + '"]], "id": 1}' ) )
 
 
    def debug_set_hardfork( self, hardfork_id ):

python_scripts/tests/debug_hardforks.py

@@ -47,7 +47,7 @@ def main( ):
    if( not data_dir.is_dir() ):
       print( 'Error: data_dir is not a directory' )
 
-   debug_node = DebugNode( str( steemd ), str( data_dir ) )
+   debug_node = DebugNode( str( steemd ), str( data_dir ), steemd_err=sys.stderr )
 
    with debug_node :
 
@@ -62,8 +62,13 @@ def run_steemd_tests( debug_node ):
    from steemapi.steemnoderpc import SteemNodeRPC
 
    try:
-      print( "Playing blockchain..." )
-      blocks = debug_node.debug_push_blocks( 0 )
+      print( 'Replaying blocks...', )
+      sys.stdout.flush()
+      total_blocks = 0
+      while( total_blocks % 100000 == 0 ):
+         total_blocks += debug_node.debug_push_blocks( 100000, skip_validate_invariants=True )
+         print( 'Blocks Replayed: ' + str( total_blocks ) )
+         sys.stdout.flush()
 
       print( "Setting the hardfork now" ) # TODO: Grab most recent hardfork num from build directory
       sys.stdout.flush()
@@ -77,7 +82,7 @@ def run_steemd_tests( debug_node ):
       sys.stdout.flush()
       rpc = SteemNodeRPC( 'ws://127.0.0.1:8090', '', '' )
       block_producers = {}
-      for i in range( blocks + 1 , blocks + 5001 ):
+      for i in range( total_blocks + 1 , total_blocks + 5001 ):
          ret = rpc.rpcexec( json.loads( '{"jsonrpc": "2.0", "method": "call", "params": [0,"get_block",[' + str( i ) + ']], "id":4}' ) )
          if( ret[ "witness" ] in block_producers ):
             block_producers[ ret[ "witness" ] ] += 1