Merge branch 'updateseeds' into patch-1

hehetown · Jun 7, 2017 · a8d0e5e · a8d0e5e
2 parents 70975df + 96bcbfc
commit a8d0e5e
Show file tree

Hide file tree

Showing 16 changed files with 299 additions and 151 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -84,6 +84,13 @@ if( CLEAR_VOTES )
   SET( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DCLEAR_VOTES" )
 endif()
 
+OPTION( SKIP_BY_TX_ID "Skip ordering operation history by transaction id (ON or OFF)" OFF )
+MESSAGE( STATUS "SKIP_BY_TX_ID: ${SKIP_BY_TX_ID}" )
+if( SKIP_BY_TX_ID )
+   SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSKIP_BY_TX_ID" )
+   SET( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSKIP_BY_TX_ID" )
+endif()
+
 IF( WIN32 )
   SET(BOOST_ROOT $ENV{BOOST_ROOT})
   set(Boost_USE_MULTITHREADED ON)

diff --git a/Dockerfile b/Dockerfile
@@ -49,6 +49,7 @@ RUN \
         -DBUILD_STEEM_TESTNET=ON \
         -DLOW_MEMORY_NODE=OFF \
         -DCLEAR_VOTES=ON \
+        -DSKIP_BY_TX_ID=ON \
         .. && \
     make -j$(nproc) chain_test test_fixed_string && \
     ./tests/chain_test && \
@@ -70,6 +71,7 @@ RUN \
         -DBUILD_STEEM_TESTNET=ON \
         -DLOW_MEMORY_NODE=OFF \
         -DCLEAR_VOTES=ON \
+        -DSKIP_BY_TX_ID=ON \
         -DCHAINBASE_CHECK_LOCKING=OFF \
         .. && \
     make -j$(nproc) chain_test && \
@@ -89,6 +91,7 @@ RUN \
         -DCMAKE_BUILD_TYPE=Release \
         -DLOW_MEMORY_NODE=ON \
         -DCLEAR_VOTES=ON \
+        -DSKIP_BY_TX_ID=ON \
         -DBUILD_STEEM_TESTNET=OFF \
         .. \
     && \
@@ -110,6 +113,7 @@ RUN \
         -DCMAKE_BUILD_TYPE=Release \
         -DLOW_MEMORY_NODE=OFF \
         -DCLEAR_VOTES=OFF \
+        -DSKIP_BY_TX_ID=ON \
         -DBUILD_STEEM_TESTNET=OFF \
         .. \
     && \

diff --git a/contrib/config-for-docker.ini b/contrib/config-for-docker.ini
@@ -32,13 +32,13 @@
 public-api = database_api login_api
 
 # Plugin(s) to enable, may be specified multiple times
-enable-plugin = witness account_history
+enable-plugin = witness
 
 # JSON list of [nblocks,nseconds] pairs, see doc/bcd-trigger.md
 bcd-trigger = [[0,10],[85,300]]
 
 # Defines a range of accounts to track as a json pair ["from","to"] [from,to]
-# track-account-range = 
+# track-account-range =
 
 # Ignore posting operations, only track transfers and account updates
 # filter-posting-ops = 

diff --git a/contrib/fullnode.config.ini b/contrib/fullnode.config.ini
@@ -1,5 +1,5 @@
 # Set larger shared-file-size than default
-shared-file-size = 54G
+shared-file-size = 52G
 
 # Set an API to be publicly available, may be specified multiple times
 public-api = database_api login_api account_by_key_api network_broadcast_api tag_api follow_api market_history_api raw_block_api

diff --git a/contrib/startpaassteemd.sh b/contrib/startpaassteemd.sh
@@ -31,6 +31,12 @@ if [[ ! -z "$STEEMD_SEED_NODES" ]]; then
     done
 fi
 
+NOW=`date +%s`
+STEEMD_FEED_START_TIME=`expr $NOW - 1209600`
+STEEMD_FEED_START_TIMESTAMP=`date --date=@$STEEMD_FEED_START_TIME +%Y-%m-%d:%H:%M:%S`
+
+ARGS+=" --follow-start-feeds=\"$STEEMD_FEED_START_TIMESTAMP\""
+
 # overwrite local config with image one
 cp /etc/steemd/fullnode.config.ini $HOME/config.ini
 
@@ -49,7 +55,7 @@ if [[ "$USE_RAMDISK" ]]; then
   ARGS+=" --shared-file-dir=/mnt/ramdisk/blockchain"
   s3cmd get s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x --wildcards 'blockchain/block*' -C /mnt/ramdisk 'blockchain/shared*'
   chown -R steemd:steemd /mnt/ramdisk/blockchain
-else  
+else
   s3cmd get s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x
 fi
 if [[ $? -ne 0 ]]; then

diff --git a/contrib/steemd.run b/contrib/steemd.run
@@ -26,7 +26,7 @@ fi
 
 # if user did pass in desired seed nodes, use
 # the ones the user specified:
-if [[ ! -z "$STEEMD_SEED_NODES" ]]; then
+if [[ ! -z "$STEEMD_SEED_NODES" ]]; then 
     for NODE in $STEEMD_SEED_NODES ; do
         ARGS+=" --seed-node=$NODE"
     done
@@ -45,6 +45,19 @@ if [[ ! -z "$STEEMD_PRIVATE_KEY" ]]; then
     ARGS+=" --private-key=$STEEMD_PRIVATE_KEY"
 fi
 
+if [[ ! -z "$TRACK_ACCOUNT" ]]; then
+    if [[ ! "$USE_WAY_TOO_MUCH_RAM" ]]; then
+        ARGS+=" --enable-plugin=witness account_history"
+    fi
+    ARGS+="  --track-account-range=[\"$TRACK_ACCOUNT\",\"$TRACK_ACCOUNT\"]"
+fi
+
+NOW=`date +%s`
+STEEMD_FEED_START_TIME=`expr $NOW - 1209600`
+STEEMD_FEED_START_TIMESTAMP=`date --date=@$STEEMD_FEED_START_TIME +%Y-%m-%d:%H:%M:%S`
+
+ARGS+=" --follow-start-feeds=\"$STEEMD_FEED_START_TIMESTAMP\""
+
 # overwrite local config with image one
 if [[ "$USE_FULL_WEB_NODE" ]]; then
   cp /etc/steemd/fullnode.config.ini $HOME/config.ini

diff --git a/doc/building.md b/doc/building.md
@@ -22,6 +22,12 @@ Clears old votes from memory that are no longer required for consensus.
 
 Builds steem for use in a private testnet. Also required for building unit tests.
 
+### SKIP_BY_TX_ID=[OFF/ON]
+
+The account history plugin does not allow querying of operations by transaction id.
+This requires around 65% of CPU time when reindexing so disabling the index is a
+huge gain if you do not need this functionality.
+
 ## Building under Docker
 
 We ship a Dockerfile.  This builds both common node type binaries.

diff --git a/doc/quickstart.md b/doc/quickstart.md
@@ -37,6 +37,12 @@ track-account-range = ["yourexchangeid", "yourexchangeid"]
 ```
 Do not add other APIs or plugins unless you know what you are doing.
 
+This configuration exists in Docker with the following command
+
+```
+docker run -d --env TRACK_ACCOUNT="yourexchangeid" steemit/steem
+```
+
 ### Resources usage
 
 Please make sure that you have enough resources available.

diff --git a/doc/seednodes.txt b/doc/seednodes.txt
@@ -28,3 +28,4 @@ seed.bitcoiner.me:2001         # bitcoiner
 seed.bhuz.info:2001            # bhuz
 78.46.95.157:2001              # pcste
 seed.steemviz.com:2001         # ausbitbank
+45.76.13.167:2001              # chitty 
diff --git a/libraries/app/database_api.cpp b/libraries/app/database_api.cpp
@@ -2348,6 +2348,9 @@ state database_api::get_state( string path )const
 
 annotated_signed_transaction database_api::get_transaction( transaction_id_type id )const
 {
+#ifdef SKIP_BY_TX_ID
+   return annotated_signed_transaction();
+#else
    return my->_db.with_read_lock( [&](){
       const auto& idx = my->_db.get_index<operation_index>().indices().get<by_transaction_id>();
       auto itr = idx.lower_bound( id );
@@ -2362,6 +2365,7 @@ annotated_signed_transaction database_api::get_transaction( transaction_id_type
       }
       FC_ASSERT( false, "Unknown Transaction ${t}", ("t",id));
    });
+#endif
 }
 
 

diff --git a/libraries/chain/include/steemit/chain/history_object.hpp b/libraries/chain/include/steemit/chain/history_object.hpp
@@ -49,13 +49,16 @@ namespace steemit { namespace chain {
                member< operation_object, uint64_t, &operation_object::virtual_op>,
                member< operation_object, operation_id_type, &operation_object::id>
             >
-         >,
+         >
+#ifndef SKIP_BY_TX_ID
+         ,
          ordered_unique< tag< by_transaction_id >,
             composite_key< operation_object,
                member< operation_object, transaction_id_type, &operation_object::trx_id>,
                member< operation_object, operation_id_type, &operation_object::id>
             >
          >
+#endif
       >,
       allocator< operation_object >
    > operation_index;

diff --git a/libraries/plugins/follow/follow_evaluators.cpp b/libraries/plugins/follow/follow_evaluators.cpp
@@ -165,54 +165,57 @@ void reblog_evaluator::do_apply( const reblog_operation& o )
       const auto& idx = db.get_index< follow_index >().indices().get< by_following_follower >();
       auto itr = idx.find( o.account );
 
-      while( itr != idx.end() && itr->following == o.account )
+      if( db.head_block_time() >= _plugin->start_feeds )
       {
-
-         if( itr->what & ( 1 << blog ) )
+         while( itr != idx.end() && itr->following == o.account )
          {
-            uint32_t next_id = 0;
-            auto last_feed = feed_idx.lower_bound( itr->follower );
 
-            if( last_feed != feed_idx.end() && last_feed->account == itr->follower )
+            if( itr->what & ( 1 << blog ) )
             {
-               next_id = last_feed->account_feed_id + 1;
-            }
+               uint32_t next_id = 0;
+               auto last_feed = feed_idx.lower_bound( itr->follower );
 
-            auto feed_itr = comment_idx.find( boost::make_tuple( c.id, itr->follower ) );
+               if( last_feed != feed_idx.end() && last_feed->account == itr->follower )
+               {
+                  next_id = last_feed->account_feed_id + 1;
+               }
 
-            if( feed_itr == comment_idx.end() )
-            {
-               db.create< feed_object >( [&]( feed_object& f )
+               auto feed_itr = comment_idx.find( boost::make_tuple( c.id, itr->follower ) );
+
+               if( feed_itr == comment_idx.end() )
                {
-                  f.account = itr->follower;
-                  f.reblogged_by.push_back( o.account );
-                  f.first_reblogged_by = o.account;
-                  f.first_reblogged_on = db.head_block_time();
-                  f.comment = c.id;
-                  f.reblogs = 1;
-                  f.account_feed_id = next_id;
-               });
-            }
-            else
-            {
-               db.modify( *feed_itr, [&]( feed_object& f )
+                  db.create< feed_object >( [&]( feed_object& f )
+                  {
+                     f.account = itr->follower;
+                     f.reblogged_by.push_back( o.account );
+                     f.first_reblogged_by = o.account;
+                     f.first_reblogged_on = db.head_block_time();
+                     f.comment = c.id;
+                     f.reblogs = 1;
+                     f.account_feed_id = next_id;
+                  });
+               }
+               else
                {
-                  f.reblogged_by.push_back( o.account );
-                  f.reblogs++;
-               });
-            }
+                  db.modify( *feed_itr, [&]( feed_object& f )
+                  {
+                     f.reblogged_by.push_back( o.account );
+                     f.reblogs++;
+                  });
+               }
 
-            const auto& old_feed_idx = db.get_index< feed_index >().indices().get< by_old_feed >();
-            auto old_feed = old_feed_idx.lower_bound( itr->follower );
+               const auto& old_feed_idx = db.get_index< feed_index >().indices().get< by_old_feed >();
+               auto old_feed = old_feed_idx.lower_bound( itr->follower );
 
-            while( old_feed->account == itr->follower && next_id - old_feed->account_feed_id > _plugin->max_feed_size )
-            {
-               db.remove( *old_feed );
-               old_feed = old_feed_idx.lower_bound( itr->follower );
-            };
-         }
+               while( old_feed->account == itr->follower && next_id - old_feed->account_feed_id > _plugin->max_feed_size )
+               {
+                  db.remove( *old_feed );
+                  old_feed = old_feed_idx.lower_bound( itr->follower );
+               };
+            }
 
-         ++itr;
+            ++itr;
+         }
       }
    }
    FC_CAPTURE_AND_RETHROW( (o) )

diff --git a/libraries/plugins/follow/follow_plugin.cpp b/libraries/plugins/follow/follow_plugin.cpp
@@ -194,39 +194,42 @@ struct post_operation_visitor
 
          const auto& feed_idx = db.get_index< feed_index >().indices().get< by_feed >();
 
-         while( itr != idx.end() && itr->following == op.author )
+         if( db.head_block_time() >= _plugin.start_feeds )
          {
-            if( itr->what & ( 1 << blog ) )
+            while( itr != idx.end() && itr->following == op.author )
             {
-               uint32_t next_id = 0;
-               auto last_feed = feed_idx.lower_bound( itr->follower );
-
-               if( last_feed != feed_idx.end() && last_feed->account == itr->follower )
+               if( itr->what & ( 1 << blog ) )
                {
-                  next_id = last_feed->account_feed_id + 1;
-               }
+                  uint32_t next_id = 0;
+                  auto last_feed = feed_idx.lower_bound( itr->follower );
 
-               if( comment_idx.find( boost::make_tuple( c.id, itr->follower ) ) == comment_idx.end() )
-               {
-                  db.create< feed_object >( [&]( feed_object& f )
+                  if( last_feed != feed_idx.end() && last_feed->account == itr->follower )
                   {
-                     f.account = itr->follower;
-                     f.comment = c.id;
-                     f.account_feed_id = next_id;
-                  });
-
-                  const auto& old_feed_idx = db.get_index< feed_index >().indices().get< by_old_feed >();
-                  auto old_feed = old_feed_idx.lower_bound( itr->follower );
+                     next_id = last_feed->account_feed_id + 1;
+                  }
 
-                  while( old_feed->account == itr->follower && next_id - old_feed->account_feed_id > _plugin.max_feed_size )
+                  if( comment_idx.find( boost::make_tuple( c.id, itr->follower ) ) == comment_idx.end() )
                   {
-                     db.remove( *old_feed );
-                     old_feed = old_feed_idx.lower_bound( itr->follower );
+                     db.create< feed_object >( [&]( feed_object& f )
+                     {
+                        f.account = itr->follower;
+                        f.comment = c.id;
+                        f.account_feed_id = next_id;
+                     });
+
+                     const auto& old_feed_idx = db.get_index< feed_index >().indices().get< by_old_feed >();
+                     auto old_feed = old_feed_idx.lower_bound( itr->follower );
+
+                     while( old_feed->account == itr->follower && next_id - old_feed->account_feed_id > _plugin.max_feed_size )
+                     {
+                        db.remove( *old_feed );
+                        old_feed = old_feed_idx.lower_bound( itr->follower );
+                     }
                   }
                }
-            }
 
-            ++itr;
+               ++itr;
+            }
          }
 
          const auto& blog_idx = db.get_index< blog_index >().indices().get< by_blog >();
@@ -345,6 +348,7 @@ void follow_plugin::plugin_set_program_options(
 {
    cli.add_options()
       ("follow-max-feed-size", boost::program_options::value< uint32_t >()->default_value( 500 ), "Set the maximum size of cached feed for an account" )
+      ("follow-start-feeds", boost::program_options::value< uint32_t >()->default_value( 0 ), "Block time (in epoch seconds) when to start calculating feeds" )
       ;
    cfg.add( cli );
 }
@@ -371,6 +375,11 @@ void follow_plugin::plugin_initialize( const boost::program_options::variables_m
          uint32_t feed_size = options[ "follow-max-feed-size" ].as< uint32_t >();
          max_feed_size = feed_size;
       }
+
+      if( options.count( "follow-start-feeds" ) )
+      {
+         start_feeds = fc::time_point_sec( options[ "follow-start-feeds" ].as< uint32_t >() );
+      }
    }
    FC_CAPTURE_AND_RETHROW()
 }

diff --git a/libraries/plugins/follow/include/steemit/follow/follow_plugin.hpp b/libraries/plugins/follow/include/steemit/follow/follow_plugin.hpp
@@ -28,6 +28,7 @@ class follow_plugin : public steemit::app::plugin
       friend class detail::follow_plugin_impl;
       std::unique_ptr<detail::follow_plugin_impl> my;
       uint32_t max_feed_size = 500;
+      fc::time_point_sec start_feeds;
 };
 
 } } //steemit::follow
diff --git a/libraries/plugins/tags/include/steemit/tags/tags_plugin.hpp b/libraries/plugins/tags/include/steemit/tags/tags_plugin.hpp
@@ -118,7 +118,13 @@ typedef multi_index_container<
    tag_object,
    indexed_by<
       ordered_unique< tag< by_id >, member< tag_object, tag_id_type, &tag_object::id > >,
-      ordered_non_unique< tag< by_comment >, member< tag_object, comment_id_type, &tag_object::comment > >,
+      ordered_unique< tag< by_comment >,
+         composite_key< tag_object,
+            member< tag_object, comment_id_type, &tag_object::comment >,
+            member< tag_object, tag_id_type, &tag_object::id >
+         >,
+         composite_key_compare< std::less< comment_id_type >, std::less< tag_id_type > >
+      >,
       ordered_unique< tag< by_author_comment >,
             composite_key< tag_object,
                member< tag_object, account_id_type, &tag_object::author >,