Merge pull request steemit#1192 from steemit/steem-prerelease-v0.19.0

Steem v0.19.0

.gitignore

@@ -37,6 +37,8 @@ tests/performance_test
 
 doxygen
 
+build
+
 wallet.json
 witness_node_data_dir
 

CMakeLists.txt

@@ -84,6 +84,13 @@ if( CLEAR_VOTES )
   SET( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DCLEAR_VOTES" )
 endif()
 
+OPTION( SKIP_BY_TX_ID "Skip ordering operation history by transaction id (ON or OFF)" OFF )
+MESSAGE( STATUS "SKIP_BY_TX_ID: ${SKIP_BY_TX_ID}" )
+if( SKIP_BY_TX_ID )
+   SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSKIP_BY_TX_ID" )
+   SET( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DSKIP_BY_TX_ID" )
+endif()
+
 IF( WIN32 )
   SET(BOOST_ROOT $ENV{BOOST_ROOT})
   set(Boost_USE_MULTITHREADED ON)

Dockerfile

@@ -49,6 +49,7 @@ RUN \
         -DBUILD_STEEM_TESTNET=ON \
         -DLOW_MEMORY_NODE=OFF \
         -DCLEAR_VOTES=ON \
+        -DSKIP_BY_TX_ID=ON \
         .. && \
     make -j$(nproc) chain_test test_fixed_string && \
     ./tests/chain_test && \
@@ -70,6 +71,7 @@ RUN \
         -DBUILD_STEEM_TESTNET=ON \
         -DLOW_MEMORY_NODE=OFF \
         -DCLEAR_VOTES=ON \
+        -DSKIP_BY_TX_ID=ON \
         -DCHAINBASE_CHECK_LOCKING=OFF \
         .. && \
     make -j$(nproc) chain_test && \
@@ -89,6 +91,7 @@ RUN \
         -DCMAKE_BUILD_TYPE=Release \
         -DLOW_MEMORY_NODE=ON \
         -DCLEAR_VOTES=ON \
+        -DSKIP_BY_TX_ID=ON \
         -DBUILD_STEEM_TESTNET=OFF \
         .. \
     && \
@@ -110,6 +113,7 @@ RUN \
         -DCMAKE_BUILD_TYPE=Release \
         -DLOW_MEMORY_NODE=OFF \
         -DCLEAR_VOTES=OFF \
+        -DSKIP_BY_TX_ID=ON \
         -DBUILD_STEEM_TESTNET=OFF \
         .. \
     && \

contrib/config-for-docker.ini

@@ -32,13 +32,13 @@
 public-api = database_api login_api
 
 # Plugin(s) to enable, may be specified multiple times
-enable-plugin = witness account_history
+enable-plugin = witness
 
 # JSON list of [nblocks,nseconds] pairs, see doc/bcd-trigger.md
 bcd-trigger = [[0,10],[85,300]]
 
 # Defines a range of accounts to track as a json pair ["from","to"] [from,to]
-# track-account-range = 
+# track-account-range =
 
 # Ignore posting operations, only track transfers and account updates
 # filter-posting-ops = 

contrib/fullnode.config.ini

@@ -1,5 +1,5 @@
 # Set larger shared-file-size than default
-shared-file-size = 48G
+shared-file-size = 50G
 
 # Set an API to be publicly available, may be specified multiple times
 public-api = database_api login_api account_by_key_api network_broadcast_api tag_api follow_api market_history_api raw_block_api

contrib/healthcheck.conf.template

@@ -13,6 +13,7 @@
       proxy_set_header Host $host;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_http_version 1.1;
+      proxy_buffering off;
       proxy_set_header Upgrade $http_upgrade;
       proxy_set_header Connection "upgrade";
       add_header 'Access-Control-Allow-Origin' '*';

contrib/startpaassteemd.sh

@@ -31,6 +31,13 @@ if [[ ! -z "$STEEMD_SEED_NODES" ]]; then
     done
 fi
 
+NOW=`date +%s`
+STEEMD_FEED_START_TIME=`expr $NOW - 1209600`
+
+ARGS+=" --follow-start-feeds=$STEEMD_FEED_START_TIME"
+
+ARGS+=" --disable-get-block"
+
 # overwrite local config with image one
 cp /etc/steemd/fullnode.config.ini $HOME/config.ini
 
@@ -49,7 +56,7 @@ if [[ "$USE_RAMDISK" ]]; then
   ARGS+=" --shared-file-dir=/mnt/ramdisk/blockchain"
   s3cmd get s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x --wildcards 'blockchain/block*' -C /mnt/ramdisk 'blockchain/shared*'
   chown -R steemd:steemd /mnt/ramdisk/blockchain
-else  
+else
   s3cmd get s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x
 fi
 if [[ $? -ne 0 ]]; then
@@ -108,6 +115,7 @@ if [[ "$USE_MULTICORE_READONLY" ]]; then
         $STEEMD \
           --rpc-endpoint=127.0.0.1:$PORT_NUM \
           --data-dir=$HOME \
+          $ARGS \
           --read-forward-rpc=127.0.0.1:8091 \
           --read-only \
           2>&1 &

contrib/steemd.run

@@ -26,7 +26,7 @@ fi
 
 # if user did pass in desired seed nodes, use
 # the ones the user specified:
-if [[ ! -z "$STEEMD_SEED_NODES" ]]; then
+if [[ ! -z "$STEEMD_SEED_NODES" ]]; then 
     for NODE in $STEEMD_SEED_NODES ; do
         ARGS+=" --seed-node=$NODE"
     done
@@ -45,6 +45,18 @@ if [[ ! -z "$STEEMD_PRIVATE_KEY" ]]; then
     ARGS+=" --private-key=$STEEMD_PRIVATE_KEY"
 fi
 
+if [[ ! -z "$TRACK_ACCOUNT" ]]; then
+    if [[ ! "$USE_WAY_TOO_MUCH_RAM" ]]; then
+        ARGS+=" --enable-plugin=witness account_history"
+    fi
+    ARGS+="  --track-account-range=[\"$TRACK_ACCOUNT\",\"$TRACK_ACCOUNT\"]"
+fi
+
+NOW=`date +%s`
+STEEMD_FEED_START_TIME=`expr $NOW - 1209600`
+
+ARGS+=" --follow-start-feeds=$STEEMD_FEED_START_TIME"
+
 # overwrite local config with image one
 if [[ "$USE_FULL_WEB_NODE" ]]; then
   cp /etc/steemd/fullnode.config.ini $HOME/config.ini

contrib/sync-sv-run.sh

@@ -38,9 +38,13 @@ if [[ ! -z "$BLOCKCHAIN_TIME" ]]; then
     echo steemdsync: waiting for steemd to exit cleanly
     while [ -e /proc/$STEEMD_PID ]; do sleep 0.1; done
     echo steemdsync: starting a new blockchainstate upload operation
-    cd $HOME
+    cd ${COMPRESSPATH:-$HOME}
     echo steemdsync: compressing blockchainstate...
-    tar cf blockchain.tar.bz2 --use-compress-prog=pbzip2 blockchain
+    tar cf blockchain.tar.bz2 --use-compress-prog=pbzip2 -C $HOME blockchain
+    if [[ ! $? -eq 0 ]]; then
+      echo NOTIFYALERT! steemdsync was unable to compress shared memory file, check the logs.
+      exit 1
+    fi
     FILE_NAME=blockchain-$VERSION-`date '+%Y%m%d-%H%M%S'`.tar.bz2
     echo steemdsync: uploading $FILE_NAME to $S3_BUCKET
     aws s3 cp blockchain.tar.bz2 s3://$S3_BUCKET/$FILE_NAME
@@ -56,6 +60,7 @@ if [[ ! -z "$BLOCKCHAIN_TIME" ]]; then
     	exit 1
     fi
     # upload a current block_log
+    cd $HOME
     aws s3 cp blockchain/block_log s3://$S3_BUCKET/block_log-intransit
     aws s3 cp s3://$S3_BUCKET/block_log-intransit s3://$S3_BUCKET/block_log-latest
     # kill the container starting the process over again

doc/building.md

@@ -22,6 +22,12 @@ Clears old votes from memory that are no longer required for consensus.
 
 Builds steem for use in a private testnet. Also required for building unit tests.
 
+### SKIP_BY_TX_ID=[OFF/ON]
+
+The account history plugin does not allow querying of operations by transaction id.
+This requires around 65% of CPU time when reindexing so disabling the index is a
+huge gain if you do not need this functionality.
+
 ## Building under Docker
 
 We ship a Dockerfile.  This builds both common node type binaries.

doc/quickstart.md

@@ -0,0 +1,63 @@
+Quickstart
+----------
+
+### Get current steemd
+Use docker:
+```
+docker run \
+    -d -p 2001:2001 -p 8090:8090 --name steemd-default \
+    steemit/steem
+```
+#### Low memory node?
+Above runs low memory node, which is suitable for:
+- seed nodes
+- witness nodes
+- exchanges, etc.
+For full api node use:
+
+```
+docker run \
+    --env USE_WAY_TOO_MUCH_RAM=1 --env USE_FULL_WEB_NODE=1 \
+    -d -p 2001:2001 -p 8090:8090 --name steemd-full \
+    steemit/steem
+```
+### Configure for your use case
+#### Full API node
+You need to use `USE_WAY_TOO_MUCH_RAM=1` and `USE_FULL_WEB_NODE=1` as stated above.
+You can Use `contrib/fullnode.config.ini` as a base for your `config.ini` file.
+
+#### Exchanges
+Use low memory node.
+
+Also make sure that your `config.ini` contains:
+```
+enable-plugin = account_history
+public-api = database_api login_api
+track-account-range = ["yourexchangeid", "yourexchangeid"]
+```
+Do not add other APIs or plugins unless you know what you are doing.
+
+This configuration exists in Docker with the following command
+
+```
+docker run -d --env TRACK_ACCOUNT="yourexchangeid" steemit/steem
+```
+
+### Resources usage
+
+Please make sure that you have enough resources available.
+Check `shared-file-size =` in your `config.ini` to reflect your needs.
+Set it to at least 20% more than current size.
+
+Provided values are expected to grow significantly over time.
+
+Blockchain data takes over **12GB** of storage space.
+
+#### Full node
+Shared memory file for full node uses over **45GB**
+
+#### Seed node
+Shared memory file for seed node uses over **3GB**
+
+#### Other use cases
+Shared memory file size varies, depends on your specific configuration but it is expected to be somewhere between "seed node" and "full node" usage.

doc/seednodes.txt

@@ -1,36 +1,31 @@
-seed.riversteem.com:2001       # riverhead
 steem-seed1.abit-more.com:2001 # abit 
 52.74.152.79:2001              # smooth.witness
 seed.steemd.com:34191          # roadscape
 steemwitness.matthewniemerg.com:2001 # complexring
-steemd.pharesim.me:2001        # pharesim
 seed.jesta.us:2001             # jesta
 seed.liondani.com:2016         # liondani
 anyx.co:2001                   # anyx
 seed.xeldal.com:12150          # xeldal
 seed.steemnodes.com:2001       # wackou
-steem.clawmap.com:2001         # steempty
 gtg.steem.house:2001           # gtg
 192.99.3.29:2001               # joseph
 5.9.18.213:2001                # pfunk
 lafonasteem.com:2001           # lafona
 seed.rossco99.com:2001         # rossco99
-212.47.249.84:40696            # ihashfury
+steem-seed.altcap.io:40696     # ihashfury
 seed.steemfeeder.com:2001      # au1nethyb1
-52.175.211.168:2001            # aizensou
 seed.roelandp.nl:2001          # roelandp
 81.89.101.133:2001             # cyrano.witness
 steem.global:2001              # klye
 seed.esteem.ws:2001            # good-karma
 176.31.126.187:2001            # timcliff
-seed.royaltiffany.me:2001      # royaltiffany
 seed.thecryptodrive.com:2001   # thecryptodrive
 steem-id.altexplorer.xyz:2001  # steem-id
 seed.bitcoiner.me:2001         # bitcoiner
 104.199.118.92:2001            # clayop
 192.99.4.226:2001              # dele-puppy
-89.36.26.82:2001               # dragosroua
-104.198.222.18:2001            # bitcoinparadise
-138.197.17.188:2001            # chitty
 88.99.33.113:2001              # cervantes
 seed.bhuz.info:2001            # bhuz
+78.46.95.157:2001              # pcste
+seed.steemviz.com:2001         # ausbitbank
+45.76.13.167:2001              # chitty