This completes steemit#1049 and steemit#1050

DINKIN · Apr 20, 2017 · 0705d7e · 0705d7e
1 parent 54eacc5
commit 0705d7e
Show file tree

Hide file tree

Showing 6 changed files with 123 additions and 59 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -3,7 +3,6 @@ FROM phusion/baseimage:0.9.19
 #ARG STEEMD_BLOCKCHAIN=https://example.com/steemd-blockchain.tbz2
 
 ENV LANG=en_US.UTF-8
-ENV VERSION=0.18.1
 
 RUN \
     apt-get update && \
@@ -31,6 +30,7 @@ RUN \
         s3cmd \
         awscli \
         jq \
+        wget \
     && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
@@ -93,6 +93,13 @@ RUN \
     make -j$(nproc) && \
     make install && \
     cd .. && \
+    ( /usr/local/steemd-default/bin/steemd --version \
+      | grep -o '[0-9]*\.[0-9]*\.[0-9]*' \
+      && echo '_' \
+      && git rev-parse --short HEAD ) \
+      | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n//g' \
+      > /etc/steemdversion && \
+    cat /etc/steemdversion && \
     rm -rfv build && \
     mkdir build && \
     cd build && \
@@ -205,4 +212,4 @@ RUN chmod +x /usr/local/bin/healthcheck.sh
 # AWS EB Docker requires a non-daemonized entrypoint
 ADD contrib/steemdentrypoint.sh /usr/local/bin/steemdentrypoint.sh
 RUN chmod +x /usr/local/bin/steemdentrypoint.sh
-CMD /usr/local/bin/steemdentrypoint.sh
+CMD /usr/local/bin/steemdentrypoint.sh
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Introducing Steem (beta)
 
-Steem is an experimental Proof of Work blockchain with an unproven consensus
+Steem is an experimental Delegated Proof of Stake blockchain with an unproven consensus
 algorithm.
 
   - Currency symbol STEEM
@@ -23,7 +23,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 IN THE SOFTWARE.
 
-# Code is Documentation
+# Blockchain consensus rules
 
 Rather than attempt to describe the rules of the blockchain, it is up to
 each individual to inspect the code to understand the consensus rules.
@@ -49,12 +49,30 @@ To run a node with *all* the data (e.g. for supporting a content website)
 that uses ca. 14GB of memory and growing:
 
     docker run \
-        --env USE_WAY_TOO_MUCH_RAM=1 \
+        --env USE_WAY_TOO_MUCH_RAM=1 --env USE_FULL_WEB_NODE=1 \
         -d -p 2001:2001 -p 8090:8090 --name steemd-full \
         steemit/steem
 
     docker logs -f steemd-full
 
+# Environment variables
+
+There are quite a few environment variables that can be set to run steemd in different ways:
+
+* `USE_WAY_TOO_MUCH_RAM` - if set to true, steemd starts a 'full node'
+* `USE_FULL_WEB_NODE` - if set to true, a default config file will be used that enables a full set of API's and associated plugins.
+* `USE_NGINX_FRONTEND` - if set to true, this will enable an NGINX reverse proxy in front of steemd that proxies websocket requests to steemd. This will also enable a custom healtcheck at the path '/health' that lists how many seconds away from current blockchain time your node is. It will return a '200' if it's less than 60 seconds away from synced.
+* `USE_MULTICORE_READONLY` - if set to true, this will enable steemd in multiple reader mode to take advantage of multiple cores (if available). Read requests are handled by the read-only nodes, and write requests are forwarded back to the single 'writer' node automatically. NGINX load balances all requests to the reader nodes, 4 per available core. This setting is still considered experimental and may have trouble with some API calls until further development is completed.
+* `HOME` - set this to the path where you want steemd to store it's data files (block log, shared memory, config file, etc). By default `/var/lib/steemd` is used and exists inside the docker container. If you want to use a different mountpoint (like a ramdisk, or a different drive) then you may want to set this variable to map the volume to your docker container.
+
+# PaaS mode
+
+Steemd now supports a PaaS mode (platform as a service) that currently works with Amazon's Elastic Beanstalk service. It can be launched using the following environment variables:
+
+* `USE_PAAS` - if set to true, steemd will launch in a format that works with AWS EB. Containers will exit upon failure so that they can be relaunched automatically by ECS. This mode assumes `USE_WAY_TOO_MUCH_RAM` and `USE_FULL_WEB_NODE`, they do not need to be also set.
+* `S3_BUCKET` - set this to the name of the S3 bucket where you will store shared memory files for steemd in Amazon S3. They will be stored compressed in bz2 format with the file name `blockchain-$VERSION-latest.tar.bz2`, where $VERSION is the release number followed by the git short commit hash stored in each docker image at `/etc/steemdversion`.
+* `SYNC_TO_S3` - if set to true, the node will function to only generate shared memory files and upload them to the specified S3 bucket. This makes fast deployments and autoscaling for steemd possible.
+
 # Seed Nodes
 
 A list of some seed nodes to get you started can be found in
@@ -76,9 +94,7 @@ on how to use lcov to check code test coverage.
 
 # System Requirements
 
-Minimum 8 GB RAM (16 GB Recommended).
-32 GB disk for full node or 12 GB for a consensus node. SSD is preferred.
-Any CPU with decent single core performance.
+For a full web node, you need at least 55GB of space available. Steemd uses a memory mapped file which currently holds 36GB of data and by default is set to use up to 40GB. The block log of the blockchain itself is a little over 10GB. It's highly recommended to run steemd on a fast disk such as an SSD or by placing the shared memory files in a ramdisk and using the `--shard-file-dir=/path` command line option to specify where. At least 16GB of memory is required for a full web node. Seed nodes (p2p mode) can run with as little as 4GB of memory. Any CPU with decent single core performance should be sufficient.
 
 On Linux use the following Virtual Memory configuration for the initial sync and subsequent replays. It is not needed for normal operation.
 

diff --git a/contrib/startpaassteemd.sh b/contrib/startpaassteemd.sh
@@ -1,10 +1,14 @@
 #!/bin/bash
-export HOME="/var/lib/steemd"
+
+VERSION=`cat /etc/steemdversion`
 
 STEEMD="/usr/local/steemd-full/bin/steemd"
 
 chown -R steemd:steemd $HOME
 
+# clean out data dir since it may be semi-persistent block storage on the ec2 with stale data
+rm -rf $HOME/*
+
 # seed nodes come from doc/seednodes.txt which is
 # installed by docker into /etc/steemd/seednodes.txt
 SEED_NODES="$(cat /etc/steemd/seednodes.txt | awk -F' ' '{print $1}')"
@@ -38,29 +42,41 @@ mv /etc/nginx/nginx.conf /etc/nginx/nginx.original.conf
 cp /etc/nginx/steemd.nginx.conf /etc/nginx/nginx.conf
 
 # get blockchain state from an S3 bucket
-# if this url is not provieded then we might as well exit
-S3_DOWNLOAD_BUCKET=steemit-$NODE_ENV-blockchainstate
-echo steemd: beginning download and decompress of s3://$S3_DOWNLOAD_BUCKET/blockchain-$VERSION-latest.tar.bz2
-if [[ ! "$SYNC_TO_S3" ]]; then
+echo steemd: beginning download and decompress of s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2
+if [[ ! "$USE_RAMDISK" ]]; then
   mkdir -p /mnt/ramdisk
-  mount -t ramfs -o size=43008m ramfs /mnt/ramdisk
-  s3cmd get s3://$S3_DOWNLOAD_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x --wildcards 'blockchain/block*' -C /mnt/ramdisk 'blockchain/shared*'
-  ln -s blockchain/block_log /mnt/ramdisk/blockchain/block_log
-  ln -s blockchain/block_log.index /mnt/ramdisk/blockchain/block_log.index
+  mount -t ramfs -o size=${RAMDISK_SIZE_IN_MB:-43008}m ramfs /mnt/ramdisk
   ARGS+=" --shared-file-dir=/mnt/ramdisk/blockchain"
+  s3cmd get s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x --wildcards 'blockchain/block*' -C /mnt/ramdisk 'blockchain/shared*'
   chown -R steemd:steemd /mnt/ramdisk/blockchain
-else
-  s3cmd get s3://$S3_DOWNLOAD_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x
-  touch /tmp/issyncnode
-  chown www-data:www-data /tmp/issyncnode
+else  
+  s3cmd get s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x
 fi
 if [[ $? -ne 0 ]]; then
-  echo error: unable to pull blockchain state from S3 - exitting
-  exit 1
+  if [[ ! "$SYNC_TO_S3" ]]; then
+    echo notifyalert steemd: unable to pull blockchain state from S3 - exiting
+    exit 1
+  else
+    echo notifysteemdsync steemdsync: shared memory file for $VERSION not found, creating a new one by replaying the blockchain
+    mkdir blockchain
+    aws s3 cp s3://$S3_BUCKET/block_log-latest blockchain/block_log
+    if [[ $? -ne 0 ]]; then
+      echo notifysteemdsync steemdsync: unable to pull latest block_log from S3, will sync from scratch.
+    else
+      ARGS+=" --replay-blockchain --force-validate"
+    fi
+    touch /tmp/isnewsync
+  fi
+fi
+
+cd $HOME
+
+if [[ "$SYNC_TO_S3" ]]; then
+  touch /tmp/issyncnode
+  chown www-data:www-data /tmp/issyncnode
 fi
 
-# change owner of downloaded blockchainstate to steemd user
-chown -R steemd:steemd /var/lib/steemd/*
+chown -R steemd:steemd $HOME/*
 
 # start multiple read-only instances based on the number of cores
 # attach to the local interface since a proxy will be used to loadbalance
@@ -92,7 +108,6 @@ if [[ "$USE_MULTICORE_READONLY" ]]; then
         $STEEMD \
           --rpc-endpoint=127.0.0.1:$PORT_NUM \
           --data-dir=$HOME \
-          --shared-file-dir=/mnt/ramdisk/blockchain \
           --read-forward-rpc=127.0.0.1:8091 \
           --read-only \
           2>&1 &

diff --git a/contrib/steemd.nginx.conf b/contrib/steemd.nginx.conf
@@ -23,15 +23,6 @@ http {
 	include /etc/nginx/mime.types;
 	default_type application/octet-stream;
 
-        # rate limiting disabled for SBDS
-        #limit_conn_zone $binary_remote_addr zone=addr:100m;
-        #limit_req_zone $binary_remote_addr zone=public:100m rate=20r/s;
-
-        #limit_conn addr 32;
-        #limit_req zone=public burst=30;
-        #limit_conn_log_level error;
-        #limit_req_log_level error;
-
     log_format main '$remote_addr - $remote_user [$time_local]  '
                     '"$request" $status $body_bytes_sent '
                     '|"$http_referer"| "$http_user_agent"';

diff --git a/contrib/steemd.run b/contrib/steemd.run
@@ -1,9 +1,9 @@
 #!/bin/bash
 
-export HOME="/var/lib/steemd"
-
 STEEMD="/usr/local/steemd-default/bin/steemd"
 
+VERSION=`cat /etc/steemdversion`
+
 if [[ "$USE_WAY_TOO_MUCH_RAM" ]]; then
     STEEMD="/usr/local/steemd-full/bin/steemd"
 fi
@@ -69,9 +69,17 @@ fi
 # who knows what else it dumps into current dir
 cd $HOME
 
+if [[ "$USE_PUBLIC_SHARED_MEMORY" ]]; then
+  echo steemd: Downloading and uncompressing blockchain-$VERSION-latest.tar.bz2 - this may take awhile.
+  wget -qO- https://s3.amazonaws.com/steemit-dev-blockchainstate/blockchain-$VERSION-latest.tar.bz2 | pbzip2 -m2000dc | tar x
+fi
+
 # slow down restart loop if flapping
 sleep 1
 
+mv /etc/nginx/nginx.conf /etc/nginx/nginx.original.conf
+cp /etc/nginx/steemd.nginx.conf /etc/nginx/nginx.conf
+
 #start multiple read-only instances based on the number of cores
 #attach to the local interface since a proxy will be used to loadbalance
 if [[ "$USE_MULTICORE_READONLY" ]]; then
@@ -83,21 +91,20 @@ if [[ "$USE_MULTICORE_READONLY" ]]; then
             $ARGS \
             $STEEMD_EXTRA_OPTS \
             2>&1 &
-    #sleep for a moment to allow the writer node to be ready to accept connections from the readers
-    sleep 5
+    # sleep for a moment to allow the writer node to be ready to accept connections from the readers
+    sleep 30
     PORT_NUM=8092
-    #don't generate endpoints in haproxy config if it already exists
-    #this prevents adding to it if the docker container is stopped/started
-    if [[ ! -f /etc/haproxy/haproxy.steem.cfg ]]; then
-        cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.steem.cfg
-        for (( i=2; i<=$(nproc); i++ ))
-          do
-            echo server server$PORT_NUM 127.0.0.1:$PORT_NUM maxconn 10000 weight 10 cookie server$PORT_NUM check >> /etc/haproxy/haproxy.steem.cfg
-            ((PORT_NUM++))
-        done
-    fi
+    cp /etc/nginx/healthcheck.conf.template /etc/nginx/healthcheck.conf
+    CORES=$(nproc)
+    PROCESSES=$((CORES * 4))
+    for (( i=2; i<=$PROCESSES; i++ ))
+      do
+        echo server 127.0.0.1:$PORT_NUM\; >> /etc/nginx/healthcheck.conf
+        ((PORT_NUM++))
+    done
+    echo } >> /etc/nginx/healthcheck.conf
     PORT_NUM=8092
-    for (( i=2; i<=$(nproc); i++ ))
+    for (( i=2; i<=$PROCESSES; i++ ))
       do
         exec chpst -usteemd \
         $STEEMD \
@@ -109,10 +116,30 @@ if [[ "$USE_MULTICORE_READONLY" ]]; then
           ((PORT_NUM++))
           sleep 1
     done
-    #start haproxy now that the config file is complete with all endpoints
-    #all of the read-only processes will connect to the write node onport 8091
-    #haproxy will balance all incoming traffic on port 8090
-    /usr/sbin/haproxy -f /etc/haproxy/haproxy.steem.cfg 2>&1
+    # start nginx now that the config file is complete with all endpoints
+    # all of the read-only processes will connect to the write node onport 8091
+    # nginx will balance all incoming traffic on port 8090
+    rm /etc/nginx/sites-enabled/default
+    cp /etc/nginx/healthcheck.conf /etc/nginx/sites-enabled/default
+    /etc/init.d/fcgiwrap restart
+    echo daemon off\; >> /etc/nginx/nginx.conf
+    service nginx restart
+elif [[ "$USE_NGINX_FRONTEND" ]]; then
+    cp /etc/nginx/healthcheck.conf.template /etc/nginx/healthcheck.conf
+    echo server 127.0.0.1:8091\; >> /etc/nginx/healthcheck.conf
+    echo } >> /etc/nginx/healthcheck.conf
+    rm /etc/nginx/sites-enabled/default
+    cp /etc/nginx/healthcheck.conf /etc/nginx/sites-enabled/default
+    /etc/init.d/fcgiwrap restart
+    service nginx restart
+    exec chpst -usteemd \
+        $STEEMD \
+            --rpc-endpoint=0.0.0.0:8091 \
+            --p2p-endpoint=0.0.0.0:2001 \
+            --data-dir=$HOME \
+            $ARGS \
+            $STEEMD_EXTRA_OPTS \
+            2>&1
 else
     exec chpst -usteemd \
         $STEEMD \

diff --git a/contrib/sync-sv-run.sh b/contrib/sync-sv-run.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+VERSION=`cat /etc/steemdversion`
+
 # if the writer node dies by itself, kill runsv causing the container to exit
 STEEMD_PID=`pgrep -f p2p-endpoint`
 if [[ ! $? -eq 0 ]]; then
@@ -33,21 +35,27 @@ if [[ ! -z "$BLOCKCHAIN_TIME" ]]; then
     echo steemdsync: compressing blockchainstate...
     tar cf blockchain.tar.bz2 --use-compress-prog=pbzip2 blockchain
     FILE_NAME=blockchain-$VERSION-`date '+%Y%m%d-%H%M%S'`.tar.bz2
-    S3_UPLOAD_BUCKET=steemit-$NODE_ENV-blockchainstate
-    echo steemdsync: uploading $FILE_NAME to $S3_UPLOAD_BUCKET
-    aws s3 cp blockchain.tar.bz2 s3://$S3_UPLOAD_BUCKET/$FILE_NAME
+    echo steemdsync: uploading $FILE_NAME to $S3_BUCKET
+    aws s3 cp blockchain.tar.bz2 s3://$S3_BUCKET/$FILE_NAME
     if [[ ! $? -eq 0 ]]; then
-    	echo NOTIFYALERT! steemdsync was unable to upload $FILE_NAME to s3://$S3_UPLOAD_BUCKET
+    	echo NOTIFYALERT! steemdsync was unable to upload $FILE_NAME to s3://$S3_BUCKET
     	exit 1
     fi
     echo steemdsync: replacing current version of blockchain-latest.tar.bz2 with $FILE_NAME
-    aws s3 cp s3://$S3_UPLOAD_BUCKET/$FILE_NAME s3://$S3_UPLOAD_BUCKET/blockchain-$VERSION-latest.tar.bz2
+    aws s3 cp s3://$S3_BUCKET/$FILE_NAME s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2
+    aws s3api put-object-acl --bucket $S3_BUCKET --key blockchain-$VERSION-latest.tar.bz2 --acl public-read 
     if [[ ! $? -eq 0 ]]; then
     	echo NOTIFYALERT! steemdsync was unable to overwrite the current blockchainstate with $FILE_NAME
     	exit 1
     fi
+    # upload a current block_log
+    aws s3 cp blockchain/block_log s3://$S3_BUCKET/block_log-intransit
+    aws s3 cp s3://$S3_BUCKET/block_log-intransit s3://$S3_BUCKET/block_log-latest
     # kill the container starting the process over again
     echo steemdsync: stopping the container after a sync operation
+    if [[ -e /tmp/isnewsync ]]; then
+      echo notifysteemdsync: steemdsync: successfully generated and uploaded new blockchain-$VERSION-latest.tar.bz2 to s3://$S3_BUCKET
+    fi
     RUN_SV_PID=`pgrep -f /etc/service/steemd`
     kill -9 $RUN_SV_PID
   fi