EBS bootstrap ammended. AWS manual created

AWS.md

@@ -0,0 +1,22 @@
+# AWS commands
+
+## Extend root volume of instance
+1) Stop instance
+
+2) Create a snapshot
+aws ec2 create-snapshot --volume-id {cur_root_volume} --description 'Initial snapshot'
+
+3) Create new root volume; get snapshot id from previous output
+aws ec2 create-volume --size {new_size_gb} --region us-west-2 --availability-zone us-west-2a --volume-type standard --snapshot-id {snapshot_id_of_root_volume}
+
+4) Detach existing root volume
+aws ec2 detach-volume --volume-id {cur_root_volume}
+
+5) Attach new root volume
+aws ec2 attach-volume --volume-id {new_root_volume_step3} --instance-id i-5fc17aca --device /dev/sda1
+
+6) Start instance and process with partition extension
+http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-expand-volume.html#recognize-expanded-volume-linux
+
+7) If all ok, delete old root volume
+ws ec2 delete-volume --volume-id {cur_root_volume_step2}

scripts/00-aws-ebs-bootstrap.sh

@@ -2,10 +2,12 @@
 
 set -x
 
+WORKDIR=/tmp
+export SSHHOME=$HOME
+
 if [ -z "$HOME" ] ; then
-    HOME=/home/src
-    mkdir -p $HOME
     export SSHHOME=$HOME
+    HOME=/tmp
     # Hack for AWS where HOME not set
     if [[ $UID -eq '0' ]]; then
         export SSHHOME=/root
@@ -24,37 +26,33 @@ export PYTHONPATH=.
 exec > >(tee -i ${LOGFILE}) 2>&1
 echo 'SUMMARY: Start time:'`date +'%Y-%m-%d %H:%M'`
 
-#export DEBIAN_FRONTEND=noninteractive
-#sudo apt-get update && \
+# export DEBIAN_FRONTEND=noninteractive
+# sudo apt-get update && \
 #    sudo apt-get install --yes language-pack-en-base zip awscli python3-lxml python3-pip git markdown python3-boto3 sqlite3 && \
 #    sudo pip3 install cssselect botocore
 
-cd $HOME
+cd $WORKDIR
 # Get config file from AWS
-aws s3 cp --region=us-west-2 s3://xd-private/etc/config $HOME/config
-source $HOME/config
+aws s3 cp --region=us-west-2 s3://xd-private/etc/config $WORKDIR/config
+source $WORKDIR/config
 
-echo "Update main project repo and switch to branch ${BRANCH}"
-#git clone ${XD_GIT}
+echo "Clone main project repo and switch to branch ${BRANCH}"
+git clone ${XD_GIT}
 cd xd/
 git checkout ${BRANCH}
-git pull
 # Export all config vars
 source scripts/config-vars.sh
 
-#mkdir -p $SSHHOME/.ssh
-echo "Update GXD repo"
-#aws s3 cp --region=us-west-2 s3://xd-private/etc/gxd_rsa $SSHHOME/.ssh/
-#chmod 600 $SSHHOME/.ssh/gxd_rsa
-
-#cat src/aws/ssh_config >> $SSHHOME/.ssh/config
-cd gxd/
-ssh-agent bash -c "ssh-add $SSHHOME/.ssh/gxd_rsa; git pull ${GXD_GIT}"
-cd ..
+mkdir -p $SSHHOME/.ssh
+echo "Clone GXD repo"
+aws s3 cp --region=us-west-2 s3://xd-private/etc/gxd_rsa $SSHHOME/.ssh/
+chmod 600 $SSHHOME/.ssh/gxd_rsa
 
+cat src/aws/ssh_config >> $SSHHOME/.ssh/config
+ssh-agent bash -c "ssh-add $SSHHOME/.ssh/gxd_rsa; git clone ${GXD_GIT}"
 
 echo "Import all .tsv to sql"
-rm meta.db && scripts/05-sql-import-receipts.sh
+scripts/05-sql-import-receipts.sh
 
 echo "Run deploy script"
 /bin/bash scripts/05-full-pipeline.sh

scripts/00-ebs-start.sh

@@ -5,7 +5,7 @@
 
 instance_id=$1
 aws ec2 start-instances --instance-ids ${instance_id}
-sleep 5
+sleep 10
 
 instance_status=$(aws ec2 describe-instances --instance-ids ${instance_id} | jq -r '.Reservations[0].Instances[0].State')