EBS scripts added

scripts/00-aws-bootstrap.sh

@@ -3,8 +3,7 @@
 set -x
 
 if [ -z "$HOME" ] ; then
-    HOME=/home/src
-    mkdir -p $HOME
+    HOME=/tmp
     export SSHHOME=$HOME
     # Hack for AWS where HOME not set
     if [[ $UID -eq '0' ]]; then

scripts/00-aws-ebs-bootstrap.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -x
+
+if [ -z "$HOME" ] ; then
+    HOME=/home/src
+    mkdir -p $HOME
+    export SSHHOME=$HOME
+    # Hack for AWS where HOME not set
+    if [[ $UID -eq '0' ]]; then
+        export SSHHOME=/root
+    fi
+fi
+
+# This script is passed as userdata to the launch-config, which the base AMI
+# executes at the end of initialization.
+
+export LC_ALL="en_US.UTF-8"
+export LOGFILE=/tmp/`date +"%Y-%m-%d"`.log
+export SUMLOGFILE=/tmp/`date +"%Y-%m-%d"`summary.log
+# To run xdfile based scripts below
+export PYTHONPATH=.
+
+exec > >(tee -i ${LOGFILE}) 2>&1
+echo 'SUMMARY: Start time:'`date +'%Y-%m-%d %H:%M'`
+
+#export DEBIAN_FRONTEND=noninteractive
+#sudo apt-get update && \
+#    sudo apt-get install --yes language-pack-en-base zip awscli python3-lxml python3-pip git markdown python3-boto3 sqlite3 && \
+#    sudo pip3 install cssselect botocore
+
+cd $HOME
+# Get config file from AWS
+aws s3 cp --region=us-west-2 s3://xd-private/etc/config $HOME/config
+source $HOME/config
+
+echo "Update main project repo and switch to branch ${BRANCH}"
+#git clone ${XD_GIT}
+cd xd/
+git checkout ${BRANCH}
+git pull
+# Export all config vars
+source scripts/config-vars.sh
+
+#mkdir -p $SSHHOME/.ssh
+echo "Update GXD repo"
+#aws s3 cp --region=us-west-2 s3://xd-private/etc/gxd_rsa $SSHHOME/.ssh/
+#chmod 600 $SSHHOME/.ssh/gxd_rsa
+
+#cat src/aws/ssh_config >> $SSHHOME/.ssh/config
+cd gxd/
+ssh-agent bash -c "ssh-add $SSHHOME/.ssh/gxd_rsa; git pull ${GXD_GIT}"
+cd ..
+
+
+echo "Import all .tsv to sql"
+rm meta.db && scripts/05-sql-import-receipts.sh
+
+echo "Run deploy script"
+/bin/bash scripts/05-full-pipeline.sh
+
+echo 'SUMMARY: End time '`date +'%Y-%m-%d %H:%M'`
+# Parse log to get summary to be mailed
+egrep -i 'ERROR|WARNING|SUMMARY' ${LOGFILE} > ${SUMLOGFILE}
+echo -e '\n' >> ${SUMLOGFILE}
+
+echo "Getting summary"
+scripts/48-stats.sh >> ${SUMLOGFILE}
+echo -e '\n' >> ${SUMLOGFILE}
+
+echo "SUMMARY: Full log file http://$BUCKET/logs/`basename ${LOGFILE}`" >> ${SUMLOGFILE}
+
+echo "Sending email"
+scripts/send-email.py $ADMIN_EMAIL "execution logs for $TODAY" ${SUMLOGFILE}
+
+echo "Copy logs to AWS"
+aws s3 cp --region ${REGION} --content-type='text/plain' ${LOGFILE} s3://${BUCKET}/logs/ --acl public-read
+aws s3 cp --region ${REGION} --content-type='text/plain' ${SUMLOGFILE} s3://${BUCKET}/logs/ --acl public-read
+
+echo "Make logs index page"
+scripts/49b-mkindex.sh

scripts/00-aws-ec2-launch-manual-ebs.sh

@@ -0,0 +1,58 @@
+#!/bin/bash -x
+#
+# Usage: $0 <config file>
+# see format below
+#
+# export KEY=
+# export BRANCH=
+# export REGION=
+# export AWS_ACCESS_KEY=
+# export AWS_SECRET_KEY=
+# export BUCKET=
+# export EMAIL=
+# export XD_GIT=
+# export GXD_GIT=
+# export XD_PROFILE=
+# export AMI_ID=ami-75fd3b15 #Ubuntu Server 16.04 LTS (HVM)
+# export SSH_SECURITY_GID=sg-e00fbe87 # SSH access
+# export INSTANCE_TYPE=r3.large
+# export QUICKRUN=True # For quickrun scipping 20- and 30- scripts
+#
+#source src/aws/config
+
+aws="aws"
+sh="bash"
+
+XDCONFIG=$1
+if [ -n "$XDCONFIG" ]; then
+    aws s3 cp $XDCONFIG s3://xd-private/etc/config
+    source ${XDCONFIG}
+    # AMIID - 16.04 LTS amd64   hvm:ebs-ssd
+    # https://cloud-images.ubuntu.com/locator/ec2/
+    AMI_ID=ami-9ece19fe
+    INSTANCE_JSON=/tmp/instance.json
+
+    #  created via IAM console: role/xd-scraper
+    $aws ec2 run-instances \
+      --key-name $KEY \
+      --region ${REGION} \
+      --instance-type ${INSTANCE_TYPE} \
+      --block-device-mappings '[{"DeviceName":"/dev/sda1","Ebs":{"DeleteOnTermination":false}}]' \
+      --instance-initiated-shutdown-behavior stop \
+      --iam-instance-profile Arn="$XD_PROFILE" \
+      --user-data file://scripts/00-aws-bootstrap.sh \
+      --image-id ${AMI_ID} > $INSTANCE_JSON
+
+    # Wait a litte before applying sec group
+    sleep 30
+    instance_id=$(cat $INSTANCE_JSON | jq -r .Instances[0].InstanceId)
+    $aws ec2 modify-instance-attribute --groups ${SSH_SECURITY_GID} --instance-id $instance_id
+
+    public_ip=$(aws ec2 describe-instances --instance-ids ${instance_id} | jq -r '.Reservations[0].Instances[0].PublicIpAddress')
+    echo "Connecting: ssh -i ~/*.pem ubuntu@$public_ip" 
+    ssh -i ~/*.pem ubuntu@$public_ip
+
+else
+    echo "Supply config file: $0 <config>"
+    exit 1
+fi