Staging

AWS.md

@@ -0,0 +1,22 @@
+# AWS commands
+
+## Extend root volume of instance
+1) Stop instance
+
+2) Create a snapshot
+aws ec2 create-snapshot --volume-id {cur_root_volume} --description 'Initial snapshot'
+
+3) Create new root volume; get snapshot id from previous output
+aws ec2 create-volume --size {new_size_gb} --region us-west-2 --availability-zone us-west-2a --volume-type standard --snapshot-id {snapshot_id_of_root_volume}
+
+4) Detach existing root volume
+aws ec2 detach-volume --volume-id {cur_root_volume}
+
+5) Attach new root volume
+aws ec2 attach-volume --volume-id {new_root_volume_step3} --instance-id i-5fc17aca --device /dev/sda1
+
+6) Start instance and process with partition extension
+http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-expand-volume.html#recognize-expanded-volume-linux
+
+7) If all ok, delete old root volume
+ws ec2 delete-volume --volume-id {cur_root_volume_step2}

scripts/00-aws-bootstrap.sh

@@ -52,7 +52,7 @@ echo "Import all .tsv to sql"
 scripts/05-sql-import-receipts.sh
 
 echo "Run deploy script"
-/bin/bash -x scripts/05-full-pipeline.sh
+/bin/bash scripts/05-full-pipeline.sh
 
 echo 'SUMMARY: End time '`date +'%Y-%m-%d %H:%M'`
 # Parse log to get summary to be mailed

scripts/00-aws-ebs-bootstrap.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+set -x
+
+WORKDIR=/tmp
+export SSHHOME=$HOME
+
+if [ -z "$HOME" ] ; then
+    export SSHHOME=$HOME
+    HOME=/tmp
+    # Hack for AWS where HOME not set
+    if [[ $UID -eq '0' ]]; then
+        export SSHHOME=/root
+    fi
+fi
+
+# This script is passed as userdata to the launch-config, which the base AMI
+# executes at the end of initialization.
+
+export LC_ALL="en_US.UTF-8"
+export LOGFILE=/tmp/`date +"%Y-%m-%d"`.log
+export SUMLOGFILE=/tmp/`date +"%Y-%m-%d"`summary.log
+# To run xdfile based scripts below
+export PYTHONPATH=.
+
+exec > >(tee -i ${LOGFILE}) 2>&1
+echo 'SUMMARY: Start time:'`date +'%Y-%m-%d %H:%M'`
+
+# export DEBIAN_FRONTEND=noninteractive
+# sudo apt-get update && \
+#    sudo apt-get install --yes language-pack-en-base zip awscli python3-lxml python3-pip git markdown python3-boto3 sqlite3 && \
+#    sudo pip3 install cssselect botocore
+
+cd $WORKDIR
+# Get config file from AWS
+aws s3 cp --region=us-west-2 s3://xd-private/etc/config $WORKDIR/config
+source $WORKDIR/config
+
+echo "Clone main project repo and switch to branch ${BRANCH}"
+git clone ${XD_GIT}
+cd xd/
+git checkout ${BRANCH}
+# Export all config vars
+source scripts/config-vars.sh
+
+mkdir -p $SSHHOME/.ssh
+echo "Clone GXD repo"
+aws s3 cp --region=us-west-2 s3://xd-private/etc/gxd_rsa $SSHHOME/.ssh/
+chmod 600 $SSHHOME/.ssh/gxd_rsa
+
+cat src/aws/ssh_config >> $SSHHOME/.ssh/config
+ssh-agent bash -c "ssh-add $SSHHOME/.ssh/gxd_rsa; git clone ${GXD_GIT}"
+
+echo "Import all .tsv to sql"
+scripts/05-sql-import-receipts.sh
+
+echo "Run deploy script"
+/bin/bash scripts/05-full-pipeline.sh
+
+echo 'SUMMARY: End time '`date +'%Y-%m-%d %H:%M'`
+# Parse log to get summary to be mailed
+egrep -i 'ERROR|WARNING|SUMMARY' ${LOGFILE} > ${SUMLOGFILE}
+echo -e '\n' >> ${SUMLOGFILE}
+
+echo "Getting summary"
+scripts/48-stats.sh >> ${SUMLOGFILE}
+echo -e '\n' >> ${SUMLOGFILE}
+
+echo "SUMMARY: Full log file http://$BUCKET/logs/`basename ${LOGFILE}`" >> ${SUMLOGFILE}
+
+echo "Sending email"
+scripts/send-email.py $ADMIN_EMAIL "execution logs for $TODAY" ${SUMLOGFILE}
+
+echo "Copy logs to AWS"
+aws s3 cp --region ${REGION} --content-type='text/plain' ${LOGFILE} s3://${BUCKET}/logs/ --acl public-read
+aws s3 cp --region ${REGION} --content-type='text/plain' ${SUMLOGFILE} s3://${BUCKET}/logs/ --acl public-read
+
+echo "Make logs index page"
+scripts/49b-mkindex.sh

scripts/00-aws-ec2-launch-manual-ebs.sh

@@ -0,0 +1,58 @@
+#!/bin/bash -x
+#
+# Usage: $0 <config file>
+# see format below
+#
+# export KEY=
+# export BRANCH=
+# export REGION=
+# export AWS_ACCESS_KEY=
+# export AWS_SECRET_KEY=
+# export BUCKET=
+# export EMAIL=
+# export XD_GIT=
+# export GXD_GIT=
+# export XD_PROFILE=
+# export AMI_ID=ami-75fd3b15 #Ubuntu Server 16.04 LTS (HVM)
+# export SSH_SECURITY_GID=sg-e00fbe87 # SSH access
+# export INSTANCE_TYPE=r3.large
+# export QUICKRUN=True # For quickrun scipping 20- and 30- scripts
+#
+#source src/aws/config
+
+aws="aws"
+sh="bash"
+
+XDCONFIG=$1
+if [ -n "$XDCONFIG" ]; then
+    aws s3 cp $XDCONFIG s3://xd-private/etc/config
+    source ${XDCONFIG}
+    # AMIID - 16.04 LTS amd64   hvm:ebs-ssd
+    # https://cloud-images.ubuntu.com/locator/ec2/
+    AMI_ID=ami-9ece19fe
+    INSTANCE_JSON=/tmp/instance.json
+
+    #  created via IAM console: role/xd-scraper
+    $aws ec2 run-instances \
+      --key-name $KEY \
+      --region ${REGION} \
+      --instance-type ${INSTANCE_TYPE} \
+      --block-device-mappings '[{"DeviceName":"/dev/sda1","Ebs":{"DeleteOnTermination":false}}]' \
+      --instance-initiated-shutdown-behavior stop \
+      --iam-instance-profile Arn="$XD_PROFILE" \
+      --user-data file://scripts/00-aws-bootstrap.sh \
+      --image-id ${AMI_ID} > $INSTANCE_JSON
+
+    # Wait a litte before applying sec group
+    sleep 30
+    instance_id=$(cat $INSTANCE_JSON | jq -r .Instances[0].InstanceId)
+    $aws ec2 modify-instance-attribute --groups ${SSH_SECURITY_GID} --instance-id $instance_id
+
+    public_ip=$(aws ec2 describe-instances --instance-ids ${instance_id} | jq -r '.Reservations[0].Instances[0].PublicIpAddress')
+    echo "Connecting: ssh -i ~/*.pem ubuntu@$public_ip" 
+    ssh -i ~/*.pem ubuntu@$public_ip
+
+else
+    echo "Supply config file: $0 <config>"
+    exit 1
+fi

scripts/00-ebs-snapshots.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# Manage snapshots for EBS storage
+# usage: $0 <instance_id>
+
+instance_id=$1
+
+instance_status=$(aws ec2 describe-instances --instance-ids ${instance_id} | jq -r '.Reservations[0].Instances[0].State')
+volume_id=$(aws ec2 describe-instances --instance-ids ${instance_id} | jq -r '.Reservations[0].Instances[0].BlockDeviceMappings[0].Ebs.VolumeId')
+
+echo "Instance status"
+echo "${instance_status}"
+
+# Get all snapshots for volume
+echo "Snapshots"
+aws ec2 describe-snapshots --filter Name=volume-id,Values=${volume_id}

scripts/00-ebs-start.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# Start ebs based instance
+# Usage: $0 <instance_id>
+#
+
+instance_id=$1
+aws ec2 start-instances --instance-ids ${instance_id}
+sleep 10
+
+instance_status=$(aws ec2 describe-instances --instance-ids ${instance_id} | jq -r '.Reservations[0].Instances[0].State')
+
+echo ${instance_status}
+
+public_ip=$(aws ec2 describe-instances --instance-ids ${instance_id} | jq -r '.Reservations[0].Instances[0].PublicIpAddress')
+
+echo "Connect in few seconds: ssh -i ~/*.pem ubuntu@$public_ip"

scripts/05-full-pipeline.sh

@@ -34,7 +34,7 @@ if [ ! -n "$QUICKRUN" ]; then
 fi
 
 # commit new puzzles and saved analysis results
-/bin/bash scripts/41-git-commit.sh incoming_$NOW
+/bin/bash scripts/41-git-commit.sh
 
 # capture all logs even if other scripts fail
 scripts/39-mkwww-logs.py -o $WWW/$NOW/index.html $TMP

scripts/05-sql-import-receipts.sh

@@ -6,5 +6,11 @@ METADB=meta.db
 
 if [ ! -f $METADB ] ; then
     sqlite3 $METADB < ./scripts/meta.sql
-    ./scripts/tsv2sqlite.py ${DEBUG} -o ${METADB} gxd/receipts.tsv
+    ./scripts/tsv2sqlite.py ${DEBUG} --tsvtype "Receipt" -o ${METADB} gxd/receipts.tsv
+    ./scripts/tsv2sqlite.py ${DEBUG} --tsvtype "Publication" -o ${METADB} gxd/publications.tsv
+    ./scripts/tsv2sqlite.py ${DEBUG} --tsvtype "Similar" -o ${METADB} gxd/similar.tsv
+else
+    echo "$METADB already exists"
 fi
+
+

scripts/20-analyze.sh

@@ -8,9 +8,12 @@ mkdir -p $PUB
 rm -f $PUB/*
 
 # regenerate pub/puzzles.tsv
-scripts/21-clean-metadata.py -o $PUB/puzzles.tsv $GXD
+# TODO: should populate puzzles table in sqlite instead
+scripts/21b-clean-metadata.py $GXD
 
-# regenerate pub/pubyears.tsv
+# generate pubyears just for now TODO: to be replaced
 scripts/22-pubyears.py
+# regenerate pub/pubyears.tsv
 scripts/25-analyze-puzzle.py -o $WWW/ -c $GXD $GXD
 scripts/26-clues-tsv.py -c $GXD -o $PUB/
+scripts/27-pubyear-stats.py -c ${GXD}