Merge pull request #22 from andjelx/az_fix_20160802

Az fix 20160802

scripts/00-aws-bootstrap.sh

@@ -17,7 +17,8 @@ fi
 export LC_ALL="en_US.UTF-8"
 export LOGFILE=/tmp/`date +"%Y-%m-%d"`.log
 export SUMLOGFILE=/tmp/`date +"%Y-%m-%d"`summary.log
-
+# To run xdfile based scripts below
+export PYTHONPATH=.
 
 exec > >(tee -i ${LOGFILE}) 2>&1
 echo 'SUMMARY: Start time:'`date +'%Y-%m-%d %H:%M'`

scripts/00-aws-ec2-launch-manual.sh

@@ -1,12 +1,14 @@
 #!/bin/bash -x
 
-source src/aws/config
+#source src/aws/config
 
 XD_PROFILE="arn:aws:iam::165509303398:instance-profile/xd-scraper"
 
-echo aws s3 cp src/aws/config.century-arcade s3://xd-private/etc/config
+echo aws s3 cp src/aws/config s3://xd-private/etc/config
 
 ami_id=ami-75fd3b15 #Ubuntu Server 16.04 LTS (HVM)
+ssh_security_gid=sg-e00fbe87
+INSTANCE_JSON=/tmp/instance.json
 
 #  created via IAM console: role/xd-scraper
 aws ec2 run-instances \
@@ -16,4 +18,10 @@ aws ec2 run-instances \
       --instance-initiated-shutdown-behavior terminate \
       --iam-instance-profile Arn="$XD_PROFILE" \
       --user-data file://scripts/00-aws-bootstrap.sh \
-      --image-id $ami_id
+      --image-id $ami_id > $INSTANCE_JSON
+
+instance_id=$(cat $INSTANCE_JSON | grep instance_id)
+echo aws ec2 modify-instance-attribute --groups $ssh_security_gid --instance-id $instance_id
+
+public_ip=$(aws ec2 describe-instances | grep PublicIp)
+echo ssh -i ~/*.pem ubuntu@$public_ip

scripts/10-import.sh

@@ -8,8 +8,8 @@
 RECENTS=$GXD/recent-downloads.tsv
 EMAILZIP=$TMP/$NOW-email.zip
 WWWZIP=$TMP/$NOW-www.zip
-sh="echo sh"
-aws="echo aws"
+sh="sh"
+aws="aws"
 
 set -e
 set -x

scripts/36-mkwww-deepclues.py

@@ -121,7 +121,7 @@ def main():
     outf = utils.open_output()
 
     similars = utils.parse_tsv('gxd/similar.tsv', 'Similar')
-    xdids_todo = [ parse_pathname(fn).base for fn in args.inputs ] 
+    xdids_todo = [ parse_pathname(fn).base for fn in args.inputs ]
     if not xdids_todo:
         xdids_todo = [ xdid for xdid, matches in metadb.get_similar_grids().items() if matches ]
 
@@ -144,13 +144,13 @@ def main():
         nstaleanswers = 0
         ntotalclues = 0
 
-        poss_answers = [] # TODO: 
+        poss_answers = [] # TODO:
         pub_uses = {}  # [pubid] -> set(ClueAnswer)
 
-        dcl_html = '' 
+        dcl_html = ''
         deepcl_html = [] # keep deep clues to parse later - per row
         for pos, mainclue, mainanswer in mainxd.iterclues():
-            deepcl_html = [] # Temporary to be replaced lately
+            deepcl_html = [] # Temporary to be replaced late
             mainca = ClueAnswer(mainxdid, mainxd.date(), mainanswer, mainclue)
 
             # 'grid position' column
@@ -233,7 +233,7 @@ def main():
                 nstaleclues += 1
             ntotalclues += 1
             # Quick and dirty - to be replaced
-            dcl_html += '<tr>' + ' '.join(deepcl_html) + '</tr>'   
+            dcl_html += '<tr>' + ' '.join(deepcl_html) + '</tr>'
 
         # Store in list to make further formatting as html table easier
         mainxd = xdfile.get_xd(mainxdid)
@@ -248,7 +248,6 @@ def main():
             diff_h += mktag('span', tagclass='main', inner='&nbsp;~&nbsp;' + mainanswer.upper())
             diff_l.append(diff_h)
         html_clues[mainxdid] = diff_l
-
         # Process for all matches
         for xdid in matches:
             xd = xdfile.get_xd(xdid)
@@ -257,11 +256,10 @@ def main():
             xddates[xdid] = xd.date()
             # output each grid
             html_grids[xdid] = grid_diff_html(xd, compare_with=mainxd)
-
             diff_l = []
             # output comparison of each set of clues
             for pos, clue, answer in xd.iterclues():
-                diff_h = mktag('div','fullgrid') + '%s.&nbsp;' %pos 
+                diff_h = mktag('div','fullgrid') + '%s.&nbsp;' %pos
                 # Sometimes can return clue == None
                 sm = difflib.SequenceMatcher(lambda x: x == ' ', mainxd.get_clue(pos) or '', clue)
                 if sm.ratio() < 0.50:
@@ -274,35 +272,30 @@ def main():
                             diff_h += '<span class="match">%s</span>' % clue[b1:b2]
                         else:
                             diff_h += '<span class="diff">%s</span>' % clue[b1:b2]
-
                 diff_h += mktag('span', tagclass=(answer == mainxd.get_answer(pos)) and 'match' or 'diff', inner='&nbsp;~&nbsp;' + answer.upper())
                 diff_h += mktag('/div')
                 diff_l.append(diff_h)
-            html_clues[xdid] = diff_l 
-
+            html_clues[xdid] = diff_l
 
-        print('writing table')
         # Wrap into table
         diff_h = mktag('table') + mktag('tr')
         # Sort by date
-        sortedkeys = sorted(xddates.items(), key=operator.itemgetter(1)) 
+        sortedkeys = sorted(xddates.items(), key=operator.itemgetter(1))
         for w, dt in sortedkeys:
             # Wrap into table
             diff_h += mktag('td') + html_grids[w] + mktag('/td')
         diff_h += mktag('/tr')
-
         for i, clue in enumerate(html_clues[sortedkeys[0][0]]):
             diff_h += mktag('tr')
             for w, dt in sortedkeys:
                 if i < len(html_clues[w]):
                     diff_h += mktag('td') + html_clues[w][i] + mktag('/td')
-            diff_h += mktag('/tr') 
+            diff_h += mktag('/tr')
         # Process deepclues
-        diff_h += mktag('table') + dcl_html + mktag('/table') 
+        diff_h += mktag('table') + dcl_html + mktag('/table')
 
         diff_h += mktag('/table')
-
-        outf.write_html('pub/deep/%s/index.html' % mainxdid, diff_h, 
+        outf.write_html('pub/deep/%s/index.html' % mainxdid, diff_h,
                     title='Deep clue comparison for ' + mainxdid)
 
 

scripts/send-email.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import xdfile.cloud
+from xdfile.cloud import xd_send_email
 import sys
 
-xdfile.cloud.xd_send_email(sys.argv[1], fromaddr='[email protected]', subject=sys.argv[2], body=open(sys.argv[3], 'r').read())
+xd_send_email(sys.argv[1], fromaddr='[email protected]', subject=sys.argv[2], body=open(sys.argv[3], 'r').read())