More ec2 fixes.

Update ami to not include any h2o by default.
Add test ssh script.

Author: tomkraljevic

ec2/h2o-cluster-download-h2o.sh

@@ -10,19 +10,42 @@ fi
 
 # Adjust based on the build of H2O you want to download.
 h2oBranch=master
-curl -o latest https://h2o-release.s3.amazonaws.com/h2o/master/latest
+
+echo "Fetching latest build number for branch ${h2oBranch}..."
+curl --silent -o latest https://h2o-release.s3.amazonaws.com/h2o/master/latest
 h2oBuild=`cat latest`
-curl -o project_version https://h2o-release.s3.amazonaws.com/h2o/master/${h2oBuild}/project_version
+
+echo "Fetching full version number for build ${h2oBuild}..."
+curl --silent -o project_version https://h2o-release.s3.amazonaws.com/h2o/master/${h2oBuild}/project_version
 h2oVersion=`cat project_version`
 
+echo "Downloading H2O version ${h2oVersion} to cluster..."
+
 i=0
 for publicDnsName in $(cat nodes-public)
 do
     i=$((i+1))
     echo "Downloading h2o.jar to node ${i}: ${publicDnsName}"
-    ssh -o StrictHostKeyChecking=no -i ${AWS_SSH_PRIVATE_KEY_FILE} ec2-user@${publicDnsName} curl -o h2o-${h2oVersion}.zip https://s3.amazonaws.com/h2o-release/h2o/${h2oBranch}/${h2oBuild}/h2o-${h2oVersion}.zip
-    ssh -o StrictHostKeyChecking=no -i ${AWS_SSH_PRIVATE_KEY_FILE} ec2-user@${publicDnsName} unzip -o h2o-${h2oVersion}.zip
-    ssh -o StrictHostKeyChecking=no -i ${AWS_SSH_PRIVATE_KEY_FILE} ec2-user@${publicDnsName} cp -f h2o-${h2oVersion}/h2o.jar .
+    ssh -o StrictHostKeyChecking=no -i ${AWS_SSH_PRIVATE_KEY_FILE} ec2-user@${publicDnsName} curl --silent -o h2o-${h2oVersion}.zip https://s3.amazonaws.com/h2o-release/h2o/${h2oBranch}/${h2oBuild}/h2o-${h2oVersion}.zip &
+done
+wait
+
+i=0
+for publicDnsName in $(cat nodes-public)
+do
+    i=$((i+1))
+    echo "Unzipping h2o.jar within node ${i}: ${publicDnsName}"
+    ssh -o StrictHostKeyChecking=no -i ${AWS_SSH_PRIVATE_KEY_FILE} ec2-user@${publicDnsName} unzip -o h2o-${h2oVersion}.zip 1> /dev/null &
+done
+wait
+
+i=0
+for publicDnsName in $(cat nodes-public)
+do
+    i=$((i+1))
+    echo "Copying h2o.jar within node ${i}: ${publicDnsName}"
+    ssh -o StrictHostKeyChecking=no -i ${AWS_SSH_PRIVATE_KEY_FILE} ec2-user@${publicDnsName} cp -f h2o-${h2oVersion}/h2o.jar . &
 done
+wait
 
 echo Success.

ec2/h2o-cluster-launch-instances.py

@@ -49,8 +49,8 @@
 # --------------------------------------------------------
 
 regionName = 'us-east-1'
-amiId = 'ami-17c18b7e'
-# amiId = 'ami-634f050a'                # old stable
+amiId = 'ami-0ff1a466'
+
 
 #--------------------------------------------------------------------------
 # No need to change anything below here.
@@ -124,10 +124,20 @@
 fpublic.close()
 fprivate.close()
 
-print
-print 'Distributing flatfile...'
+print 'Sleeping for 60 seconds for ssh to be available...'
+time.sleep(60)
 
 d = os.path.dirname(os.path.realpath(__file__))
+
+print 'Testing ssh access...'
+cmd = d + '/' + 'h2o-cluster-test-ssh.sh'
+rv = os.system(cmd)
+if rv != 0:
+    print 'Failed.'
+    sys.exit(1)
+
+print
+print 'Distributing flatfile...'
 cmd = d + '/' + 'h2o-cluster-distribute-flatfile.sh'
 rv = os.system(cmd)
 if rv != 0:

ec2/h2o-cluster-test-ssh.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+if [ -z ${AWS_SSH_PRIVATE_KEY_FILE} ]
+then
+    echo "ERROR: You must set AWS_SSH_PRIVATE_KEY_FILE in the environment."
+    exit 1
+fi
+
+i=0
+maxRetries=12
+retries=0
+for publicDnsName in $(cat nodes-public)
+do
+    if [ ${retries} -ge ${maxRetries} ]
+    then
+        echo "ERROR: Too many ssh retries."
+        exit 1
+    fi
+
+    i=$((i+1))
+    echo "Testing ssh to node ${i}: ${publicDnsName}"
+
+    while true
+    do
+        ssh -o StrictHostKeyChecking=no -i ${AWS_SSH_PRIVATE_KEY_FILE} ec2-user@${publicDnsName} hostname
+        if [ $? -eq 0 ]
+        then
+            break
+        else
+            retries=$((retries+1))
+
+            if [ ${retries} -ge ${maxRetries} ]
+            then
+                echo "ERROR: Too many ssh retries."
+                exit 1
+            fi
+
+            echo "Sleeping 5 seconds before retrying..."
+            sleep 5
+        fi
+    done
+done
+
+echo Success.