Added h2o-cluster-distribute-aws-credentials.sh.

Updated AMI.
Now supports S3N access.

ec2/README.txt

@@ -19,8 +19,14 @@ At a minimum, you need to specify an ssh key name and a security group name.
 
 % ./h2o-cluster-launch-instances.py
 % ./h2o-cluster-distribute-h2o.sh  --OR--  ./h2o-cluster-download-h2o.sh
+% [optional] ./h2o-cluster-distribute-aws-credentials.sh
 
-(Download may be faster than distribute, since download pulls from S3.)
+Note:  Download may be faster than distribute, since download pulls from S3.
+
+Note:  Distributing the AWS credentials copies the Amazon AWS_ACCESS_KEY_ID
+       and AWS_SECRET_ACCESS_KEY to the instances.  This enables S3 and S3N
+       access.  Take precaution when putting your security keys in the 
+       cloud.
 
 
 STEP 2:  Start H2O, one H2O node per EC2 instance
@@ -61,6 +67,10 @@ Control files (generated when starting the cluster and/or H2O)
     project_version (produced by h2o-cluster-download-h2o.sh)
         Full project version number for the requested build.
 
+    core-site.xml (produced by ./h2o-cluster-distribute-aws-credentials.sh)
+    aws_credentials.properties (produced by ./h2o-cluster-distribute-aws-credentials.sh)
+        AWS credentials copied to each instance.
+
 
 Stopping/Terminating the cluster
 --------------------------------

ec2/ami/start-h2o-bg.sh

@@ -18,6 +18,28 @@ ${JAVA_HOME}/bin/java -version
 # Check that we can at least run H2O with the given java.
 ${JAVA_HOME}/bin/java -jar h2o.jar -version
 
+# HDFS credentials.
+hdfs_config_option=""
+hdfs_config_value=""
+hdfs_option=""
+hdfs_option_value=""
+hdfs_version=""
+if [ -f .ec2/core-site.xml ]
+then
+    hdfs_config_option="-hdfs_config"
+    hdfs_config_value=".ec2/core-site.xml"
+    hdfs_option="-hdfs"
+    hdfs_option_value="hdfs://10.78.14.235:9000"
+    hdfs_version="-hdfs_version=0.20.2"
+fi
+
+# AWS credentials.
+aws_credentials=""
+if [ -f .ec2/aws_credentials.properties ]
+then
+    aws_credentials="--aws_credentials=.ec2/aws_credentials.properties"
+fi
+
 # Start H2O disowned in the background.
-nohup ${JAVA_HOME}/bin/java -Xmx${xmxMb}m -jar h2o.jar -name H2ODemo -flatfile flatfile.txt -port 54321 -beta -ice_root ${d}/ice_root 1> h2o.out 2> h2o.err &
+nohup ${JAVA_HOME}/bin/java -Xmx${xmxMb}m -jar h2o.jar -name H2ODemo -flatfile flatfile.txt -port 54321 -beta -ice_root ${d}/ice_root ${hdfs_config_option} ${hdfs_config_value} ${hdfs_option} ${hdfs_option_value} ${hdfs_version} ${aws_credentials} 1> h2o.out 2> h2o.err &
 

ec2/h2o-cluster-distribute-aws-credentials.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+set -e
+
+if [ -z ${AWS_ACCESS_KEY_ID} ]
+then
+    echo "ERROR: You must set AWS_ACCESS_KEY_ID in the environment."
+    exit 1
+fi
+
+if [ -z ${AWS_SECRET_ACCESS_KEY} ]
+then
+    echo "ERROR: You must set AWS_SECRET_ACCESS_KEY in the environment."
+    exit 1
+fi
+
+if [ -z ${AWS_SSH_PRIVATE_KEY_FILE} ]
+then
+    echo "ERROR: You must set AWS_SSH_PRIVATE_KEY_FILE in the environment."
+    exit 1
+fi
+
+coreSiteFileName=core-site.xml
+rm -f ${coreSiteFileName}
+cat <<EOF1 > ${coreSiteFileName}
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+<!--
+<property>
+  <name>fs.default.name</name>
+  <value>s3n://h2o-datasets</value>
+</property>
+-->
+
+<property>
+  <name>fs.s3n.awsAccessKeyId</name>
+  <value>${AWS_ACCESS_KEY_ID}</value>
+</property>
+
+<property>
+  <name>fs.s3n.awsSecretAccessKey</name>
+  <value>${AWS_SECRET_ACCESS_KEY}</value>
+</property>
+
+</configuration>
+EOF1
+
+awsCredentialsPropertiesFileName=aws_credentials.properties
+rm -f ${awsCredentialsPropertiesFileName}
+cat <<EOF2 > ${awsCredentialsPropertiesFileName}
+accessKey=${AWS_ACCESS_KEY_ID}
+secretKey=${AWS_SECRET_ACCESS_KEY}
+EOF2
+
+i=0
+for publicDnsName in $(cat nodes-public)
+do
+    i=$((i+1))
+    echo "Copying aws credential files to node ${i}: ${publicDnsName}"
+    ssh -o StrictHostKeyChecking=no -i ${AWS_SSH_PRIVATE_KEY_FILE} ec2-user@${publicDnsName} mkdir -p .ec2
+    scp -p -o StrictHostKeyChecking=no -i ${AWS_SSH_PRIVATE_KEY_FILE} ${coreSiteFileName} ${awsCredentialsPropertiesFileName} ec2-user@${publicDnsName}:.ec2
+done
+
+echo Success.

ec2/h2o-cluster-launch-instances.py

@@ -49,7 +49,7 @@
 # --------------------------------------------------------
 
 regionName = 'us-east-1'
-amiId = 'ami-0ff1a466'
+amiId = 'ami-ed550784'
 
 
 #--------------------------------------------------------------------------