Merge branch 'MM_S3_client'

src/main/java/water/H2O.java

@@ -917,17 +917,10 @@ public static void main( String[] args ) {
     Log.POST(380,"");
   }
 
-  // Default location of the AWS credentials file
-  public static final String DEFAULT_CREDENTIALS_LOCATION = "AwsCredentials.properties";
-  public static PropertiesCredentials getAWSCredentials() throws IOException {
-    File credentials = new File(Objects.firstNonNull(OPT_ARGS.aws_credentials, DEFAULT_CREDENTIALS_LOCATION));
-    return new PropertiesCredentials(credentials);
-  }
-
   /** Starts the local k-v store.
-* Initializes the local k-v store, local node and the local cloud with itself
-* as the only member.
-*/
+   * Initializes the local k-v store, local node and the local cloud with itself
+   * as the only member.
+   */
   private static void startLocalNode() {
     // Figure self out; this is surprisingly hard
     initializeNetworkSockets();

src/main/java/water/deploy/EC2.java

@@ -6,6 +6,7 @@
 import org.apache.commons.codec.binary.Base64;
 
 import water.H2O;
+import water.persist.PersistS3;
 import water.util.Log;
 import water.util.Utils;
 
@@ -63,17 +64,8 @@ public class EC2 {
    * Create or terminate EC2 instances. Uses their Name tag to find existing ones.
    */
   public Cloud resize() throws Exception {
-    PropertiesCredentials credentials;
-    try {
-      credentials = H2O.getAWSCredentials();
-    } catch( Exception ex ) {
-      System.out.println("Please provide your AWS credentials in './AwsCredentials.properties'");
-      System.out.println("File format is:");
-      System.out.println("accessKey=XXXXXXXXXXXXXXXXXXXX");
-      System.out.println("secretKey=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX");
-      throw ex;
-    }
-    AmazonEC2Client ec2 = new AmazonEC2Client(credentials);
+
+    AmazonEC2Client ec2 = new AmazonEC2Client(new PersistS3.H2OAWSCredentialsProviderChain());
     ec2.setEndpoint("ec2." + region + ".amazonaws.com");
     DescribeInstancesResult describeInstancesResult = ec2.describeInstances();
     List<Reservation> reservations = describeInstancesResult.getReservations();

src/main/java/water/persist/PersistS3.java

@@ -13,11 +13,12 @@
 import water.util.Log;
 import water.util.RIStream;
 
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
+import com.amazonaws.*;
+import com.amazonaws.auth.*;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.*;
+import com.google.common.base.Objects;
 import com.google.common.io.ByteStreams;
 
 /** Persistence backend for S3 */
@@ -35,7 +36,7 @@ public static AmazonS3 getClient() {
       synchronized( _lock ) {
         if( _s3 == null ) {
           try {
-            _s3 = new AmazonS3Client(H2O.getAWSCredentials(), s3ClientCfg());
+            _s3 = new AmazonS3Client(new H2OAWSCredentialsProviderChain(), s3ClientCfg());
           } catch( Throwable e ) {
             StringBuilder msg = new StringBuilder();
             msg.append(e.getMessage() + "\n");
@@ -49,6 +50,43 @@ public static AmazonS3 getClient() {
     return _s3;
   }
 
+  /** Modified version of default credentials provider which includes H2O-specific
+   * credentials provider.
+   */
+  public static class H2OAWSCredentialsProviderChain extends AWSCredentialsProviderChain {
+    public H2OAWSCredentialsProviderChain() {
+      super(new H2OArgCredentialsProvider(),
+            new InstanceProfileCredentialsProvider(),
+            new EnvironmentVariableCredentialsProvider(),
+            new SystemPropertiesCredentialsProvider());
+    }
+  }
+
+  /** A simple credentials provider reading file-based credentials from given
+   * command argument <code>--aws_credentials</code>.
+   */
+  static class H2OArgCredentialsProvider implements AWSCredentialsProvider {
+
+    // Default location of the AWS credentials file
+    public static final String DEFAULT_CREDENTIALS_LOCATION = "AwsCredentials.properties";
+
+    @Override public AWSCredentials getCredentials() {
+      File credentials = new File(Objects.firstNonNull(H2O.OPT_ARGS.aws_credentials, DEFAULT_CREDENTIALS_LOCATION));
+      try {
+        return new PropertiesCredentials(credentials);
+      } catch (IOException e) {
+        throw new AmazonClientException("Unable to load AWS credentials from file " + credentials);
+      }
+    }
+
+    @Override public void refresh() {}
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName();
+    }
+  }
+
   public static final class H2SO3InputStream extends RIStream {
     Key _k;
     long _to;