Add support for AWS IAM Instance Profiles

By using the command line option --aws-instance-profile, WAL-E can
request and use credentials from the instance metadata.  This option
has high precedence and is mutually exclusive with assigning one's own
AWS credentials.

Editorialized by Daniel Farina <[email protected]>

README.rst

@@ -514,6 +514,26 @@ can increase throughput significantly.
 As of version 0.7.x, ``--pool-size`` defaults to 8.
 
 
+Using AWS IAM Instance Profiles
+'''''''''''''''''''''''''''''''
+
+Storing credentials on AWS EC2 instances has usability and security
+drawbacks.  When using WAL-E with AWS S3 and AWS EC2, most uses of
+WAL-E would benefit from use with the `AWS Instance Profile feature`_,
+which automatically generates and rotates credentials on behalf of an
+instance.
+
+To instruct WAL-E to use these credentials for access to S3, pass the
+``--aws-instance-profile`` flag.
+
+.. _AWS Instance Profile feature:
+   http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AESDG-chapter-instancedata.html
+
+Instance profiles may *not* be preferred in more complex scenarios
+when one has multiple AWS IAM policies written for multiple programs
+run on an instance, or an existing key management infrastructure.
+
+
 Development
 -----------
 

tests/test_aws_instance_profiles.py

@@ -0,0 +1,45 @@
+import pytest
+
+import boto
+import boto.provider
+from boto import utils
+
+from wal_e.blobstore.s3 import s3_credentials
+
+META_DATA_CREDENTIALS = {
+    "Code": "Success",
+    "LastUpdated": "2014-01-11T02:13:53Z",
+    "Type": "AWS-HMAC",
+    "AccessKeyId": None,
+    "SecretAccessKey": None,
+    "Token": None,
+    "Expiration": "2014-01-11T08:16:59Z"
+}
+
+
+def boto_flat_metadata():
+    return tuple(int(x) for x in boto.__version__.split('.')) >= (2, 9, 0)
+
+
+@pytest.fixture()
+def metadata(monkeypatch):
+    m = dict(**META_DATA_CREDENTIALS)
+    m['AccessKeyId'] = 'foo'
+    m['SecretAccessKey'] = 'bar'
+    m['Token'] = 'baz'
+    monkeypatch.setattr(boto.provider.Provider,
+                        '_credentials_need_refresh',
+                        lambda self: False)
+    if boto_flat_metadata():
+        m = {'irrelevant': m}
+    else:
+        m = {'iam': {'security-credentials': {'irrelevant': m}}}
+    monkeypatch.setattr(utils, 'get_instance_metadata',
+                        lambda *args, **kwargs: m)
+
+
+def test_profile_provider(metadata):
+    ipp = s3_credentials.InstanceProfileCredentials()
+    assert ipp.get_access_key() == 'foo'
+    assert ipp.get_secret_key() == 'bar'
+    assert ipp.get_security_token() == 'baz'

wal_e/blobstore/s3/__init__.py

@@ -1,11 +1,13 @@
 from wal_e.blobstore.s3.s3_credentials import Credentials
+from wal_e.blobstore.s3.s3_credentials import InstanceProfileCredentials
 from wal_e.blobstore.s3.s3_util import do_lzop_get
 from wal_e.blobstore.s3.s3_util import uri_get_file
 from wal_e.blobstore.s3.s3_util import uri_put_file
 from wal_e.blobstore.s3.s3_util import write_and_return_error
 
 __all__ = [
     'Credentials',
+    'InstanceProfileCredentials',
     'do_lzop_get',
     'uri_put_file',
     'uri_get_file',

wal_e/blobstore/s3/s3_credentials.py

@@ -1,4 +1,33 @@
 from boto import provider
 from functools import partial
+from wal_e.exception import UserException
+
+
+class InstanceProfileProvider(provider.Provider):
+    """Override boto Provider to control use of the AWS metadata store
+
+    In particular, prevent boto from looking in a series of places for
+    keys outside off WAL-E's control (e.g. boto.cfg, environment
+    variables, and so on).  As-is that precedence and detection code
+    is in one big ream, and so a method override and some internal
+    symbols are used to excise most of that cleverness.
+
+    Also take this opportunity to inject a WAL-E-friendly exception to
+    help the user with missing keys.
+
+    """
+
+    def get_credentials(self, access_key=None, secret_key=None,
+                        security_token=None):
+        if self.MetadataServiceSupport[self.name]:
+            self._populate_keys_from_metadata_server()
+
+        if not self._secret_key:
+            raise UserException('Could not retrieve secret key from instance '
+                                'profile.',
+                                hint='Check that your instance has an IAM '
+                                'profile or set --aws-access-key-id')
+
 
 Credentials = partial(provider.Provider, "aws")
+InstanceProfileCredentials = partial(InstanceProfileProvider, 'aws')

wal_e/cmd.py

@@ -170,11 +170,17 @@ def build_parser():
         formatter_class=argparse.RawDescriptionHelpFormatter,
         description=__doc__)
 
-    parser.add_argument('-k', '--aws-access-key-id',
-                        help='public AWS access key. Can also be defined in '
-                        'an environment variable. If both are defined, '
-                        'the one defined in the program arguments takes '
-                        'precedence.')
+    aws_group = parser.add_mutually_exclusive_group()
+    aws_group.add_argument('-k', '--aws-access-key-id',
+                           help='public AWS access key. Can also be defined '
+                           'in an environment variable. If both are defined, '
+                           'the one defined in the programs arguments takes '
+                           'precedence.')
+
+    aws_group.add_argument('--aws-instance-profile', action='store_true',
+                           help='Use the IAM Instance Profile associated '
+                           'with this instance to authenticate with the S3 '
+                           'API.')
 
     parser.add_argument('-a', '--wabs-account-name',
                         help='Account name of Windows Azure Blob Service '
@@ -377,6 +383,13 @@ def s3_explicit_creds(args):
     return s3.Credentials(access_key, secret_key, security_token)
 
 
+def s3_instance_profile(args):
+    from wal_e.blobstore import s3
+
+    assert args.aws_instance_profile
+    return s3.InstanceProfileCredentials()
+
+
 def configure_backup_cxt(args):
     # Try to find some WAL-E prefix to store data in.
     prefix = (args.s3_prefix or args.wabs_prefix
@@ -405,9 +418,14 @@ def configure_backup_cxt(args):
     # backend data stores, yielding value adhering to the
     # 'operator.Backup' protocol.
     if store.is_s3:
-        creds = s3_explicit_creds(args)
-        from wal_e.operator.s3_operator import S3Backup
-        return S3Backup(store, creds, gpg_key_id)
+        if args.aws_instance_profile:
+            creds = s3_instance_profile(args)
+        else:
+            creds = s3_explicit_creds(args)
+
+        from wal_e.operator import s3_operator
+
+        return s3_operator.S3Backup(store, creds, gpg_key_id)
     elif store.is_wabs:
         account_name = args.wabs_account_name or os.getenv('WABS_ACCOUNT_NAME')
         if account_name is None: