diff --git a/README.rst b/README.rst index 0682c7db..9621e535 100644 --- a/README.rst +++ b/README.rst @@ -514,6 +514,26 @@ can increase throughput significantly. As of version 0.7.x, ``--pool-size`` defaults to 8. +Using AWS IAM Instance Profiles +''''''''''''''''''''''''''''''' + +Storing credentials on AWS EC2 instances has usability and security +drawbacks. When using WAL-E with AWS S3 and AWS EC2, most uses of +WAL-E would benefit from use with the `AWS Instance Profile feature`_, +which automatically generates and rotates credentials on behalf of an +instance. + +To instruct WAL-E to use these credentials for access to S3, pass the +``--aws-instance-profile`` flag. + +.. _AWS Instance Profile feature: + http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AESDG-chapter-instancedata.html + +Instance profiles may *not* be preferred in more complex scenarios +when one has multiple AWS IAM policies written for multiple programs +run on an instance, or an existing key management infrastructure. + + Development ----------- diff --git a/tests/test_aws_instance_profiles.py b/tests/test_aws_instance_profiles.py new file mode 100644 index 00000000..81ea569c --- /dev/null +++ b/tests/test_aws_instance_profiles.py @@ -0,0 +1,45 @@ +import pytest + +import boto +import boto.provider +from boto import utils + +from wal_e.blobstore.s3 import s3_credentials + +META_DATA_CREDENTIALS = { + "Code": "Success", + "LastUpdated": "2014-01-11T02:13:53Z", + "Type": "AWS-HMAC", + "AccessKeyId": None, + "SecretAccessKey": None, + "Token": None, + "Expiration": "2014-01-11T08:16:59Z" +} + + +def boto_flat_metadata(): + return tuple(int(x) for x in boto.__version__.split('.')) >= (2, 9, 0) + + +@pytest.fixture() +def metadata(monkeypatch): + m = dict(**META_DATA_CREDENTIALS) + m['AccessKeyId'] = 'foo' + m['SecretAccessKey'] = 'bar' + m['Token'] = 'baz' + monkeypatch.setattr(boto.provider.Provider, + '_credentials_need_refresh', + lambda self: False) + if boto_flat_metadata(): + m = {'irrelevant': m} + else: + m = {'iam': {'security-credentials': {'irrelevant': m}}} + monkeypatch.setattr(utils, 'get_instance_metadata', + lambda *args, **kwargs: m) + + +def test_profile_provider(metadata): + ipp = s3_credentials.InstanceProfileCredentials() + assert ipp.get_access_key() == 'foo' + assert ipp.get_secret_key() == 'bar' + assert ipp.get_security_token() == 'baz' diff --git a/wal_e/blobstore/s3/__init__.py b/wal_e/blobstore/s3/__init__.py index b3129fd8..96a8011e 100644 --- a/wal_e/blobstore/s3/__init__.py +++ b/wal_e/blobstore/s3/__init__.py @@ -1,4 +1,5 @@ from wal_e.blobstore.s3.s3_credentials import Credentials +from wal_e.blobstore.s3.s3_credentials import InstanceProfileCredentials from wal_e.blobstore.s3.s3_util import do_lzop_get from wal_e.blobstore.s3.s3_util import uri_get_file from wal_e.blobstore.s3.s3_util import uri_put_file @@ -6,6 +7,7 @@ __all__ = [ 'Credentials', + 'InstanceProfileCredentials', 'do_lzop_get', 'uri_put_file', 'uri_get_file', diff --git a/wal_e/blobstore/s3/s3_credentials.py b/wal_e/blobstore/s3/s3_credentials.py index fe8e36b6..500c66a6 100644 --- a/wal_e/blobstore/s3/s3_credentials.py +++ b/wal_e/blobstore/s3/s3_credentials.py @@ -1,4 +1,33 @@ from boto import provider from functools import partial +from wal_e.exception import UserException + + +class InstanceProfileProvider(provider.Provider): + """Override boto Provider to control use of the AWS metadata store + + In particular, prevent boto from looking in a series of places for + keys outside off WAL-E's control (e.g. boto.cfg, environment + variables, and so on). As-is that precedence and detection code + is in one big ream, and so a method override and some internal + symbols are used to excise most of that cleverness. + + Also take this opportunity to inject a WAL-E-friendly exception to + help the user with missing keys. + + """ + + def get_credentials(self, access_key=None, secret_key=None, + security_token=None): + if self.MetadataServiceSupport[self.name]: + self._populate_keys_from_metadata_server() + + if not self._secret_key: + raise UserException('Could not retrieve secret key from instance ' + 'profile.', + hint='Check that your instance has an IAM ' + 'profile or set --aws-access-key-id') + Credentials = partial(provider.Provider, "aws") +InstanceProfileCredentials = partial(InstanceProfileProvider, 'aws') diff --git a/wal_e/cmd.py b/wal_e/cmd.py index e6e0733a..d758edee 100755 --- a/wal_e/cmd.py +++ b/wal_e/cmd.py @@ -170,11 +170,17 @@ def build_parser(): formatter_class=argparse.RawDescriptionHelpFormatter, description=__doc__) - parser.add_argument('-k', '--aws-access-key-id', - help='public AWS access key. Can also be defined in ' - 'an environment variable. If both are defined, ' - 'the one defined in the program arguments takes ' - 'precedence.') + aws_group = parser.add_mutually_exclusive_group() + aws_group.add_argument('-k', '--aws-access-key-id', + help='public AWS access key. Can also be defined ' + 'in an environment variable. If both are defined, ' + 'the one defined in the programs arguments takes ' + 'precedence.') + + aws_group.add_argument('--aws-instance-profile', action='store_true', + help='Use the IAM Instance Profile associated ' + 'with this instance to authenticate with the S3 ' + 'API.') parser.add_argument('-a', '--wabs-account-name', help='Account name of Windows Azure Blob Service ' @@ -377,6 +383,13 @@ def s3_explicit_creds(args): return s3.Credentials(access_key, secret_key, security_token) +def s3_instance_profile(args): + from wal_e.blobstore import s3 + + assert args.aws_instance_profile + return s3.InstanceProfileCredentials() + + def configure_backup_cxt(args): # Try to find some WAL-E prefix to store data in. prefix = (args.s3_prefix or args.wabs_prefix @@ -405,9 +418,14 @@ def configure_backup_cxt(args): # backend data stores, yielding value adhering to the # 'operator.Backup' protocol. if store.is_s3: - creds = s3_explicit_creds(args) - from wal_e.operator.s3_operator import S3Backup - return S3Backup(store, creds, gpg_key_id) + if args.aws_instance_profile: + creds = s3_instance_profile(args) + else: + creds = s3_explicit_creds(args) + + from wal_e.operator import s3_operator + + return s3_operator.S3Backup(store, creds, gpg_key_id) elif store.is_wabs: account_name = args.wabs_account_name or os.getenv('WABS_ACCOUNT_NAME') if account_name is None: