Merge branch 'release-1.14.10'

* release-1.14.10:
  Bumping version to 1.14.10
  Update changelog based on model updates
  Delegate jsonfilecache to botocore
  Add changelog entry for stdin bugfix
  Defer resolution of binary_stdin

.changes/1.14.10.json

@@ -0,0 +1,17 @@
+[
+  {
+    "category": "``apigateway``", 
+    "description": "Update apigateway command to latest version", 
+    "type": "api-change"
+  }, 
+  {
+    "category": "``ses``", 
+    "description": "Update ses command to latest version", 
+    "type": "api-change"
+  }, 
+  {
+    "category": "s3", 
+    "description": "Fixes a bug where calling the CLI from a context that has no stdin resulted in every command failing instead of only commands that required stdin.", 
+    "type": "bugfix"
+  }
+]

CHANGELOG.rst

@@ -2,6 +2,14 @@
 CHANGELOG
 =========
 
+1.14.10
+=======
+
+* api-change:``apigateway``: Update apigateway command to latest version
+* api-change:``ses``: Update ses command to latest version
+* bugfix:s3: Fixes a bug where calling the CLI from a context that has no stdin resulted in every command failing instead of only commands that required stdin.
+
+
 1.14.9
 ======
 

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.14.9'
+__version__ = '1.14.10'
 
 #
 # Get our data path to be added to botocore's search path

awscli/compat.py

@@ -56,6 +56,14 @@
     default_pager = 'less -R'
 
 
+class StdinMissingError(Exception):
+    def __init__(self):
+        message = (
+            'stdin is required for this operation, but is not available.'
+        )
+        super(StdinMissingError, self).__init__(message)
+
+
 class NonTranslatedStdout(object):
     """ This context manager sets the line-end translation mode for stdout.
 
@@ -76,13 +84,15 @@ def __exit__(self, type, value, traceback):
             import msvcrt
             msvcrt.setmode(sys.stdout.fileno(), self.previous_mode)
 
+
 def ensure_text_type(s):
     if isinstance(s, six.text_type):
         return s
     if isinstance(s, six.binary_type):
         return s.decode('utf-8')
     raise ValueError("Expected str, unicode or bytes, received %s." % type(s))
 
+
 if six.PY3:
     import locale
     import urllib.parse as urlparse
@@ -91,7 +101,10 @@ def ensure_text_type(s):
 
     raw_input = input
 
-    binary_stdin = sys.stdin.buffer
+    def get_binary_stdin():
+        if sys.stdin is None:
+            raise StdinMissingError()
+        return sys.stdin.buffer
 
     def get_binary_stdout():
         return sys.stdout.buffer
@@ -138,7 +151,10 @@ def bytes_print(statement, stdout=None):
 
     raw_input = raw_input
 
-    binary_stdin = sys.stdin
+    def get_binary_stdin():
+        if sys.stdin is None:
+            raise StdinMissingError()
+        return sys.stdin
 
     def get_binary_stdout():
         return sys.stdout

awscli/customizations/assumerole.py

@@ -1,10 +1,11 @@
 import os
-import json
 import logging
 
 from botocore.exceptions import ProfileNotFound
+from botocore.credentials import JSONFileCache
 
 LOG = logging.getLogger(__name__)
+CACHE_DIR = os.path.expanduser(os.path.join('~', '.aws', 'cli', 'cache'))
 
 
 def register_assume_role_provider(event_handlers):
@@ -37,52 +38,4 @@ def inject_assume_role_provider_cache(session, **kwargs):
                   "JSONFileCache for assume-role.")
         return
     provider = cred_chain.get_provider('assume-role')
-    provider.cache = JSONFileCache()
-
-
-class JSONFileCache(object):
-    """JSON file cache.
-
-    This provides a dict like interface that stores JSON serializable
-    objects.
-
-    The objects are serialized to JSON and stored in a file.  These
-    values can be retrieved at a later time.
-
-    """
-
-    CACHE_DIR = os.path.expanduser(os.path.join('~', '.aws', 'cli', 'cache'))
-
-    def __init__(self, working_dir=CACHE_DIR):
-        self._working_dir = working_dir
-
-    def __contains__(self, cache_key):
-        actual_key = self._convert_cache_key(cache_key)
-        return os.path.isfile(actual_key)
-
-    def __getitem__(self, cache_key):
-        """Retrieve value from a cache key."""
-        actual_key = self._convert_cache_key(cache_key)
-        try:
-            with open(actual_key) as f:
-                return json.load(f)
-        except (OSError, ValueError, IOError):
-            raise KeyError(cache_key)
-
-    def __setitem__(self, cache_key, value):
-        full_key = self._convert_cache_key(cache_key)
-        try:
-            file_content = json.dumps(value)
-        except (TypeError, ValueError):
-            raise ValueError("Value cannot be cached, must be "
-                             "JSON serializable: %s" % value)
-        if not os.path.isdir(self._working_dir):
-            os.makedirs(self._working_dir)
-        with os.fdopen(os.open(full_key,
-                               os.O_WRONLY | os.O_CREAT, 0o600), 'w') as f:
-            f.truncate()
-            f.write(file_content)
-
-    def _convert_cache_key(self, cache_key):
-        full_path = os.path.join(self._working_dir, cache_key + '.json')
-        return full_path
+    provider.cache = JSONFileCache(CACHE_DIR)

awscli/customizations/s3/s3handler.py

@@ -46,7 +46,7 @@
 from awscli.customizations.s3.utils import DeleteSourceFileSubscriber
 from awscli.customizations.s3.utils import DeleteSourceObjectSubscriber
 from awscli.customizations.s3.utils import DeleteCopySourceObjectSubscriber
-from awscli.compat import binary_stdin
+from awscli.compat import get_binary_stdin
 
 
 LOGGER = logging.getLogger(__name__)
@@ -471,6 +471,7 @@ def _add_additional_subscribers(self, subscribers, fileinfo):
             subscribers.append(ProvideSizeSubscriber(int(expected_size)))
 
     def _get_filein(self, fileinfo):
+        binary_stdin = get_binary_stdin()
         return NonSeekableStream(binary_stdin)
 
     def _format_local_path(self, path):

doc/source/conf.py

@@ -50,9 +50,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.14'
+version = '1.14.'
 # The full version, including alpha/beta/rc tags.
-release = '1.14.9'
+release = '1.14.10'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

setup.cfg

@@ -4,7 +4,7 @@ universal = 1
 
 [metadata]
 requires-dist =
-        botocore==1.8.13
+        botocore==1.8.14
         colorama>=0.2.5,<=0.3.7
         docutils>=0.10
         rsa>=3.1.2,<=3.5.0

setup.py

@@ -23,7 +23,7 @@ def find_version(*file_paths):
     raise RuntimeError("Unable to find version string.")
 
 
-requires = ['botocore==1.8.13',
+requires = ['botocore==1.8.14',
             'colorama>=0.2.5,<=0.3.7',
             'docutils>=0.10',
             'rsa>=3.1.2,<=3.5.0',

tests/functional/s3/test_cp_command.py

@@ -18,6 +18,12 @@
 from awscli.compat import six
 
 
+class BufferedBytesIO(six.BytesIO):
+    @property
+    def buffer(self):
+        return self
+
+
 class TestCPCommand(BaseAWSCommandParamsTest):
 
     prefix = 's3 cp '
@@ -484,9 +490,8 @@ def test_streaming_upload(self):
             'ETag': '"c8afdb36c52cf4727836669019e69222"'
         }]
 
-        binary_stdin = six.BytesIO(b'foo\n')
-        location = "awscli.customizations.s3.s3handler.binary_stdin"
-        with mock.patch(location, binary_stdin):
+        binary_stdin = BufferedBytesIO(b'foo\n')
+        with mock.patch('sys.stdin', binary_stdin):
             self.run_cmd(command)
 
         self.assertEqual(len(self.operations_called), 1)
@@ -506,9 +511,8 @@ def test_streaming_upload_with_expected_size(self):
             'ETag': '"c8afdb36c52cf4727836669019e69222"'
         }]
 
-        binary_stdin = six.BytesIO(b'foo\n')
-        location = "awscli.customizations.s3.s3handler.binary_stdin"
-        with mock.patch(location, binary_stdin):
+        binary_stdin = BufferedBytesIO(b'foo\n')
+        with mock.patch('sys.stdin', binary_stdin):
             self.run_cmd(command)
 
         self.assertEqual(len(self.operations_called), 1)
@@ -533,9 +537,8 @@ def test_streaming_upload_error(self):
         }]
         self.http_response.status_code = 404
 
-        binary_stdin = six.BytesIO(b'foo\n')
-        location = "awscli.customizations.s3.s3handler.binary_stdin"
-        with mock.patch(location, binary_stdin):
+        binary_stdin = BufferedBytesIO(b'foo\n')
+        with mock.patch('sys.stdin', binary_stdin):
             _, stderr, _ = self.run_cmd(command, expected_rc=1)
 
         error_message = (
@@ -544,6 +547,20 @@ def test_streaming_upload_error(self):
         )
         self.assertIn(error_message, stderr)
 
+    def test_streaming_upload_when_stdin_unavailable(self):
+        command = "s3 cp - s3://bucket/streaming.txt"
+        self.parsed_responses = [{
+            'ETag': '"c8afdb36c52cf4727836669019e69222"'
+        }]
+
+        with mock.patch('sys.stdin', None):
+            _, stderr, _ = self.run_cmd(command, expected_rc=1)
+
+        expected_message = (
+            'stdin is required for this operation, but is not available'
+        )
+        self.assertIn(expected_message, stderr)
+
     def test_streaming_download(self):
         command = "s3 cp s3://bucket/streaming.txt -"
         self.parsed_responses = [

tests/unit/customizations/test_assumerole.py

@@ -10,19 +10,12 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-import shutil
-import tempfile
-import os
-import platform
-from datetime import datetime, timedelta
-
 import mock
+
 from botocore.hooks import HierarchicalEmitter
-from botocore.exceptions import PartialCredentialsError
 from botocore.exceptions import ProfileNotFound
-from dateutil.tz import tzlocal
 
-from awscli.testutils import unittest, skip_if_windows
+from awscli.testutils import unittest
 from awscli.customizations import assumerole
 
 
@@ -58,64 +51,3 @@ def test_no_registration_if_profile_does_not_exist(self):
 
         credential_provider = session.get_component.return_value
         self.assertFalse(credential_provider.get_provider.called)
-
-
-class TestJSONCache(unittest.TestCase):
-    def setUp(self):
-        self.tempdir = tempfile.mkdtemp()
-        self.cache = assumerole.JSONFileCache(self.tempdir)
-
-    def tearDown(self):
-        shutil.rmtree(self.tempdir)
-
-    def test_supports_contains_check(self):
-        # By default the cache is empty because we're
-        # using a new temp dir everytime.
-        self.assertTrue('mykey' not in self.cache)
-
-    def test_add_key_and_contains_check(self):
-        self.cache['mykey'] = {'foo': 'bar'}
-        self.assertTrue('mykey' in self.cache)
-
-    def test_added_key_can_be_retrieved(self):
-        self.cache['mykey'] = {'foo': 'bar'}
-        self.assertEqual(self.cache['mykey'], {'foo': 'bar'})
-
-    def test_only_accepts_json_serializable_data(self):
-        with self.assertRaises(ValueError):
-            # set()'s cannot be serialized to a JSOn string.
-            self.cache['mykey'] = set()
-
-    def test_can_override_existing_values(self):
-        self.cache['mykey'] = {'foo': 'bar'}
-        self.cache['mykey'] = {'baz': 'newvalue'}
-        self.assertEqual(self.cache['mykey'], {'baz': 'newvalue'})
-
-    def test_can_add_multiple_keys(self):
-        self.cache['mykey'] = {'foo': 'bar'}
-        self.cache['mykey2'] = {'baz': 'qux'}
-        self.assertEqual(self.cache['mykey'], {'foo': 'bar'})
-        self.assertEqual(self.cache['mykey2'], {'baz': 'qux'})
-
-    def test_working_dir_does_not_exist(self):
-        working_dir = os.path.join(self.tempdir, 'foo')
-        cache = assumerole.JSONFileCache(working_dir)
-        cache['foo'] = {'bar': 'baz'}
-        self.assertEqual(cache['foo'], {'bar': 'baz'})
-
-    def test_key_error_raised_when_cache_key_does_not_exist(self):
-        with self.assertRaises(KeyError):
-            self.cache['foo']
-
-    def test_file_is_truncated_before_writing(self):
-        self.cache['mykey'] = {
-            'really long key in the cache': 'really long value in cache'}
-        # Now overwrite it with a smaller value.
-        self.cache['mykey'] = {'a': 'b'}
-        self.assertEqual(self.cache['mykey'], {'a': 'b'})
-
-    @skip_if_windows('File permissions tests not supported on Windows.')
-    def test_permissions_for_file_restricted(self):
-        self.cache['mykey'] = {'foo': 'bar'}
-        filename = os.path.join(self.tempdir, 'mykey.json')
-        self.assertEqual(os.stat(filename).st_mode & 0xFFF, 0o600)