Merge branch 'release-1.10.20'

* release-1.10.20:
  Bumping version to 1.10.20
  Update CHANGELOG
  Shutdown task executor before aborting multipart uploads
  Update existing examples for KMS

CHANGELOG.rst

@@ -2,6 +2,16 @@
 CHANGELOG
 =========
 
+1.10.20
+=======
+
+* feature:``iot``: Add commands for managing CA certificates.
+* bugfix:``ec2 wait``: Fix issues with waiting on incorrect error code.
+* bugfix:``s3``: Fix issue where multipart uploads were not being properly
+  aborted after Cntrl-C.
+  (`issue 1905 <https://github.com/aws/aws-cli/pull/1905>`__)
+
+
 1.10.19
 =======
 

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.10.19'
+__version__ = '1.10.20'
 
 #
 # Get our data path to be added to botocore's search path

awscli/customizations/s3/s3handler.py

@@ -101,28 +101,35 @@ def call(self, files):
             self.executor.print_thread.set_total_files(total_files)
             self.executor.print_thread.set_total_parts(total_parts)
             self.executor.initiate_shutdown()
-            self.executor.wait_until_shutdown()
-            self._shutdown()
+            self._finalize_shutdown()
         except Exception as e:
             LOGGER.debug('Exception caught during task execution: %s',
                          str(e), exc_info=True)
             self.result_queue.put(PrintTask(message=str(e), error=True))
             self.executor.initiate_shutdown(
                 priority=self.executor.IMMEDIATE_PRIORITY)
-            self._shutdown()
-            self.executor.wait_until_shutdown()
+            self._finalize_shutdown()
         except KeyboardInterrupt:
             self.result_queue.put(PrintTask(message=("Cleaning up. "
                                                      "Please wait..."),
                                             error=True))
             self.executor.initiate_shutdown(
                 priority=self.executor.IMMEDIATE_PRIORITY)
-            self._shutdown()
-            self.executor.wait_until_shutdown()
+            self._finalize_shutdown()
         return CommandResult(self.executor.num_tasks_failed,
                              self.executor.num_tasks_warned)
 
-    def _shutdown(self):
+    def _finalize_shutdown(self):
+        # Run all remaining tasks needed to completely shutdown the
+        # S3 handler.  This method will block until shutdown is complete.
+        # The order here is important.  We need to wait until all the
+        # tasks have been completed before we can cleanup.  Otherwise
+        # we can have race conditions where we're trying to cleanup
+        # uploads/downloads that are still in progress.
+        self.executor.wait_until_shutdown()
+        self._cleanup()
+
+    def _cleanup(self):
         # And finally we need to make a pass through all the existing
         # multipart uploads and abort any pending multipart uploads.
         self._abort_pending_multipart_uploads()

awscli/examples/kms/create-alias.rst

@@ -1,5 +1,7 @@
-The following command creates an alias for a customer master key::
+The following command creates an alias named ``example-alias`` for the customer master key (CMK) identified by key ID ``1234abcd-12ab-34cd-56ef-1234567890ab``.
 
-    $ aws kms create-alias --alias-name alias/my-alias --target-key-id arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012
+.. code::
 
-Note that all alias names must begin with ``alias/``.
+    aws kms create-alias --alias-name alias/example-alias --target-key-id 1234abcd-12ab-34cd-56ef-1234567890ab
+
+Alias names must begin with ``alias/``. Do not use alias names that begin with ``alias/aws``; these are reserved for use by AWS.

awscli/examples/kms/decrypt.rst

@@ -1,9 +1,39 @@
-The following command decrypts a KMS encrypted, binary encoded version of a 256 bit key named ``encryptedkey-binary`` in the current folder::
+The following command demonstrates the recommended way to decrypt data with the AWS CLI.
 
-  aws kms decrypt --ciphertext-blob fileb://encryptedkey-binary
+.. code::
 
-The fileb:// prefix instructs the AWS CLI not to attempt to decode the binary data in the file.
-
-The output of this command includes the ID of the key used to decrypt the key, and a base64 encoded version of the decrypted key. Use ``base64 -d`` to reverse the base64 encoding and view the original hexadecimal (or otherwise encoded) version of the key. The following command uses an AWS CLI query to isolate the base64 plaintext and pipe it into ``base64``::
+    aws kms decrypt --ciphertext-blob fileb://ExampleEncryptedFile --output text --query Plaintext | base64 --decode > ExamplePlaintextFile
 
-  aws kms decrypt --ciphertext-blob fileb://encryptedkey-binary --query Plaintext --output text | base64 -d
+The command does several things:
+
+#. Uses the ``fileb://`` prefix to specify the ``--ciphertext-blob`` parameter.
+
+    The ``fileb://`` prefix instructs the CLI to read the encrypted data, called the *ciphertext*, from a file and pass the file's contents to the command's ``--ciphertext-blob`` parameter.  If the file is not in the current directory, type the full path to file. For example: ``fileb:///var/tmp/ExampleEncryptedFile`` or ``fileb://C:\Temp\ExampleEncryptedFile``.
+
+    For more information about reading AWS CLI parameter values from a file, see `Loading Parameters from a File <https://docs.aws.amazon.com/cli/latest/userguide/cli-using-param.html#cli-using-param-file>`_ in the *AWS Command Line Interface User Guide* and `Best Practices for Local File Parameters <https://blogs.aws.amazon.com/cli/post/TxLWWN1O25V1HE/Best-Practices-for-Local-File-Parameters>`_ on the AWS Command Line Tool Blog.
+
+    The command assumes the ciphertext in ``ExampleEncryptedFile`` is binary data. The `encrypt examples <encrypt.html#examples>`_ demonstrate how to save a ciphertext this way.
+
+#. Uses the ``--output`` and ``--query`` parameters to control the command's output.
+
+    These parameters extract the decrypted data, called the *plaintext*, from the command's output. For more information about controlling output, see `Controlling Command Output <https://docs.aws.amazon.com/cli/latest/userguide/controlling-output.html>`_ in the *AWS Command Line Interface User Guide*.
+
+#. Uses the ``base64`` utility.
+
+    This utility decodes the extracted plaintext to binary data. The plaintext that is returned by a successful ``decrypt`` command is base64-encoded text. You must decode this text to obtain the original plaintext.
+
+#. Saves the binary plaintext to a file.
+
+    The final part of the command (``> ExamplePlaintextFile``) saves the binary plaintext data to a file.
+
+**Example: Using the AWS CLI to decrypt data from the Windows command prompt**
+
+The preceding example assumes the ``base64`` utility is available, which is commonly the case on Linux and Mac OS X. For the Windows command prompt, use ``certutil`` instead of ``base64``. This requires two commands, as shown in the following examples.
+
+.. code::
+
+    aws kms decrypt --ciphertext-blob fileb://ExampleEncryptedFile --output text --query Plaintext > ExamplePlaintextFile.base64
+
+.. code::
+
+    certutil -decode ExamplePlaintextFile.base64 ExamplePlaintextFile

awscli/examples/kms/encrypt.rst

@@ -1,14 +1,39 @@
-This example shows how to encrypt the string ``"1\!2@3#4$5%6^7&8*9(0)-_=+"``
-and save the binary contents to a file::
+The following command demonstrates the recommended way to encrypt data with the AWS CLI.
 
-  aws kms encrypt --key-id my-key-id --plaintext "1\!2@3#4$5%6^7&8*9(0)-_=+" --query CiphertextBlob --output text | base64 --decode > /tmp/encrypted
+.. code::
 
+    aws kms encrypt --key-id 1234abcd-12ab-34cd-56ef-1234567890ab --plaintext fileb://ExamplePlaintextFile --output text --query CiphertextBlob | base64 --decode > ExampleEncryptedFile
 
-If you want to decrypt the contents of the file above you can use this
-command::
+The command does several things:
 
-  echo "Decrypted is: $(aws kms decrypt --ciphertext-blob fileb:///tmp/encrypted  --output text --query Plaintext | base64 --decode)"
+#. Uses the ``fileb://`` prefix to specify the ``--plaintext`` parameter.
 
-Note the use of the ``fileb://`` prefix in the ``decrypt`` command above.  This
-indicates that the referenced file contains binary contents, and that the file
-contents are not decoded before being used.
+    The ``fileb://`` prefix instructs the CLI to read the data to encrypt, called the *plaintext*, from a file and pass the file's contents to the command's ``--plaintext`` parameter. If the file is not in the current directory, type the full path to file. For example: ``fileb:///var/tmp/ExamplePlaintextFile`` or ``fileb://C:\Temp\ExamplePlaintextFile``.
+
+    For more information about reading AWS CLI parameter values from a file, see `Loading Parameters from a File <https://docs.aws.amazon.com/cli/latest/userguide/cli-using-param.html#cli-using-param-file>`_ in the *AWS Command Line Interface User Guide* and `Best Practices for Local File Parameters <https://blogs.aws.amazon.com/cli/post/TxLWWN1O25V1HE/Best-Practices-for-Local-File-Parameters>`_ on the AWS Command Line Tool Blog.
+
+#. Uses the ``--output`` and ``--query`` parameters to control the command's output.
+
+    These parameters extract the encrypted data, called the *ciphertext*, from the command's output.
+
+    For more information about controlling output, see `Controlling Command Output <https://docs.aws.amazon.com/cli/latest/userguide/controlling-output.html>`_ in the *AWS Command Line Interface User Guide*.
+
+#. Uses the ``base64`` utility to decode the extracted output.
+
+    This utility decodes the extracted ciphertext to binary data. The ciphertext that is returned by a successful ``encrypt`` command is base64-encoded text. You must decode this text before you can use the AWS CLI to decrypt it.
+
+#. Saves the binary ciphertext to a file.
+
+    The final part of the command (``> ExampleEncryptedFile``) saves the binary ciphertext to a file to make decryption easier. For an example command that uses the AWS CLI to decrypt data, see the `decrypt examples <decrypt.html#examples>`_.
+
+**Example: Using the AWS CLI to encrypt data from the Windows command prompt**
+
+The preceding example assumes the ``base64`` utility is available, which is commonly the case on Linux and Mac OS X. For the Windows command prompt, use ``certutil`` instead of ``base64``. This requires two commands, as shown in the following examples.
+
+.. code::
+
+    aws kms encrypt --key-id 1234abcd-12ab-34cd-56ef-1234567890ab --plaintext fileb://ExamplePlaintextFile --output text --query CiphertextBlob > C:\Temp\ExampleEncryptedFile.base64
+
+.. code::
+
+    certutil -decode C:\Temp\ExampleEncryptedFile.base64 C:\Temp\ExampleEncryptedFile

doc/source/conf.py

@@ -52,7 +52,7 @@
 # The short X.Y version.
 version = '1.10.'
 # The full version, including alpha/beta/rc tags.
-release = '1.10.19'
+release = '1.10.20'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

setup.cfg

@@ -4,7 +4,7 @@ universal = 1
 
 [metadata]
 requires-dist =
-        botocore==1.4.10
+        botocore==1.4.11
         colorama>=0.2.5,<=0.3.3
         docutils>=0.10
         rsa>=3.1.2,<=3.3.0

setup.py

@@ -6,7 +6,7 @@
 import awscli
 
 
-requires = ['botocore==1.4.10',
+requires = ['botocore==1.4.11',
             'colorama>=0.2.5,<=0.3.3',
             'docutils>=0.10',
             'rsa>=3.1.2,<=3.3.0',

tests/integration/customizations/s3/test_plugin.py

@@ -28,8 +28,6 @@
 import shutil
 import copy
 
-import botocore.session
-from botocore.exceptions import ClientError
 from awscli.compat import six
 from nose.plugins.attrib import attr
 
@@ -62,6 +60,19 @@ def aws(command, collect_memory=False, env_vars=None, wait_for_finish=True,
                 input_file=input_file)
 
 
+def wait_for_process_exit(process, timeout=60):
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        rc = process.poll()
+        if rc is not None:
+            break
+        time.sleep(1)
+    else:
+        process.kill()
+        raise AssertionError("CLI did not exist within %s seconds of "
+                             "receiving a Ctrl+C" % timeout)
+
+
 def _running_on_rhel():
     return (
         hasattr(platform, 'linux_distribution') and
@@ -351,26 +362,39 @@ def test_download_ctrl_c_does_not_hang(self):
         process = aws('s3 cp s3://%s/foo.txt %s' %
                       (bucket_name, local_foo_txt), wait_for_finish=False)
         # Give it some time to start up and enter it's main task loop.
-        time.sleep(1)
+        time.sleep(2)
         # The process has 60 seconds to finish after being sent a Ctrl+C,
         # otherwise the test fails.
         process.send_signal(signal.SIGINT)
-        deadline = time.time() + 60
-        while time.time() < deadline:
-            rc = process.poll()
-            if rc is not None:
-                break
-            time.sleep(1)
-        else:
-            process.kill()
-            self.fail("CLI did not exist within 30 seconds of "
-                      "receiving a Ctrl+C")
+        wait_for_process_exit(process, timeout=60)
         # A Ctrl+C should have a non-zero RC.
         # We either caught the process in
         # its main polling loop (rc=1), or it was successfully terminated by
         # the SIGINT (rc=-2).
         self.assertIn(process.returncode, [1, -2])
 
+    @attr('slow')
+    @skip_if_windows('SIGINT not supported on Windows.')
+    def test_cleans_up_aborted_uploads(self):
+        bucket_name = self.create_bucket()
+        foo_txt = self.files.create_file('foo.txt', '')
+        with open(foo_txt, 'wb') as f:
+            for i in range(20):
+                f.write(b'a' * 1024 * 1024)
+
+        process = aws('s3 cp %s s3://%s/' % (foo_txt, bucket_name),
+                      wait_for_finish=False)
+        time.sleep(3)
+        # The process has 60 seconds to finish after being sent a Ctrl+C,
+        # otherwise the test fails.
+        process.send_signal(signal.SIGINT)
+        wait_for_process_exit(process, timeout=60)
+        uploads_after = self.client.list_multipart_uploads(
+            Bucket=bucket_name).get('Uploads', [])
+        self.assertEqual(uploads_after, [],
+                         "Not all multipart uploads were properly "
+                         "aborted after receiving Ctrl-C: %s" % uploads_after)
+
     def test_cp_to_nonexistent_bucket(self):
         foo_txt = self.files.create_file('foo.txt', 'this is foo.txt')
         p = aws('s3 cp %s s3://noexist-bucket-foo-bar123/foo.txt' % (foo_txt,))