Merge pull request boto#2684 from kyleknap/sigv4-s3

Fix s3 create multipart upload for sigv4

boto/auth.py

@@ -345,7 +345,7 @@ def canonical_query_string(self, http_request):
         for param in sorted(http_request.params):
             value = boto.utils.get_utf8_value(http_request.params[param])
             l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'),
-                                urllib.parse.quote(value.decode('utf-8'), safe='-_.~')))
+                                urllib.parse.quote(value, safe='-_.~')))
         return '&'.join(l)
 
     def canonical_headers(self, headers_to_sign):
@@ -494,10 +494,25 @@ def add_auth(self, req, **kwargs):
         if self._provider.security_token:
             req.headers['X-Amz-Security-Token'] = self._provider.security_token
         qs = self.query_string(req)
-        if qs and req.method == 'POST':
+
+        qs_to_post = qs
+
+        # We do not want to include any params that were mangled into
+        # the params if performing s3-sigv4 since it does not
+        # belong in the body of a post for some requests.  Mangled
+        # refers to items in the query string URL being added to the
+        # http response params. However, these params get added to
+        # the body of the request, but the query string URL does not
+        # belong in the body of the request. ``unmangled_resp`` is the
+        # response that happened prior to the mangling.  This ``unmangled_req``
+        # kwarg will only appear for s3-sigv4.
+        if 'unmangled_req' in kwargs:
+            qs_to_post = self.query_string(kwargs['unmangled_req'])
+
+        if qs_to_post and req.method == 'POST':
             # Stash request parameters into post body
             # before we generate the signature.
-            req.body = qs
+            req.body = qs_to_post
             req.headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'
             req.headers['Content-Length'] = str(len(req.body))
         else:
@@ -549,6 +564,17 @@ def canonical_uri(self, http_request):
         encoded = urllib.parse.quote(unquoted)
         return encoded
 
+    def canonical_query_string(self, http_request):
+        # Note that we just do not return an empty string for
+        # POST request. Query strings in url are included in canonical
+        # query string.
+        l = []
+        for param in sorted(http_request.params):
+            value = boto.utils.get_utf8_value(http_request.params[param])
+            l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'),
+                                urllib.parse.quote(value, safe='-_.~')))
+        return '&'.join(l)
+
     def host_header(self, host, http_request):
         port = http_request.port
         secure = http_request.protocol == 'https'
@@ -641,6 +667,13 @@ def mangle_path_and_params(self, req):
 
         if modified_req.params is None:
             modified_req.params = {}
+        else:
+            # To keep the original request object untouched. We must make
+            # a copy of the params dictionary. Because the copy of the
+            # original request directly refers to the params dictionary
+            # of the original request.
+            copy_params = req.params.copy()
+            modified_req.params = copy_params
 
         raw_qs = parsed_path.query
         existing_qs = urllib.parse.parse_qs(
@@ -670,9 +703,10 @@ def add_auth(self, req, **kwargs):
                 req.headers['x-amz-content-sha256'] = req.headers.pop('_sha256')
             else:
                 req.headers['x-amz-content-sha256'] = self.payload(req)
-
-        req = self.mangle_path_and_params(req)
-        return super(S3HmacAuthV4Handler, self).add_auth(req, **kwargs)
+        updated_req = self.mangle_path_and_params(req)
+        return super(S3HmacAuthV4Handler, self).add_auth(updated_req,
+                                                         unmangled_req=req,
+                                                         **kwargs)
 
     def presign(self, req, expires, iso_date=None):
         """

tests/integration/s3/test_multipart.py

@@ -32,10 +32,14 @@
 # bigger than 5M. Hence we just use 1 part so we can keep
 # things small and still test logic.
 
+import os
 import unittest
 import time
 from boto.compat import StringIO
 
+import mock
+
+import boto
 from boto.s3.connection import S3Connection
 
 
@@ -163,3 +167,31 @@ def test_etag_of_parts(self):
             pn += 1
         # Can't complete 2 small parts so just clean up.
         mpu.cancel_upload()
+
+
+class S3MultiPartUploadSigV4Test(unittest.TestCase):
+    s3 = True
+
+    def setUp(self):
+        self.env_patch = mock.patch('os.environ', {'S3_USE_SIGV4': True})
+        self.env_patch.start()
+        self.conn = boto.s3.connect_to_region('us-west-2')
+        self.bucket_name = 'multipart-%d' % int(time.time())
+        self.bucket = self.conn.create_bucket(self.bucket_name,
+                                              location='us-west-2')
+
+    def tearDown(self):
+        for key in self.bucket:
+            key.delete()
+        self.bucket.delete()
+        self.env_patch.stop()
+
+    def test_initiate_multipart(self):
+        key_name = "multipart"
+        multipart_upload = self.bucket.initiate_multipart_upload(key_name)
+        multipart_uploads = self.bucket.get_all_multipart_uploads()
+        for upload in multipart_uploads:
+            # Check that the multipart upload was created.
+            self.assertEqual(upload.key_name, multipart_upload.key_name)
+            self.assertEqual(upload.id, multipart_upload.id)
+        multipart_upload.cancel_upload()