diff --git a/api-datatypes.go b/api-datatypes.go index e81a3bd41c..58e3c5e0e7 100644 --- a/api-datatypes.go +++ b/api-datatypes.go @@ -82,6 +82,9 @@ type ObjectInfo struct { // x-amz-meta-* headers stripped "x-amz-meta-" prefix containing the first value. UserMetadata StringMap `json:"userMetadata"` + // x-amz-tagging values in their k/v values. + UserTags map[string]string `json:"userTags"` + // Owner name. Owner struct { DisplayName string `json:"name"` diff --git a/api-get-object.go b/api-get-object.go index 0c1653cdc7..14370b0cdb 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/s3utils/utils.go b/pkg/s3utils/utils.go index 982dddf4bb..d01878dc14 100644 --- a/pkg/s3utils/utils.go +++ b/pkg/s3utils/utils.go @@ -1,6 +1,6 @@ /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. + * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -229,29 +229,39 @@ func QueryEncode(v url.Values) string { return buf.String() } +// TagDecode - decodes canonical tag into map of key and value. +func TagDecode(ctag string) map[string]string { + if ctag == "" { + return map[string]string{} + } + tags := strings.Split(ctag, "&") + tagMap := make(map[string]string, len(tags)) + var err error + for _, tag := range tags { + kvs := strings.SplitN(tag, "=", 2) + if len(kvs) == 0 { + return map[string]string{} + } + if len(kvs) == 1 { + return map[string]string{} + } + tagMap[kvs[0]], err = url.PathUnescape(kvs[1]) + if err != nil { + continue + } + } + return tagMap +} + // TagEncode - encodes tag values in their URL encoded form. In // addition to the percent encoding performed by urlEncodePath() used // here, it also percent encodes '/' (forward slash) func TagEncode(tags map[string]string) string { - if tags == nil { - return "" + values := url.Values{} + for k, v := range tags { + values[k] = []string{v} } - var buf bytes.Buffer - keys := make([]string, 0, len(tags)) - for k := range tags { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - v := tags[k] - prefix := percentEncodeSlash(EncodePath(k)) + "=" - if buf.Len() > 0 { - buf.WriteByte('&') - } - buf.WriteString(prefix) - buf.WriteString(percentEncodeSlash(EncodePath(v))) - } - return buf.String() + return QueryEncode(values) } // if object matches reserved string, no need to encode them diff --git a/pkg/s3utils/utils_test.go b/pkg/s3utils/utils_test.go index e2fd3089d2..125456031c 100644 --- a/pkg/s3utils/utils_test.go +++ b/pkg/s3utils/utils_test.go @@ -20,6 +20,7 @@ package s3utils import ( "errors" "net/url" + "reflect" "testing" ) @@ -312,6 +313,81 @@ func TestQueryEncode(t *testing.T) { } } +// Tests tag decode to map +func TestTagDecode(t *testing.T) { + testCases := []struct { + // canonical input + canonicalInput string + + // Expected result. + resultMap map[string]string + }{ + {"k=thisisthe%25url", map[string]string{"k": "thisisthe%url"}}, + {"k=%E6%9C%AC%E8%AA%9E", map[string]string{"k": "本語"}}, + {"k=%E6%9C%AC%E8%AA%9E.1", map[string]string{"k": "本語.1"}}, + {"k=%3E123", map[string]string{"k": ">123"}}, + {"k=myurl%23link", map[string]string{"k": "myurl#link"}}, + {"k=space%20in%20url", map[string]string{"k": "space in url"}}, + {"k=url%2Bpath", map[string]string{"k": "url+path"}}, + {"k=url%2Fpath", map[string]string{"k": "url/path"}}, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run("", func(t *testing.T) { + gotResult := TagDecode(testCase.canonicalInput) + if !reflect.DeepEqual(testCase.resultMap, gotResult) { + t.Errorf("Expected %s, got %s", testCase.resultMap, gotResult) + } + }) + } +} + +// Tests tag encode function for user tags. +func TestTagEncode(t *testing.T) { + testCases := []struct { + // Input. + inputMap map[string]string + // Expected result. + result string + }{ + {map[string]string{ + "k": "thisisthe%url", + }, "k=thisisthe%25url"}, + {map[string]string{ + "k": "本語", + }, "k=%E6%9C%AC%E8%AA%9E"}, + {map[string]string{ + "k": "本語.1", + }, "k=%E6%9C%AC%E8%AA%9E.1"}, + {map[string]string{ + "k": ">123", + }, "k=%3E123"}, + {map[string]string{ + "k": "myurl#link", + }, "k=myurl%23link"}, + {map[string]string{ + "k": "space in url", + }, "k=space%20in%20url"}, + {map[string]string{ + "k": "url+path", + }, "k=url%2Bpath"}, + {map[string]string{ + "k": "url/path", + }, "k=url%2Fpath"}, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run("", func(t *testing.T) { + gotResult := TagEncode(testCase.inputMap) + if testCase.result != gotResult { + t.Errorf("Expected %s, got %s", testCase.result, gotResult) + } + }) + } +} + // Tests validate the URL path encoder. func TestEncodePath(t *testing.T) { testCases := []struct { @@ -327,6 +403,7 @@ func TestEncodePath(t *testing.T) { {"myurl#link", "myurl%23link"}, {"space in url", "space%20in%20url"}, {"url+path", "url%2Bpath"}, + {"url/path", "url/path"}, } for i, testCase := range testCases { diff --git a/transport.go b/transport.go index 00ee4dad67..fb624854ba 100644 --- a/transport.go +++ b/transport.go @@ -38,7 +38,8 @@ var DefaultTransport = func(secure bool) (http.RoundTripper, error) { }).DialContext, MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024, - IdleConnTimeout: 90 * time.Second, + ResponseHeaderTimeout: 60 * time.Second, + IdleConnTimeout: 60 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, // Set this value so that the underlying transport round-tripper diff --git a/utils.go b/utils.go index a6ffedcbae..685e98c70e 100644 --- a/utils.go +++ b/utils.go @@ -174,6 +174,7 @@ func extractObjMetadata(header http.Header) http.Header { "X-Amz-Object-Lock-Retain-Until-Date", "X-Amz-Object-Lock-Legal-Hold", "X-Amz-Website-Redirect-Location", + "X-Amz-Server-Side-Encryption", "X-Amz-Meta-", // Add new headers to be preserved. // if you add new headers here, please extend @@ -256,6 +257,7 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn userMetadata[strings.TrimPrefix(k, "X-Amz-Meta-")] = v[0] } } + userTags := s3utils.TagDecode(h.Get(amzTaggingHeader)) // Save object metadata info. return ObjectInfo{ @@ -270,6 +272,7 @@ func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectIn // which are not part of object metadata. Metadata: metadata, UserMetadata: userMetadata, + UserTags: userTags, }, nil }