Fix storage class related issues (minio#5338)

- Update startup banner to print storage class in capitals. This makes it easier to identify different storage classes available. - Update response metadata to not send STANDARD storage class. This is in accordance with AWS S3 behaviour. - Update minio-go library to bring in storage class related changes. This is needed to make transparent translation of storage class headers for Minio S3 Gateway.
bhavik1st · Jan 4, 2018 · 1e5fb4b · 1e5fb4b
1 parent 6f7c6fc
commit 1e5fb4b
Show file tree

Hide file tree

Showing 18 changed files with 7,334 additions and 28 deletions.
diff --git a/cmd/fs-v1-metadata.go b/cmd/fs-v1-metadata.go
@@ -122,7 +122,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of
 	// response headers. e.g, X-Minio-* or X-Amz-*.
-	objInfo.UserDefined = cleanMetaETag(m.Meta)
+	objInfo.UserDefined = cleanMetadata(m.Meta)
 
 	// Success..
 	return objInfo

diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go
@@ -187,14 +187,26 @@ func getCompleteMultipartMD5(parts []CompletePart) (string, error) {
 	return s3MD5, nil
 }
 
-// Clean meta etag keys 'md5Sum', 'etag'.
-func cleanMetaETag(metadata map[string]string) map[string]string {
-	return cleanMetadata(metadata, "md5Sum", "etag")
+// Clean unwanted fields from metadata
+func cleanMetadata(metadata map[string]string) map[string]string {
+	// Remove STANDARD StorageClass
+	metadata = removeStandardStorageClass(metadata)
+	// Clean meta etag keys 'md5Sum', 'etag'.
+	return cleanMetadataKeys(metadata, "md5Sum", "etag")
 }
 
-// Clean metadata takes keys to be filtered
-// and returns a new map with the keys filtered.
-func cleanMetadata(metadata map[string]string, keyNames ...string) map[string]string {
+// Filter X-Amz-Storage-Class field only if it is set to STANDARD.
+// This is done since AWS S3 doesn't return STANDARD Storage class as response header.
+func removeStandardStorageClass(metadata map[string]string) map[string]string {
+	if metadata[amzStorageClass] == standardStorageClass {
+		delete(metadata, amzStorageClass)
+	}
+	return metadata
+}
+
+// cleanMetadataKeys takes keyNames to be filtered
+// and returns a new map with all the entries with keyNames removed.
+func cleanMetadataKeys(metadata map[string]string, keyNames ...string) map[string]string {
 	var newMeta = make(map[string]string)
 	for k, v := range metadata {
 		if contains(keyNames, k) {

diff --git a/cmd/object-api-utils_test.go b/cmd/object-api-utils_test.go
@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"reflect"
 	"testing"
 )
 
@@ -197,3 +198,100 @@ func TestIsMinioMetaBucketName(t *testing.T) {
 		}
 	}
 }
+
+// Tests RemoveStandardStorageClass method. Expectation is metadata map
+// should be cleared of x-amz-storage-class, if it is set to STANDARD
+func TestRemoveStandardStorageClass(t *testing.T) {
+	tests := []struct {
+		name     string
+		metadata map[string]string
+		want     map[string]string
+	}{
+		{
+			name:     "1",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD"},
+			want:     map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"},
+		},
+		{
+			name:     "2",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
+			want:     map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
+		},
+		{
+			name:     "3",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"},
+			want:     map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"},
+		},
+	}
+	for _, tt := range tests {
+		if got := removeStandardStorageClass(tt.metadata); !reflect.DeepEqual(got, tt.want) {
+			t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got)
+		}
+	}
+}
+
+// Tests CleanMetadata method. Expectation is metadata map
+// should be cleared of etag, md5Sum and x-amz-storage-class, if it is set to STANDARD
+func TestCleanMetadata(t *testing.T) {
+	tests := []struct {
+		name     string
+		metadata map[string]string
+		want     map[string]string
+	}{
+		{
+			name:     "1",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD"},
+			want:     map[string]string{"content-type": "application/octet-stream"},
+		},
+		{
+			name:     "2",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
+			want:     map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
+		},
+		{
+			name:     "3",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "md5Sum": "abcde"},
+			want:     map[string]string{"content-type": "application/octet-stream"},
+		},
+	}
+	for _, tt := range tests {
+		if got := cleanMetadata(tt.metadata); !reflect.DeepEqual(got, tt.want) {
+			t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got)
+		}
+	}
+}
+
+// Tests CleanMetadataKeys method. Expectation is metadata map
+// should be cleared of keys passed to CleanMetadataKeys method
+func TestCleanMetadataKeys(t *testing.T) {
+	tests := []struct {
+		name     string
+		metadata map[string]string
+		keys     []string
+		want     map[string]string
+	}{
+		{
+			name:     "1",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD", "md5": "abcde"},
+			keys:     []string{"etag", "md5"},
+			want:     map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "STANDARD"},
+		},
+		{
+			name:     "2",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY", "md5sum": "abcde"},
+			keys:     []string{"etag", "md5sum"},
+			want:     map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "REDUCED_REDUNDANCY"},
+		},
+		{
+			name:     "3",
+			metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "xyz": "abcde"},
+			keys:     []string{"etag", "xyz"},
+			want:     map[string]string{"content-type": "application/octet-stream"},
+		},
+	}
+	for _, tt := range tests {
+		if got := cleanMetadataKeys(tt.metadata, tt.keys...); !reflect.DeepEqual(got, tt.want) {
+			t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got)
+		}
+	}
+}
diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go
@@ -198,15 +198,15 @@ func printStorageClassInfoMsg(storageInfo StorageInfo) {
 func getStandardStorageClassInfoMsg(storageInfo StorageInfo) string {
 	var msg string
 	if maxDiskFailures := storageInfo.Backend.standardSCParity - storageInfo.Backend.OfflineDisks; maxDiskFailures >= 0 {
-		msg += fmt.Sprintf("Objects with Standard class can withstand [%d] drive failure(s).\n", maxDiskFailures)
+		msg += fmt.Sprintf("Objects with "+standardStorageClass+" class can withstand [%d] drive failure(s).\n", maxDiskFailures)
 	}
 	return msg
 }
 
 func getRRSStorageClassInfoMsg(storageInfo StorageInfo) string {
 	var msg string
 	if maxDiskFailures := storageInfo.Backend.rrSCParity - storageInfo.Backend.OfflineDisks; maxDiskFailures >= 0 {
-		msg += fmt.Sprintf("Objects with Reduced Redundancy class can withstand [%d] drive failure(s).\n", maxDiskFailures)
+		msg += fmt.Sprintf("Objects with "+reducedRedundancyStorageClass+" class can withstand [%d] drive failure(s).\n", maxDiskFailures)
 	}
 	return msg
 }

diff --git a/cmd/server-startup-msg_test.go b/cmd/server-startup-msg_test.go
@@ -172,7 +172,7 @@ func TestGetStandardStorageClassInfoMsg(t *testing.T) {
 				standardSCParity int
 				rrSCParity       int
 			}{Erasure, 15, 1, 5, 3},
-		}, "Objects with Standard class can withstand [4] drive failure(s).\n"},
+		}, "Objects with " + standardStorageClass + " class can withstand [4] drive failure(s).\n"},
 		{"2", StorageInfo{
 			Total: 30 * humanize.GiByte,
 			Free:  3 * humanize.GiByte,
@@ -183,7 +183,7 @@ func TestGetStandardStorageClassInfoMsg(t *testing.T) {
 				standardSCParity int
 				rrSCParity       int
 			}{Erasure, 10, 0, 5, 3},
-		}, "Objects with Standard class can withstand [5] drive failure(s).\n"},
+		}, "Objects with " + standardStorageClass + " class can withstand [5] drive failure(s).\n"},
 		{"3", StorageInfo{
 			Total: 15 * humanize.GiByte,
 			Free:  2 * humanize.GiByte,
@@ -194,7 +194,7 @@ func TestGetStandardStorageClassInfoMsg(t *testing.T) {
 				standardSCParity int
 				rrSCParity       int
 			}{Erasure, 12, 3, 6, 2},
-		}, "Objects with Standard class can withstand [3] drive failure(s).\n"},
+		}, "Objects with " + standardStorageClass + " class can withstand [3] drive failure(s).\n"},
 	}
 	for _, tt := range tests {
 		if got := getStandardStorageClassInfoMsg(tt.args); got != tt.want {
@@ -219,7 +219,7 @@ func TestGetRRSStorageClassInfoMsg(t *testing.T) {
 				standardSCParity int
 				rrSCParity       int
 			}{Erasure, 15, 1, 5, 3},
-		}, "Objects with Reduced Redundancy class can withstand [2] drive failure(s).\n"},
+		}, "Objects with " + reducedRedundancyStorageClass + " class can withstand [2] drive failure(s).\n"},
 		{"2", StorageInfo{
 			Total: 30 * humanize.GiByte,
 			Free:  3 * humanize.GiByte,
@@ -230,7 +230,7 @@ func TestGetRRSStorageClassInfoMsg(t *testing.T) {
 				standardSCParity int
 				rrSCParity       int
 			}{Erasure, 16, 0, 5, 3},
-		}, "Objects with Reduced Redundancy class can withstand [3] drive failure(s).\n"},
+		}, "Objects with " + reducedRedundancyStorageClass + " class can withstand [3] drive failure(s).\n"},
 		{"3", StorageInfo{
 			Total: 15 * humanize.GiByte,
 			Free:  2 * humanize.GiByte,
@@ -241,7 +241,7 @@ func TestGetRRSStorageClassInfoMsg(t *testing.T) {
 				standardSCParity int
 				rrSCParity       int
 			}{Erasure, 12, 3, 6, 5},
-		}, "Objects with Reduced Redundancy class can withstand [2] drive failure(s).\n"},
+		}, "Objects with " + reducedRedundancyStorageClass + " class can withstand [2] drive failure(s).\n"},
 	}
 	for _, tt := range tests {
 		if got := getRRSStorageClassInfoMsg(tt.args); got != tt.want {

diff --git a/cmd/xl-v1-metadata.go b/cmd/xl-v1-metadata.go
@@ -296,7 +296,7 @@ func (m xlMetaV1) ToObjectInfo(bucket, object string) ObjectInfo {
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of
 	// response headers. e.g, X-Minio-* or X-Amz-*.
-	objInfo.UserDefined = cleanMetaETag(m.Meta)
+	objInfo.UserDefined = cleanMetadata(m.Meta)
 
 	// Success.
 	return objInfo

diff --git a/cmd/xl-v1-object.go b/cmd/xl-v1-object.go
@@ -347,7 +347,7 @@ func (xl xlObjects) getObjectInfo(bucket, object string) (objInfo ObjectInfo, er
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of
 	// response headers. e.g, X-Minio-* or X-Amz-*.
-	objInfo.UserDefined = cleanMetaETag(xlMetaMap)
+	objInfo.UserDefined = cleanMetadata(xlMetaMap)
 
 	// Success.
 	return objInfo, nil

diff --git a/vendor/github.com/minio/minio-go/Makefile b/vendor/github.com/minio/minio-go/Makefile