Trim all etags with single function (minio#1197)

Add retry code to retry more errors

api-get-object.go

@@ -622,8 +622,7 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op
 	}
 
 	// Trim off the odd double quotes from ETag in the beginning and end.
-	md5sum := strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
-	md5sum = strings.TrimSuffix(md5sum, "\"")
+	md5sum := trimEtag(resp.Header.Get("ETag"))
 
 	// Parse the date.
 	date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified"))

api-list.go

@@ -23,7 +23,6 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
-	"strings"
 
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 )
@@ -155,6 +154,7 @@ func (c Client) listObjectsV2(bucketName, objectPrefix string, recursive, metada
 
 			// If contents are available loop through and send over channel.
 			for _, object := range result.Contents {
+				object.ETag = trimEtag(object.ETag)
 				select {
 				// Send object content.
 				case objectStatCh <- object:
@@ -724,8 +724,7 @@ func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsI
 		// Append to parts info.
 		for _, part := range listObjPartsResult.ObjectParts {
 			// Trim off the odd double quotes from ETag in the beginning and end.
-			part.ETag = strings.TrimPrefix(part.ETag, "\"")
-			part.ETag = strings.TrimSuffix(part.ETag, "\"")
+			part.ETag = trimEtag(part.ETag)
 			partsInfo[part.PartNumber] = part
 		}
 		// Keep part number marker, for the next iteration.

api-put-object-multipart.go

@@ -294,8 +294,7 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID
 	objPart.Size = size
 	objPart.PartNumber = partNumber
 	// Trim off the odd double quotes from ETag in the beginning and end.
-	objPart.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
-	objPart.ETag = strings.TrimSuffix(objPart.ETag, "\"")
+	objPart.ETag = trimEtag(resp.Header.Get("ETag"))
 	return objPart, nil
 }
 

api-put-object-streaming.go

@@ -407,8 +407,7 @@ func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string,
 
 	var objInfo ObjectInfo
 	// Trim off the odd double quotes from ETag in the beginning and end.
-	objInfo.ETag = strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
-	objInfo.ETag = strings.TrimSuffix(objInfo.ETag, "\"")
+	objInfo.ETag = trimEtag(resp.Header.Get("ETag"))
 	// A success here means data was written to server successfully.
 	objInfo.Size = size
 

api-stat.go

@@ -141,8 +141,7 @@ func (c Client) statObject(ctx context.Context, bucketName, objectName string, o
 	}
 
 	// Trim off the odd double quotes from ETag in the beginning and end.
-	md5sum := strings.TrimPrefix(resp.Header.Get("ETag"), "\"")
-	md5sum = strings.TrimSuffix(md5sum, "\"")
+	md5sum := trimEtag(resp.Header.Get("ETag"))
 
 	// Parse content length is exists
 	var size int64 = -1

api.go

@@ -560,6 +560,13 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque
 	var bodySeeker io.Seeker // Extracted seeker from io.Reader.
 	var reqRetry = MaxRetry  // Indicates how many times we can retry the request
 
+	defer func() {
+		if err != nil {
+			// close idle connections before returning, upon error.
+			c.httpClient.CloseIdleConnections()
+		}
+	}()
+
 	if metadata.contentBody != nil {
 		// Check if body is seekable then it is retryable.
 		bodySeeker, isRetryable = metadata.contentBody.(io.Seeker)

retry.go

@@ -91,31 +91,42 @@ func isHTTPReqErrorRetryable(err error) bool {
 	if err == nil {
 		return false
 	}
-	switch e := err.(type) {
-	case *url.Error:
-		switch e.Err.(type) {
-		case *net.DNSError, *net.OpError, net.UnknownNetworkError:
-			return true
+	// We need to figure if the error either a timeout
+	// or a non-temporary error.
+	e, ok := err.(net.Error)
+	if ok {
+		urlErr, ok := e.(*url.Error)
+		if ok {
+			switch urlErr.Err.(type) {
+			case *net.DNSError, *net.OpError, net.UnknownNetworkError:
+				return true
+			}
 		}
-		if strings.Contains(err.Error(), "Connection closed by foreign host") {
-			return true
-		} else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") {
-			// If error is - tlsHandshakeTimeoutError, retry.
-			return true
-		} else if strings.Contains(err.Error(), "i/o timeout") {
-			// If error is - tcp timeoutError, retry.
-			return true
-		} else if strings.Contains(err.Error(), "connection timed out") {
-			// If err is a net.Dial timeout, retry.
-			return true
-		} else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") {
-			// If error is transport connection broken, retry.
-			return true
-		} else if strings.Contains(err.Error(), "net/http: timeout awaiting response headers") {
-			// Retry errors due to server not sending the response before timeout
+		if e.Timeout() {
 			return true
 		}
 	}
+	if strings.Contains(err.Error(), "Connection closed by foreign host") {
+		return true
+	} else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") {
+		// If error is - tlsHandshakeTimeoutError, retry.
+		return true
+	} else if strings.Contains(err.Error(), "i/o timeout") {
+		// If error is - tcp timeoutError, retry.
+		return true
+	} else if strings.Contains(err.Error(), "connection timed out") {
+		// If err is a net.Dial timeout, retry.
+		return true
+	} else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") {
+		// If error is transport connection broken, retry.
+		return true
+	} else if strings.Contains(err.Error(), "net/http: timeout awaiting response headers") {
+		// Retry errors due to server not sending the response before timeout
+		return true
+	} else if strings.Contains(err.Error(), "connection reset by peer") {
+		// Retry errors due to connection reset by peer.
+		return true
+	}
 	return false
 }
 

utils.go

@@ -36,6 +36,11 @@ import (
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 )
 
+func trimEtag(etag string) string {
+	etag = strings.TrimPrefix(etag, "\"")
+	return strings.TrimSuffix(etag, "\"")
+}
+
 // xmlDecoder provide decoded value in xml.
 func xmlDecoder(body io.Reader, v interface{}) error {
 	d := xml.NewDecoder(body)