Use retry mechanism when initializing configuration (minio#6475)

Currently, one node in a cluster can fail to boot with the following error message:

```
ERROR Unable to initialize config system: Storage resources are insufficient for the write operation
```

  This happens when disks are formatted, read quorum is met but write
quorum is not met. In checkServerConfig(), a insufficient read quorum
error is replaced by errConfigNotFound, the code will generate a
new config json and try to save it, but it will fail because write
quorum is not met.

  Replacing read quorum with errConfigNotFound is also wrong because it
can lead, in rare cases, to overwrite the config set by the user.

  So, this commit adds a retry mechanism in configuration initialization
to retry only with read or write quorum errors.

  This commit will also fix the following cases:
 - Read quorum is lost just after the initialization of the object layer.
 - Write quorum not met when upgrading configuration version.

cmd/config.go

@@ -160,8 +160,8 @@ func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error {
 	}
 
 	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
-		// Convert ObjectNotFound, Quorum errors into errConfigNotFound
-		if isErrObjectNotFound(err) || isInsufficientReadQuorum(err) {
+		// Convert ObjectNotFound to errConfigNotFound
+		if isErrObjectNotFound(err) {
 			return errConfigNotFound
 		}
 		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
@@ -187,8 +187,8 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*by
 	var buffer bytes.Buffer
 	// Read entire content by setting size to -1
 	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
-		// Convert ObjectNotFound, IncompleteBody and Quorum errors into errConfigNotFound
-		if isErrObjectNotFound(err) || isErrIncompleteBody(err) || isInsufficientReadQuorum(err) {
+		// Convert ObjectNotFound and IncompleteBody errors into errConfigNotFound
+		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
 			return nil, errConfigNotFound
 		}
 
@@ -218,7 +218,32 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	if objAPI == nil {
 		return errInvalidArgument
 	}
-	return initConfig(objAPI)
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Initializing configuration needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	//  - Write quorum not met when upgrading configuration
+	//    version is needed.
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case _ = <-retryTimerCh:
+			err := initConfig(objAPI)
+			if err != nil {
+				if isInsufficientReadQuorum(err) || isInsufficientWriteQuorum(err) {
+					logger.Info("Waiting for configuration to be initialized..")
+					continue
+				}
+				return err
+			}
+
+			return nil
+		}
+	}
 }
 
 // NewConfigSys - creates new config system object.

cmd/object-api-errors.go

@@ -395,3 +395,9 @@ func isInsufficientReadQuorum(err error) bool {
 	_, ok := err.(InsufficientReadQuorum)
 	return ok
 }
+
+// isInsufficientWriteQuorum - Check if error type is InsufficientWriteQuorum.
+func isInsufficientWriteQuorum(err error) bool {
+	_, ok := err.(InsufficientWriteQuorum)
+	return ok
+}