Bring etcd support for bucket DNS federation

- Supports centralized `config.json`
- Supports centralized `bucket` service records
  for client lookups
- implement a new proxy forwarder

cmd/admin-handlers.go

@@ -818,8 +818,8 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 
 	// Update local credentials in memory.
 	globalServerConfig.SetCredential(creds)
-	if err = globalServerConfig.Save(); err != nil {
-		writeErrorResponseJSON(w, ErrInternalError, r.URL)
+	if err = globalServerConfig.Save(getConfigFile()); err != nil {
+		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
 

cmd/api-errors.go

@@ -17,10 +17,12 @@
 package cmd
 
 import (
+	"context"
 	"encoding/xml"
 	"fmt"
 	"net/http"
 
+	"github.com/coreos/etcd/client"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
@@ -864,34 +866,42 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminInvalidAccessKey
 	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
-	}
-
-	if apiErr != ErrNone {
-		// If there was a match in the above switch case.
-		return apiErr
-	}
-
-	switch err { // SSE errors
+	// SSE errors
 	case errInsecureSSERequest:
-		return ErrInsecureSSECustomerRequest
+		apiErr = ErrInsecureSSECustomerRequest
 	case errInvalidSSEAlgorithm:
-		return ErrInvalidSSECustomerAlgorithm
+		apiErr = ErrInvalidSSECustomerAlgorithm
 	case errInvalidSSEKey:
-		return ErrInvalidSSECustomerKey
+		apiErr = ErrInvalidSSECustomerKey
 	case errMissingSSEKey:
-		return ErrMissingSSECustomerKey
+		apiErr = ErrMissingSSECustomerKey
 	case errMissingSSEKeyMD5:
-		return ErrMissingSSECustomerKeyMD5
+		apiErr = ErrMissingSSECustomerKeyMD5
 	case errSSEKeyMD5Mismatch:
-		return ErrSSECustomerKeyMD5Mismatch
+		apiErr = ErrSSECustomerKeyMD5Mismatch
 	case errObjectTampered:
-		return ErrObjectTampered
+		apiErr = ErrObjectTampered
 	case errEncryptedObject:
-		return ErrSSEEncryptedObject
+		apiErr = ErrSSEEncryptedObject
 	case errInvalidSSEParameters:
-		return ErrInvalidSSECustomerParameters
+		apiErr = ErrInvalidSSECustomerParameters
 	case errSSEKeyMismatch:
-		return ErrAccessDenied // no access without correct key
+		apiErr = ErrAccessDenied // no access without correct key
+	case context.Canceled, context.DeadlineExceeded:
+		apiErr = ErrOperationTimedOut
+	}
+
+	if apiErr != ErrNone {
+		// If there was a match in the above switch case.
+		return apiErr
+	}
+
+	// etcd specific errors, a key is always a bucket for us return
+	// ErrNoSuchBucket in such a case.
+	if e, ok := err.(*client.Error); ok {
+		if e.Code == client.ErrorCodeKeyNotFound {
+			return ErrNoSuchBucket
+		}
 	}
 
 	switch err.(type) {

cmd/bucket-handlers.go

@@ -28,6 +28,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/coreos/etcd/client"
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
@@ -158,11 +159,28 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	// Invoke the list buckets.
-	bucketsInfo, err := listBuckets(ctx)
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
+	// If etcd, dns federation configured list buckets from etcd.
+	var bucketsInfo []BucketInfo
+	if globalDNSConfig != nil {
+		dnsBuckets, err := globalDNSConfig.List()
+		if err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+		for _, dnsRecord := range dnsBuckets {
+			bucketsInfo = append(bucketsInfo, BucketInfo{
+				Name:    dnsRecord.Key,
+				Created: dnsRecord.CreationDate,
+			})
+		}
+	} else {
+		// Invoke the list buckets.
+		var err error
+		bucketsInfo, err = listBuckets(ctx)
+		if err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
 	}
 
 	// Generate response.
@@ -353,12 +371,33 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+	if globalDNSConfig != nil {
+		if _, err := globalDNSConfig.Get(bucket); err != nil {
+			if client.IsKeyNotFound(err) {
+				// Proceed to creating a bucket.
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
+					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					return
+				}
+				if err = globalDNSConfig.Put(bucket); err != nil {
+					objectAPI.DeleteBucket(ctx, bucket)
+					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					return
+				}
+
+				// Make sure to add Location information here only for bucket
+				w.Header().Set("Location", getObjectLocation(r, globalDomainName, bucket, ""))
+
+				writeSuccessResponseHeadersOnly(w)
+				return
+			}
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+
+		}
+		writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL)
 		return
 	}
-	defer bucketLock.Unlock()
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
@@ -660,6 +699,15 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		logger.LogIf(ctx, nerr.Err)
 	}
 
+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(bucket); err != nil {
+			// Deleting DNS entry failed, attempt to create the bucket again.
+			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+	}
+
 	// Write success response.
 	writeSuccessNoContent(w)
 }

cmd/common-main.go

@@ -18,15 +18,19 @@ package cmd
 
 import (
 	"errors"
+	"net"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
+	etcdc "github.com/coreos/etcd/client"
 	"github.com/minio/cli"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
 )
 
 // Check for updates and print a notification message
@@ -42,6 +46,18 @@ func checkUpdate(mode string) {
 }
 
 func initConfig() {
+	if globalEtcdClient != nil {
+		if err := loadConfig(); err != nil {
+			if etcdc.IsKeyNotFound(err) {
+				logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.")
+				logger.Info("Created minio configuration file successfully at", globalEtcdClient.Endpoints())
+			} else {
+				logger.FatalIf(err, "Unable to load config version: '%s'.", serverConfigVersion)
+			}
+		}
+		return
+	}
+
 	// Config file does not exist, we create it fresh and return upon success.
 	if isFile(getConfigFile()) {
 		logger.FatalIf(migrateConfig(), "Config migration failed")
@@ -125,6 +141,31 @@ func handleCommonEnvVars() {
 
 	globalDomainName, globalIsEnvDomainName = os.LookupEnv("MINIO_DOMAIN")
 
+	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
+	if ok {
+		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
+		var err error
+		globalEtcdClient, err = etcdc.New(etcdc.Config{
+			Endpoints: etcdEndpoints,
+			Transport: &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+				Dial: (&net.Dialer{
+					Timeout:   30 * time.Second,
+					KeepAlive: 30 * time.Second,
+				}).Dial,
+				TLSHandshakeTimeout: 10 * time.Second,
+			},
+		})
+		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+	}
+
+	globalDomainIP = os.Getenv("MINIO_DOMAIN_IP")
+	if globalDomainName != "" && globalDomainIP != "" && globalEtcdClient != nil {
+		var err error
+		globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIP, globalMinioPort, globalEtcdClient)
+		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainName)
+	}
+
 	if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" {
 		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
 		if err != nil {

cmd/config-current.go

@@ -128,9 +128,9 @@ func (s *serverConfig) GetCacheConfig() CacheConfig {
 }
 
 // Save config.
-func (s *serverConfig) Save() error {
+func (s *serverConfig) Save(configFile string) error {
 	// Save config file.
-	return quick.Save(getConfigFile(), s)
+	return quick.Save(configFile, s)
 }
 
 // Returns the string describing a difference with the given
@@ -230,7 +230,10 @@ func newServerConfig() *serverConfig {
 // found, otherwise use default parameters
 func newConfig() error {
 	// Initialize server config.
-	srvCfg := newServerConfig()
+	srvCfg, err := newQuickConfig(newServerConfig())
+	if err != nil {
+		return err
+	}
 
 	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
@@ -269,7 +272,29 @@ func newConfig() error {
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return globalServerConfig.Save()
+	return globalServerConfig.Save(getConfigFile())
+}
+
+// newQuickConfig - initialize a new server config, with an allocated
+// quick.Config interface.
+func newQuickConfig(srvCfg *serverConfig) (*serverConfig, error) {
+	if globalEtcdClient == nil {
+		qcfg, err := quick.NewLocalConfig(srvCfg)
+		if err != nil {
+			return nil, err
+		}
+
+		srvCfg.Config = qcfg
+		return srvCfg, nil
+	}
+
+	qcfg, err := quick.NewEtcdConfig(srvCfg, globalEtcdClient)
+	if err != nil {
+		return nil, err
+	}
+
+	srvCfg.Config = qcfg
+	return srvCfg, nil
 }
 
 // getValidConfig - returns valid server configuration
@@ -279,7 +304,14 @@ func getValidConfig() (*serverConfig, error) {
 		Browser: true,
 	}
 
-	if _, err := quick.Load(getConfigFile(), srvCfg); err != nil {
+	var err error
+	srvCfg, err = newQuickConfig(srvCfg)
+	if err != nil {
+		return nil, err
+	}
+
+	configFile := getConfigFile()
+	if err = srvCfg.Load(configFile); err != nil {
 		return nil, err
 	}
 

cmd/config-current_test.go

@@ -50,7 +50,7 @@ func TestServerConfig(t *testing.T) {
 	}
 
 	// Attempt to save.
-	if err := globalServerConfig.Save(); err != nil {
+	if err := globalServerConfig.Save(getConfigFile()); err != nil {
 		t.Fatalf("Unable to save updated config file %s", err)
 	}
 

cmd/config-versions.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/quick"
 )
 
 /////////////////// Config V1 ///////////////////
@@ -585,6 +586,8 @@ type serverConfigV22 struct {
 // IMPORTANT NOTE: When updating this struct make sure that
 // serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV23 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
 	Version string `json:"version"`
 
 	// S3 API configuration.