Merge pull request cesanta#79 from carsonoid/mongo_failover_retry

Retry connection to mongo when we get a EOF error.

auth_server/authn/mongo_auth.go

@@ -17,7 +17,10 @@
 package authn
 
 import (
+	"errors"
 	"fmt"
+	"io"
+	"time"
 
 	"github.com/cesanta/docker_auth/auth_server/mgo_session"
 	"github.com/golang/glog"
@@ -80,6 +83,20 @@ func NewMongoAuth(c *MongoAuthConfig) (*MongoAuth, error) {
 }
 
 func (mauth *MongoAuth) Authenticate(account string, password PasswordString) (bool, error) {
+	for true {
+		result, err := mauth.authenticate(account, password)
+		if err == io.EOF {
+			glog.Warningf("EOF error received from Mongo. Retrying connection")
+			time.Sleep(time.Second)
+			continue
+		}
+		return result, err
+	}
+
+	return false, errors.New("Unable to communicate with Mongo.")
+}
+
+func (mauth *MongoAuth) authenticate(account string, password PasswordString) (bool, error) {
 	// Copy our session
 	tmp_session := mauth.session.Copy()
 	// Close up when we are done

auth_server/authz/acl_mongo.go

@@ -3,13 +3,13 @@ package authz
 import (
 	"errors"
 	"fmt"
-	"sync"
-	"time"
-
 	"github.com/cesanta/docker_auth/auth_server/mgo_session"
 	"github.com/golang/glog"
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
+	"io"
+	"sync"
+	"time"
 )
 
 type MongoACL []MongoACLEntry
@@ -115,13 +115,20 @@ func (ma *aclMongoAuthorizer) continuouslyUpdateACLCache() {
 		aclAge := time.Now().Sub(ma.lastCacheUpdate)
 		glog.V(2).Infof("Updating ACL at %s (ACL age: %s. CacheTTL: %s)", tick, aclAge, ma.config.CacheTTL)
 
-		err := ma.updateACLCache()
-		if err == nil {
-			continue
+		for true {
+			err := ma.updateACLCache()
+			if err == nil {
+				break
+			} else if err == io.EOF {
+				glog.Warningf("EOF error received from Mongo. Retrying connection")
+				time.Sleep(time.Second)
+				continue
+			} else {
+				glog.Errorf("Failed to update ACL. ERROR: %s", err)
+				glog.Warningf("Using stale ACL (Age: %s, TTL: %s)", aclAge, ma.config.CacheTTL)
+				break
+			}
 		}
-
-		glog.Errorf("Failed to update ACL. ERROR: %s", err)
-		glog.Warningf("Using stale ACL (Age: %s, TTL: %s)", aclAge, ma.config.CacheTTL)
 	}
 }